diff --git a/.gitignore b/.gitignore index d35eb70..25e9882 100644 --- a/.gitignore +++ b/.gitignore @@ -6,8 +6,8 @@ *.log *.txt *.xls -*.xlsx *.csv# +*.exe *.doc *.docx /bin/__pycache__ diff --git a/README.markdown b/README.markdown index 52320b6..707228a 100644 --- a/README.markdown +++ b/README.markdown @@ -6,24 +6,26 @@ This project contains tools for decoding, converting, and visualizing telemetry - **Project structure** - **Requirements** -- ** Linux Setup** -- ** Windows 10 Setup** +- **Linux Setup** +- **Windows 10 Setup** - **Using the tools** - *Plot all ASOTR data* - *Plot ASOTR data in specified date borders (for MVN reports)* - **Contacts** **Note 1**: \ - path where is mvn_flight scripts is cloned from heagit -**Note 2**: \ - path with raw MVN ASOTR data -**Note 3**: \ - path with raw MVN data +**Note 2**: \ - path with raw MVN ASOTR data ## Project structure -> `/bin/` - ready-to-use binary files and python scripts for assembling, extracting and plotting telemetry data ---> `/bin/asotr_unzip_plot.sh` or `/bin/asotr_unzip_plot.bat` — scripts to automate the unpack-convert-plot chain for Linux/Windows. +--> `/bin/asotr_unzip_plot.sh` or `/bin/asotr_unzip_plot.bat` — scripts to automate the unpack-convert-plot chain ASOTR data (Linux/Windows). --> `/bin/plot_flight_borders.py` — Python script to generate plots from CSV data. --> `/asotr_csv/` — binary parser to collect data from raw ASOTR data to CSV. This parser written using the Rust language. For more informatin see documentation in `/asotr_csv/README.markdown`). --> `/data/` — folder containing processed CSV files. --> `/plots/` — folder where output plots are saved. +--> `/bin/plot_flight_borders.sh` or `/bin/plot_flight_borders.bat` - scripts to automate plot generations for MVN reports +--> `/bin/tm_brd_parser.py` - python script for demostration processing BRD telemetry data (1 Hz evolution) +--> `/bin/tm_wheel_parser.py` - python script for demostration processing BRD wheel telemetry data +-> `/asotr_csv/` — binary parser to collect data from raw ASOTR data to CSV. This parser written using the Rust language. For more informatin see documentation in `/asotr_csv/README.markdown`). +-> `/data/` — folder containing processed CSV files. +-> `/plots/` — folder where output plots are saved. ## Requirements - Python version 3.10 or upper. For checking type: @@ -31,21 +33,11 @@ This project contains tools for decoding, converting, and visualizing telemetry python3 --version ``` -Python dependencies: -```bash -pip install matplotlib pandas -``` - - Rust compiler version 1.83.0 or upper. For checking type: ```bash rustc --version ``` -Additional tools (Linux only): -```bash -sudo apt install unzip git -``` - ## Linux Setup 1. Clone the repo to your computer: ```bash @@ -56,7 +48,7 @@ git clone http://heagit.cosmos.ru/gamkov/mvn_flight.git ```bash sudo apt update sudo apt install python3 python3-pip -pip3 install matplotlib pandas +pip3 install matplotlib pandas openpyxl scipy ``` 1. Install Rust compiler (if you do not have): @@ -74,7 +66,7 @@ git clone http://heagit.cosmos.ru/gamkov/mvn_flight.git 3. Open CMD and install python dependencies: ```cmd -pip install matplotlib pandas +pip install matplotlib pandas openpyxl scipy ``` 4. Install Rust compiler (if you do not have): @@ -87,23 +79,41 @@ For more detailed information you can go to: https://doc.rust-lang.ru/book/ch01- ### Plot all ASOTR data 1. Donwload data from science data server to directory \. If you don't have MVN data, you might download it from server with science SRG data (IP: 193.232.11.95). -For questions about downloading science data contact Shtykovsky A. (a.shtykovsky@cosmos.ru) or Chelovekov I. (chelovekov@cosmos.ru) +For questions about downloading science data contact Shtykovsky A. (a.shtykovsky@cosmos.ru) or Chelovekov I. (chelovekov@cosmos.ru) + +Linux (for example): +``` +sftp username@IP_address +cd mvn/data/data/2025 +get -r 20250101* +``` + +Windows: +``` +Open filezilla +connect to sft server +get data from: /export/home/user_name/mvn/data/data/2025 +``` + + 2. Parse all raw data from ASOTR into csv files and plot all csv data: Linux: ```bash -./bin/asotr_unzip_plot.sh / +cd bin +./asotr_unzip_plot.sh / ``` Windows: ```cmd -.\bin\asotr_unzip_plot.bat \ +cd bin +asotr_unzip_plot.bat \ ``` csv data will be in directory: ```cmd -./data/asotr/ +/data/asotr/ ``` ### Plot ASOTR data in specified date borders (for MVN reports) @@ -113,12 +123,14 @@ csv data will be in directory: Linux: ```bash -./bin/plot_flight_borders.sh ../data/asotr/ 10.03.2025_00:00:00 23.04.2025_23:59:59 +cd bin +./plot_flight_borders.sh ../data/asotr/ 10.03.2025_00:00:00 23.04.2025_23:59:59 ``` Windows: ```cmd -.\bin\plot_flight_borders.bat ..\data\asotr\ 10.03.2025_00:00:00 23.04.2025_23:59:59 +cd bin +plot_flight_borders.bat ..\data\asotr\ 10.03.2025_00:00:00 23.04.2025_23:59:59 ``` where: diff --git a/asotr_csv/src/main.rs b/asotr_csv/src/main.rs index 4031ea8..b9ef8ce 100644 --- a/asotr_csv/src/main.rs +++ b/asotr_csv/src/main.rs @@ -174,7 +174,7 @@ pub mod asotr_data { let mut fname = String::new(); let msg_prev = format!("Error parsing filename {}:", filename_full); - match filename_full.rfind('/') { + match filename_full.rfind('/').or_else(|| filename_full.rfind('\\')) { Some(val) => { fname = (filename_full[val+1..filename_full.len()]).to_string(); } _ => { fname = filename_full.clone(); } } diff --git a/asotr_csv/target/.rustc_info.json b/asotr_csv/target/.rustc_info.json index 06ac6b8..cebc56f 100644 --- a/asotr_csv/target/.rustc_info.json +++ b/asotr_csv/target/.rustc_info.json @@ -1 +1 @@ -{"rustc_fingerprint":2742313010855374649,"outputs":{"15729799797837862367":{"success":true,"status":"","code":0,"stdout":"___\nlib___.rlib\nlib___.so\nlib___.so\nlib___.a\nlib___.so\n/home/danila/.rustup/toolchains/stable-x86_64-unknown-linux-gnu\noff\npacked\nunpacked\n___\ndebug_assertions\npanic=\"unwind\"\nproc_macro\ntarget_abi=\"\"\ntarget_arch=\"x86_64\"\ntarget_endian=\"little\"\ntarget_env=\"gnu\"\ntarget_family=\"unix\"\ntarget_feature=\"fxsr\"\ntarget_feature=\"sse\"\ntarget_feature=\"sse2\"\ntarget_has_atomic=\"16\"\ntarget_has_atomic=\"32\"\ntarget_has_atomic=\"64\"\ntarget_has_atomic=\"8\"\ntarget_has_atomic=\"ptr\"\ntarget_os=\"linux\"\ntarget_pointer_width=\"64\"\ntarget_vendor=\"unknown\"\nunix\n","stderr":""},"4614504638168534921":{"success":true,"status":"","code":0,"stdout":"rustc 1.83.0 (90b35a623 2024-11-26)\nbinary: rustc\ncommit-hash: 90b35a6239c3d8bdabc530a6a0816f7ff89a0aaf\ncommit-date: 2024-11-26\nhost: x86_64-unknown-linux-gnu\nrelease: 1.83.0\nLLVM version: 19.1.1\n","stderr":""}},"successes":{}} \ No newline at end of file +{"rustc_fingerprint":14808889899039181664,"outputs":{"7971740275564407648":{"success":true,"status":"","code":0,"stdout":"___.exe\nlib___.rlib\n___.dll\n___.dll\n___.lib\n___.dll\nC:\\Users\\danil\\.rustup\\toolchains\\stable-x86_64-pc-windows-msvc\npacked\n___\ndebug_assertions\npanic=\"unwind\"\nproc_macro\ntarget_abi=\"\"\ntarget_arch=\"x86_64\"\ntarget_endian=\"little\"\ntarget_env=\"msvc\"\ntarget_family=\"windows\"\ntarget_feature=\"cmpxchg16b\"\ntarget_feature=\"fxsr\"\ntarget_feature=\"sse\"\ntarget_feature=\"sse2\"\ntarget_feature=\"sse3\"\ntarget_has_atomic=\"128\"\ntarget_has_atomic=\"16\"\ntarget_has_atomic=\"32\"\ntarget_has_atomic=\"64\"\ntarget_has_atomic=\"8\"\ntarget_has_atomic=\"ptr\"\ntarget_os=\"windows\"\ntarget_pointer_width=\"64\"\ntarget_vendor=\"pc\"\nwindows\n","stderr":""},"17747080675513052775":{"success":true,"status":"","code":0,"stdout":"rustc 1.87.0 (17067e9ac 2025-05-09)\nbinary: rustc\ncommit-hash: 17067e9ac6d7ecb70e50f92c1944e545188d2359\ncommit-date: 2025-05-09\nhost: x86_64-pc-windows-msvc\nrelease: 1.87.0\nLLVM version: 20.1.1\n","stderr":""}},"successes":{}} \ No newline at end of file diff --git a/bin/asotr.py b/bin/asotr.py index 0607446..0b89898 100644 --- a/bin/asotr.py +++ b/bin/asotr.py @@ -232,12 +232,10 @@ def get_data(path, asotr_kit, start_date, end_date, time_accuracy): fname = [path + fname_temp, path + fname_tempSet, path + fname_pow] - dateparse = lambda x: datetime.strptime(x, "%d.%m.%Y %H:%M:%S.%f") - try: - data = [ pd.read_csv(fname[0], sep=";", parse_dates=["timestamp"], date_parser=dateparse), - pd.read_csv(fname[1], sep=";", parse_dates=["timestamp"], date_parser=dateparse), - pd.read_csv(fname[2], sep=";", parse_dates=["timestamp"], date_parser=dateparse),] + data = [ pd.read_csv(fname[0], sep=";", parse_dates=["timestamp"], date_format="%d.%m.%Y %H:%M:%S.%f"), + pd.read_csv(fname[1], sep=";", parse_dates=["timestamp"], date_format="%d.%m.%Y %H:%M:%S.%f"), + pd.read_csv(fname[2], sep=";", parse_dates=["timestamp"], date_format="%d.%m.%Y %H:%M:%S.%f"),] except FileNotFoundError: print(f'Error opening file: one (or all) file not found in directory: \n{fname}') return diff --git a/bin/asotr_unzip_plot.bat b/bin/asotr_unzip_plot.bat index e073082..5011894 100755 --- a/bin/asotr_unzip_plot.bat +++ b/bin/asotr_unzip_plot.bat @@ -12,10 +12,10 @@ copy "..\asotr_csv\target\release\asotr_csv.exe" .\ set "path_=%~1" REM unpacking recursively archive using Python script -python recursive_unpack_targz.py "%path_%" +python recursive_unpack_targz.py %path_% REM run parser -asotr_csv.exe -d "%path_%" +asotr_csv.exe -d %path_% REM plot data python plot_asotr_flight_all.py diff --git a/bin/brd_wheel_1Hz_parser.py b/bin/brd_wheel_1Hz_parser.py deleted file mode 100644 index b4bbfeb..0000000 --- a/bin/brd_wheel_1Hz_parser.py +++ /dev/null @@ -1,140 +0,0 @@ -import pandas as pd -import os -import re -from pathlib import Path -import matplotlib.pyplot as plt -from datetime import datetime, timedelta - -tstamp_s = '%d.%m.%Y %H:%M:%S.%f' -ox_dtime_format = '%d.%m.%Y %H:%M' - -path_itog_brd_data = '../data/brd_data/' - -class PathFileNotFound(Exception): - pass - -def find_required_files(root_dir, pattern): - result = [] - for dirpath, _, filenames in os.walk(root_dir): - for filename in filenames: - match = re.match(pattern, filename) - if match: - result.append(dirpath + '/' + filename) - - if len(result) == 0: - raise PathFileNotFound(f'error: check that the path is correct ({root_dir}) or files pattern is correct ({pattern})') - - return sorted(result) - -def read_files_into_df(fname_list, column_list, dtype_columns={}): - data_itog = pd.DataFrame() - epoch_start = pd.Timestamp('2000-01-01') - - for fname in fname_list: - data = pd.read_csv(fname, sep=r'\s+', dtype=str) - data = data.dropna() - data = data[column_list] - - if 'TIME' in column_list: - # convert TIME value to human-readable timestamp (sinse epoch 01.01.2000) - time = data['TIME'].astype(float) - tstamp = epoch_start + pd.to_timedelta(time, unit='s') - timestamp = tstamp.dt.strftime(tstamp_s) - data['timestamp'] = timestamp - - # clear dataframe rows where time value == 0 - data['time'] = time - data_clear = data.query('time != 0.0') - - data_itog = pd.concat([data_itog, data_clear], ignore_index=True) - - return data_itog - - -def collect_tm_brd_files(root_dir_tm_data, column_list, column_list_itog): - patterns_tm = [r'mvn_tm_brd01_(.*)', r'mvn_tm_brd02_(.*)', r'mvn_tm_brd03_(.*)', - r'mvn_tm_brd04_(.*)'] - - for pattern in patterns_tm: - fname = path_itog_brd_data + pattern[:12] + '.csv' - try: - found_files = find_required_files(root_dir_tm_data, pattern) - data = read_files_into_df(found_files, column_list, dtype_columns={11: float}) - except KeyError as e: - print(f'error in collect_tm_brd_files: the specified column name was not found in the data file (path: {root_dir_tm_data}) ({e})') - break - except Exception as e: - print(f'error in collect_tm_brd_files: {e}') - break - - data.to_csv(fname, index=False, sep=';', columns=column_list_itog, encoding='utf-8-sig') - print('data saved: ' + fname) - - -def collect_tm_brd_wheel_data(root_dir_wheel_data, column_list, column_list_itog): - patterns_wheel = [r'mvn_wheel_brd01_(.*)', r'mvn_wheel_brd02_(.*)', r'mvn_wheel_brd03_(.*)', - r'mvn_wheel_brd04_(.*)'] - - for pattern in patterns_wheel: - fname = path_itog_brd_data + pattern[:15] + '.csv' - try: - found_files = find_required_files(root_dir_wheel_data, pattern) - data = read_files_into_df(found_files, column_list, dtype_columns={0: float, 1: int}) - except KeyError as e: - print(f'error in collect_tm_brd_wheel_data: the specified column name was not found in the data file (path: {root_dir_tm_data}) ({e})') - break - except Exception as e: - print(f'error in collect_tm_brd_wheel_data: {e}') - break - - mask = data['STATE'] == '0' - data = data[mask] - data.to_csv(fname, index=False, sep=';', columns=column_list_itog, encoding='utf-8-sig') - print('data saved: ' + fname) - - -### collect raw tm brd data into one file for each brd ### - -root_dir_tm_data = '/home/danila/Danila/work/MVN/flight/brd_data/arch_for_MB/archive_tm_data_txt/' -column_list = ['TIME', 'PER_1Hz', 'ST_HV'] -column_list_itog = ['TIME', 'timestamp', 'PER_1Hz', 'ST_HV'] - -collect_tm_brd_files(root_dir_tm_data, column_list, column_list_itog) - - -### collect raw tm wheel data into one file for each brd ### - -root_dir_wheel_data = '/home/danila/Danila/work/MVN/flight/brd_data/arch_for_MB/archive_wheel_data_txt/' -column_list = ['TIME', 'STATE'] -column_list_itog = ['TIME', 'timestamp', 'STATE'] - -collect_tm_brd_wheel_data(root_dir_wheel_data, column_list, column_list_itog) - - -## plot 'evolution' 1 Hz from tm brd data - -fname = path_itog_brd_data + 'mvn_tm_brd01.csv' -dateparse = lambda x: datetime.strptime(x, tstamp_s) -df = pd.read_csv(fname, sep=';', parse_dates=['timestamp'], date_parser=dateparse) - -plt.plot(df['timestamp'], df['PER_1Hz'], '.') -plt.show() - - -## parse and plot wheel csv data - -border_clr_wheel = 2 -fname = path_itog_brd_data + 'mvn_wheel_brd01.csv' -wheel_df = pd.read_csv(fname, sep=';') -wheel_df['TIME_diff'] = wheel_df['TIME'].diff() -median_tdiff = wheel_df['TIME_diff'].median() - -wheel_df_clear = wheel_df[(wheel_df['TIME_diff'] > median_tdiff - border_clr_wheel) & - (wheel_df['TIME_diff'] < median_tdiff + border_clr_wheel)] - -wheel_df_peaks = wheel_df[(wheel_df['TIME_diff'] <= median_tdiff - border_clr_wheel) | - (wheel_df['TIME_diff'] >= median_tdiff + border_clr_wheel)] - - -plt.plot(wheel_df_clear['TIME'], wheel_df_clear['TIME_diff']) -plt.show() diff --git a/bin/plot_asotr_flight_all.py b/bin/plot_asotr_flight_all.py index eb8fd9a..63c16fe 100644 --- a/bin/plot_asotr_flight_all.py +++ b/bin/plot_asotr_flight_all.py @@ -31,21 +31,18 @@ width=[1, 1, 1, 1, 1, 1] marker = ['-', '-', '-', '-', '--', '-']; width_arr = [1, 0.5, 0.2, 0.1, 1, 1] -dateparse = lambda x: datetime.strptime(x, "%d.%m.%Y %H:%M:%S.%f") -dparse_b = lambda x: datetime.strptime(x, '%Y-%m-%d %H:%M:%S') - data_b = pd.read_excel(fname_B, sheet_name=0, usecols=[0,1,2], header=4, names=['turn_num', 'beta_angle', 'timestamp'], parse_dates=['timestamp'], - date_parser=dparse_b) + date_format='%Y-%m-%d %H:%M:%S') fname = [path + fname, path + fname_pow] -data = [pd.read_csv(fname[0], sep=';', parse_dates=['timestamp'], date_parser=dateparse), - pd.read_csv(fname[1], sep=';', parse_dates=['timestamp'], date_parser=dateparse)] +data = [pd.read_csv(fname[0], sep=';', parse_dates=['timestamp'], date_format="%d.%m.%Y %H:%M:%S.%f"), + pd.read_csv(fname[1], sep=';', parse_dates=['timestamp'], date_format="%d.%m.%Y %H:%M:%S.%f")] ch= [[], [], [], [], [], []] ch_signs = ["temp", "pow"] @@ -80,7 +77,7 @@ if plot_windows == 1: ax.tick_params(axis="both", width=1, labelsize=font) ax.grid(visible=True, linestyle = 'dotted') - ax.set_ylabel('Температура, $^\circ$C', fontsize=font) + ax.set_ylabel(r"Температура, $^\circ$C", fontsize=font) ax.set_xlabel('Время', fontsize=font) ax.legend(fontsize=font) @@ -117,7 +114,7 @@ elif plot_windows == 2: ax1.tick_params(axis="both", width=1, labelsize=font) ax1.grid(visible=True, linestyle = 'dotted') - ax1.set_ylabel('Температура, $^\circ$C', fontsize=font) + ax1.set_ylabel(r"Температура, $^\circ$C", fontsize=font) ax1.set_xlabel('Время', fontsize=font) ax1.legend(fontsize=font, loc='lower right') diff --git a/bin/plot_flight_borders.bat b/bin/plot_flight_borders.bat index 25eb6e7..652b4dc 100755 --- a/bin/plot_flight_borders.bat +++ b/bin/plot_flight_borders.bat @@ -6,16 +6,16 @@ IF "%~2"=="" ( goto :EOF ) -set "path_csv_data=..\data\asotr" +set "path_csv_data=..\data\asotr\" set "begin=%~1" set "end=%~2" REM run Python-script with parameters -python plot_flight_borders.py -s "%path_csv_data%" -c 111100 -a 01 -b %begin% -e %end% -python plot_flight_borders.py -s "%path_csv_data%" -c 001000 -a 01 -b %begin% -e %end% -python plot_flight_borders.py -s "%path_csv_data%" -c 000011 -a 01 -b %begin% -e %end% -python plot_flight_borders.py -s "%path_csv_data%" -c 111100 -a 02 -b %begin% -e %end% -python plot_flight_borders.py -s "%path_csv_data%" -c 010100 -a 02 -b %begin% -e %end% -python plot_flight_borders.py -s "%path_csv_data%" -c 010000 -a 02 -b %begin% -e %end% -python plot_flight_borders.py -s "%path_csv_data%" -c 000100 -a 02 -b %begin% -e %end% -python plot_flight_borders.py -s "%path_csv_data%" -c 000011 -a 02 -b %begin% -e %end% +python plot_flight_borders.py -s %path_csv_data% -c 111100 -a 01 -b %begin% -e %end% +python plot_flight_borders.py -s %path_csv_data% -c 001000 -a 01 -b %begin% -e %end% +python plot_flight_borders.py -s %path_csv_data% -c 000011 -a 01 -b %begin% -e %end% +python plot_flight_borders.py -s %path_csv_data% -c 111100 -a 02 -b %begin% -e %end% +python plot_flight_borders.py -s %path_csv_data% -c 010100 -a 02 -b %begin% -e %end% +python plot_flight_borders.py -s %path_csv_data% -c 010000 -a 02 -b %begin% -e %end% +python plot_flight_borders.py -s %path_csv_data% -c 000100 -a 02 -b %begin% -e %end% +python plot_flight_borders.py -s %path_csv_data% -c 000011 -a 02 -b %begin% -e %end% diff --git a/bin/plot_flight_borders.py b/bin/plot_flight_borders.py index 31c99f3..1b81a65 100644 --- a/bin/plot_flight_borders.py +++ b/bin/plot_flight_borders.py @@ -15,7 +15,7 @@ def plot_asotr_borders(path_with_data, ch, asotr_kit, begin, end, font=14, cmd=0 plot_windows = 2 channels = list(map(int, ch)) - pict_name = (f'../plots/reports/ASOTR{asotr_kit}_flight_T_P_{asotr.convert_to_str(channels)}_{begin[0:5].replace(".", "")}_{end[0:5].replace(".", "")}_{end[6:]}.png') + pict_name = (f'../plots/reports/ASOTR{asotr_kit}_flight_T_P_{asotr.convert_to_str(channels)}_{begin[0:5].replace(".", "")}_{end[0:5].replace(".", "")}.png') plot_task = {"temp": 1, "temp_set": 1, "pow": 1} ox_dtime_format = "%d.%m.%Y" @@ -57,7 +57,7 @@ def plot_asotr_borders(path_with_data, ch, asotr_kit, begin, end, font=14, cmd=0 ax.tick_params(axis="both", width=1, labelsize=font) ax.grid(visible=True, linestyle="dotted") - ax.set_ylabel("Температура, $^\circ$C", fontsize=font) + ax.set_ylabel(r"Температура, $^\circ$C", fontsize=font) ax.set_xlabel("Время", fontsize=font) ax.legend(fontsize=font) @@ -131,7 +131,7 @@ def plot_asotr_borders(path_with_data, ch, asotr_kit, begin, end, font=14, cmd=0 ax1.tick_params(axis="both", width=1, labelsize=font) ax1.grid(visible=True, linestyle="dotted") - ax1.set_ylabel("Температура, $^\circ$C", fontsize=font) + ax1.set_ylabel(r"Температура, $^\circ$C", fontsize=font) ax1.set_xlabel("Время", fontsize=font) ax1.legend(fontsize=font) diff --git a/bin/tm_brd_parser.py b/bin/tm_brd_parser.py new file mode 100644 index 0000000..792fd03 --- /dev/null +++ b/bin/tm_brd_parser.py @@ -0,0 +1,101 @@ +import pandas as pd +import os +import re +from pathlib import Path +import matplotlib.pyplot as plt +from datetime import datetime, timedelta +import sys + +tstamp_s = '%d.%m.%Y %H:%M:%S.%f' +ox_dtime_format = '%d.%m.%Y %H:%M' + +path_itog_brd_data = '../data/brd_data/' + + +class PathFileNotFound(Exception): + pass + + +def find_required_files(root_dir, pattern): + result = [] + for dirpath, _, filenames in os.walk(root_dir): + for filename in filenames: + match = re.match(pattern, filename) + if match: + result.append(dirpath + '/' + filename) + + if len(result) == 0: + raise PathFileNotFound( + f'error: check that the path is correct ({root_dir}) or files pattern is correct ({pattern})') + + return sorted(result) + + +def read_files_into_df(fname_list, column_list, dtype_columns={}): + data_itog = pd.DataFrame() + epoch_start = pd.Timestamp('2000-01-01') + + for fname in fname_list: + data = pd.read_csv(fname, sep=r'\s+', dtype=str) + data = data.dropna() + data = data[column_list] + + if 'TIME' in column_list: + # convert TIME value to human-readable timestamp (sinse epoch 01.01.2000) + time = data['TIME'].astype(float) + tstamp = epoch_start + pd.to_timedelta(time, unit='s') + timestamp = tstamp.dt.strftime(tstamp_s) + data['timestamp'] = timestamp + + # clear dataframe rows where time value == 0 + data['time'] = time + data_clear = data.query('time != 0.0') + + data_itog = pd.concat([data_itog, data_clear], ignore_index=True) + + return data_itog + + +def collect_tm_brd_files(root_dir_tm_data, column_list, column_list_itog): + patterns_tm = [r'mvn_tm_brd01_(.*)', r'mvn_tm_brd02_(.*)', r'mvn_tm_brd03_(.*)', + r'mvn_tm_brd04_(.*)'] + + for pattern in patterns_tm: + fname = path_itog_brd_data + pattern[:12] + '.csv' + try: + found_files = find_required_files(root_dir_tm_data, pattern) + data = read_files_into_df(found_files, column_list, dtype_columns={11: float}) + except KeyError as e: + print( + f'error in collect_tm_brd_files: the specified column name was not found in the data file (path: {root_dir_tm_data}) ({e})') + break + except Exception as e: + print(f'error in collect_tm_brd_files: {e}') + break + + data.to_csv(fname, index=False, sep=';', columns=column_list_itog, encoding='utf-8-sig') + print('data saved: ' + fname) + + +if __name__ == "__main__": + + if len(sys.argv) != 2: + print("Usage: python tm_brd_parser.py /path/to/tm_brd_data/") + else: + root_dir_tm_data = sys.argv[1] + + print('collect raw brd tm data into one file for each brd') + + column_list = ['TIME', 'PER_1Hz', 'ST_HV'] + column_list_itog = ['TIME', 'timestamp', 'PER_1Hz', 'ST_HV'] + + collect_tm_brd_files(root_dir_tm_data, column_list, column_list_itog) + + ## plot 'evolution' 1 Hz from tm brd data + print('plot evolution 1 Hz from tm brd data') + + fname = path_itog_brd_data + 'mvn_tm_brd01.csv' + df = pd.read_csv(fname, sep=';', parse_dates=['timestamp'], date_format="%d.%m.%Y %H:%M:%S.%f") + + plt.plot(df['timestamp'], df['PER_1Hz'], '.') + plt.show() diff --git a/bin/tm_wheel_parser.py b/bin/tm_wheel_parser.py new file mode 100644 index 0000000..296d0f7 --- /dev/null +++ b/bin/tm_wheel_parser.py @@ -0,0 +1,114 @@ +import pandas as pd +import os +import re +from pathlib import Path +import matplotlib.pyplot as plt +from datetime import datetime, timedelta +import sys + +tstamp_s = '%d.%m.%Y %H:%M:%S.%f' +ox_dtime_format = '%d.%m.%Y %H:%M' + +path_itog_brd_data = '../data/brd_data/' + + +class PathFileNotFound(Exception): + pass + + +def find_required_files(root_dir, pattern): + result = [] + for dirpath, _, filenames in os.walk(root_dir): + for filename in filenames: + match = re.match(pattern, filename) + if match: + result.append(dirpath + '/' + filename) + + if len(result) == 0: + raise PathFileNotFound( + f'error: check that the path is correct ({root_dir}) or files pattern is correct ({pattern})') + + return sorted(result) + + +def read_files_into_df(fname_list, column_list, dtype_columns={}): + data_itog = pd.DataFrame() + epoch_start = pd.Timestamp('2000-01-01') + + for fname in fname_list: + data = pd.read_csv(fname, sep=r'\s+', dtype=str) + data = data.dropna() + data = data[column_list] + + if 'TIME' in column_list: + # convert TIME value to human-readable timestamp (sinse epoch 01.01.2000) + time = data['TIME'].astype(float) + tstamp = epoch_start + pd.to_timedelta(time, unit='s') + timestamp = tstamp.dt.strftime(tstamp_s) + data['timestamp'] = timestamp + + # clear dataframe rows where time value == 0 + data['time'] = time + data_clear = data.query('time != 0.0') + + data_itog = pd.concat([data_itog, data_clear], ignore_index=True) + + return data_itog + + +def collect_tm_brd_wheel_data(root_dir_wheel_data, column_list, column_list_itog): + patterns_wheel = [r'mvn_wheel_brd01_(.*)', r'mvn_wheel_brd02_(.*)', r'mvn_wheel_brd03_(.*)', + r'mvn_wheel_brd04_(.*)'] + + for pattern in patterns_wheel: + fname = path_itog_brd_data + pattern[:15] + '.csv' + try: + found_files = find_required_files(root_dir_wheel_data, pattern) + data = read_files_into_df(found_files, column_list, dtype_columns={0: float, 1: int}) + except KeyError as e: + print( + f'error in collect_tm_brd_wheel_data: the specified column name was not found in the data file (path: {root_dir_tm_data}) ({e})') + break + except Exception as e: + print(f'error in collect_tm_brd_wheel_data: {e}') + break + + mask = data['STATE'] == '0' + data = data[mask] + data.to_csv(fname, index=False, sep=';', columns=column_list_itog, encoding='utf-8-sig') + print('data saved: ' + fname) + + +if __name__ == "__main__": + + if len(sys.argv) != 2: + print("Usage: python tm_wheel_parser.py /path/to/tm_brd_data/") + else: + root_dir_wheel_data = sys.argv[1] + + ### collect raw tm wheel data into one file for each brd ### + print('collect raw tm wheel data into one file for each brd') + + column_list = ['TIME', 'STATE'] + column_list_itog = ['TIME', 'timestamp', 'STATE'] + + collect_tm_brd_wheel_data(root_dir_wheel_data, column_list, column_list_itog) + + ## parse and plot wheel csv data + print('parse and plot wheel csv data') + + border_clr_wheel = 2 + fname = path_itog_brd_data + 'mvn_wheel_brd01.csv' + wheel_df = pd.read_csv(fname, sep=';') + wheel_df['TIME_diff'] = wheel_df['TIME'].diff() + median_tdiff = wheel_df['TIME_diff'].median() + + wheel_df_clear = wheel_df[(wheel_df['TIME_diff'] > median_tdiff - border_clr_wheel) & + (wheel_df['TIME_diff'] < median_tdiff + border_clr_wheel)] + + wheel_df_peaks = wheel_df[(wheel_df['TIME_diff'] <= median_tdiff - border_clr_wheel) | + (wheel_df['TIME_diff'] >= median_tdiff + border_clr_wheel)] + + plt.plot(wheel_df_clear['TIME'], wheel_df_clear['TIME_diff'], '-') + plt.plot(wheel_df_peaks['TIME'], wheel_df_peaks['TIME_diff'], '.') + plt.show() diff --git a/data/asotr/beta_2025.xlsx b/data/asotr/beta_2025.xlsx new file mode 100644 index 0000000..fc9b520 Binary files /dev/null and b/data/asotr/beta_2025.xlsx differ