diff --git a/.gitignore b/.gitignore index 8ce8877..1a60579 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,5 @@ *.zip *.log *.txt +*.report_* /__pycache__ -/data -/reports diff --git a/asotr.py b/asotr.py index 4d1bba6..8eb8959 100644 --- a/asotr.py +++ b/asotr.py @@ -8,7 +8,16 @@ import json import pytz from matplotlib import dates -fname_json_decode = '/home/danila/Danila/work/MVN/Soft/PID/python/decode_asotr_cmd.json' +""" +module asotr parses csv data from flight device ASOTR MVN + +Danila Gamkov [] +danila_gamkov@cosmos.ru +# License: IKI RAN +""" +__author__ = 'Danila Gamkov' + +fname_json_decode = './decode_asotr_cmd.json' def get_utc_seconds(timestamp_str, timestamp_format): dt_obj = datetime.strptime(timestamp_str, timestamp_format) @@ -115,8 +124,20 @@ def cmd_flight_parse(asotr_data): return (decode_list, temperature_list) -# accuracy: minutes, hours -def find_best_time_idx(time_arr, user_time, accuracy='minutes'): +# accuracy: 'seconds', 'minutes', 'hours' +def find_best_time_idx(time_arr, user_time, accuracy='minutes') -> int: + """ + finds best time index according specified user_time in time_arr and with specified accuracy + + Args: + time_arr(pandas.core.series.Series[datetime]): timestamp array + user_time(string): time value, which index required find in time_arr (string in the following format: d.m.Y HH:MM:SS) + accuracy(string): specify accuracy which time index required find in time_arr(type 'seconds', 'minutes' or 'hours') + + Returns: + int: index that has been found in time_arr (or -1 if index has not been found) + """ + tstamp = datetime.strptime(user_time, "%d.%m.%Y %H:%M:%S") if accuracy == 'minutes': delta = timedelta(minutes=30) @@ -166,15 +187,20 @@ def find_best_time_idx(time_arr, user_time, accuracy='minutes'): return j return mid - # raise ValueError("data not found!") def find_time_idx(data_list, keys_list, timestamp, accuracy): out_dict = dict.fromkeys(keys_list, -1) for i, elem in enumerate(data_list): - out_dict[keys_list[i]] = find_best_time_idx(elem['timestamp'], timestamp, accuracy) + idx = find_best_time_idx(elem['timestamp'], timestamp, accuracy) + if idx != -1: + out_dict[keys_list[i]] = idx + else: + raise TimeIndexNotFound(f'index corresponding to time {timestamp} in times array not found!') + return out_dict + def get_cmd_data(fname): asotr_data = pd.read_csv(fname, delimiter=';') cmd_list, temperature_list = cmd_flight_parse(asotr_data) @@ -191,9 +217,22 @@ def get_data(path, asotr_kit, start_date, end_date, time_accuracy): dateparse = lambda x: datetime.strptime(x, "%d.%m.%Y %H:%M:%S.%f") - data = [ pd.read_csv(fname[0], sep=";", parse_dates=["timestamp"], date_parser=dateparse), + try: + data = [ pd.read_csv(fname[0], sep=";", parse_dates=["timestamp"], date_parser=dateparse), pd.read_csv(fname[1], sep=";", parse_dates=["timestamp"], date_parser=dateparse), pd.read_csv(fname[2], sep=";", parse_dates=["timestamp"], date_parser=dateparse),] + except FileNotFoundError: + print(f'Error opening file: one (or all) file not found in directory: \n{fname}') + return + except pd.errors.EmptyDataError: + print(f'Error opening file: one (or all) file is empty or have incorrect format. \nLook at the files: {fname}') + return + except pd.errors.ParserError: + print(f'Error parsing file: file have incorrect structure. \nLook at the files: {fname}') + return + except Exception as e: + print(f'Error parsing file: {e}. \nLook at the files: {fname}') + return ch = [[], [], [], [], [], []] data_dict = { @@ -208,8 +247,12 @@ def get_data(path, asotr_kit, start_date, end_date, time_accuracy): idxb = dict.fromkeys(ch_signs, -1) idxe = dict.fromkeys(ch_signs, -1) - idxb = find_time_idx(data, ch_signs, start_date, time_accuracy) - idxe = find_time_idx(data, ch_signs, end_date, time_accuracy) + try: + idxb = find_time_idx(data, ch_signs, start_date, time_accuracy) + idxe = find_time_idx(data, ch_signs, end_date, time_accuracy) + except Exception as e: + print(e) + return data_dict["time_temp"] = data[0]["timestamp"][idxb["temp"] : idxe["temp"]] data_dict["time_temp_set"] = data[1]["timestamp"][idxb["temp_set"] : idxe["temp_set"]] @@ -324,8 +367,7 @@ def plot_signal_profile(time, data, pattern_t, pattern, method, shift_flag, peak plt.grid(True) plt.show() -def insert_temp_data_from_flight_cmd(fname_cmd_temp): - dir_asotr = '/home/danila/Danila/work/MVN/Soft/asotr_csv/data/' +def insert_temp_data_from_flight_cmd(fname_cmd_temp, dir_asotr): fname_asotr = [f'{dir_asotr}asotr01_data_T.csv', f'{dir_asotr}asotr02_data_T.csv'] df_cmd = pd.read_csv(fname_cmd_temp, sep=';') @@ -473,13 +515,11 @@ def plot_step_response_in_thermocycle(data_info, thermocycle_info, interp, def plot_imp_response(data, data_info, plot_info, thermocycle_info): - title = f'{plot_info["title"]}, канал {data_info["channel"][2]} АСОТР КДИ СПИН-X, период опроса {data_info["period"]} ({thermocycle_info["date"]})' fig = plt.figure(figsize=(11, 6), dpi=200) fig.suptitle(title, fontsize=plot_info['font']) ax1 = fig.add_subplot(1,1,1) - date_formatter = dates.DateFormatter(plot_info['ox_dtime_format']) ax1.xaxis.set_major_formatter(date_formatter) @@ -499,13 +539,12 @@ def plot_imp_response(data, data_info, plot_info, thermocycle_info): #timestamp as string format: dd:mm:YYYY HH:MM:SS -def insert_data_cyclo(base_time_str, fname): - path = '/home/danila/Danila/work/MVN/flight/experiments/' +def insert_data_cyclo(base_time_str, fname, path): time_format = "%d.%m.%Y %H:%M:%S" cyclogram_file = path + fname df = pd.read_excel(cyclogram_file) - base_time = pd.to_datetime(base_time_str) + base_time = pd.to_datetime(base_time_str, format='%d.%m.%Y %H:%M:%S') df['timestamp'] = df.iloc[:, 0].apply(lambda x: base_time + timedelta(seconds=x)) df.iloc[:, 0] = df['timestamp'].dt.strftime(time_format) df = df.drop(['timestamp'], axis=1) diff --git a/data/experiments/cyclogram_imp_ident_ch1.xls b/data/experiments/cyclogram_imp_ident_ch1.xls new file mode 100644 index 0000000..3c0eb7e Binary files /dev/null and b/data/experiments/cyclogram_imp_ident_ch1.xls differ diff --git a/data/experiments/cyclogram_step_ident_ch1.xls b/data/experiments/cyclogram_step_ident_ch1.xls new file mode 100644 index 0000000..cec3d6a Binary files /dev/null and b/data/experiments/cyclogram_step_ident_ch1.xls differ diff --git a/data/experiments/step_impulse/asotr01_ch1_25042025/cyclogram_imp_ident_ch1.xls b/data/experiments/step_impulse/asotr01_ch1_25042025/cyclogram_imp_ident_ch1.xls new file mode 100644 index 0000000..d845514 Binary files /dev/null and b/data/experiments/step_impulse/asotr01_ch1_25042025/cyclogram_imp_ident_ch1.xls differ diff --git a/data/experiments/step_impulse/asotr01_ch1_25042025/cyclogram_step_ident_ch1.xls b/data/experiments/step_impulse/asotr01_ch1_25042025/cyclogram_step_ident_ch1.xls new file mode 100644 index 0000000..57b9ccf Binary files /dev/null and b/data/experiments/step_impulse/asotr01_ch1_25042025/cyclogram_step_ident_ch1.xls differ diff --git a/data/experiments/step_impulse/asotr01_ch2_30042025/cyclogram_imp_ident_ch2.xls b/data/experiments/step_impulse/asotr01_ch2_30042025/cyclogram_imp_ident_ch2.xls new file mode 100644 index 0000000..2f73dc9 Binary files /dev/null and b/data/experiments/step_impulse/asotr01_ch2_30042025/cyclogram_imp_ident_ch2.xls differ diff --git a/data/experiments/step_impulse/asotr01_ch2_30042025/cyclogram_step_ident_ch2.xls b/data/experiments/step_impulse/asotr01_ch2_30042025/cyclogram_step_ident_ch2.xls new file mode 100644 index 0000000..f3c1020 Binary files /dev/null and b/data/experiments/step_impulse/asotr01_ch2_30042025/cyclogram_step_ident_ch2.xls differ diff --git a/data/experiments/step_impulse/asotr01_ch3_01052025/cyclogram_imp_ident_ch3.xls b/data/experiments/step_impulse/asotr01_ch3_01052025/cyclogram_imp_ident_ch3.xls new file mode 100644 index 0000000..da10b38 Binary files /dev/null and b/data/experiments/step_impulse/asotr01_ch3_01052025/cyclogram_imp_ident_ch3.xls differ diff --git a/data/experiments/step_impulse/asotr01_ch3_01052025/cyclogram_step_ident_ch3.xls b/data/experiments/step_impulse/asotr01_ch3_01052025/cyclogram_step_ident_ch3.xls new file mode 100644 index 0000000..0327824 Binary files /dev/null and b/data/experiments/step_impulse/asotr01_ch3_01052025/cyclogram_step_ident_ch3.xls differ diff --git a/data/experiments/step_impulse/asotr01_ch4_02052025/cyclogram_imp_ident_ch4.xls b/data/experiments/step_impulse/asotr01_ch4_02052025/cyclogram_imp_ident_ch4.xls new file mode 100644 index 0000000..c329047 Binary files /dev/null and b/data/experiments/step_impulse/asotr01_ch4_02052025/cyclogram_imp_ident_ch4.xls differ diff --git a/data/experiments/step_impulse/asotr01_ch4_02052025/cyclogram_step_ident_ch4.xls b/data/experiments/step_impulse/asotr01_ch4_02052025/cyclogram_step_ident_ch4.xls new file mode 100644 index 0000000..923fcd5 Binary files /dev/null and b/data/experiments/step_impulse/asotr01_ch4_02052025/cyclogram_step_ident_ch4.xls differ diff --git a/data/experiments/step_impulse/asotr02_ch1_03052025/cyclogram_imp_ident_ch1.xls b/data/experiments/step_impulse/asotr02_ch1_03052025/cyclogram_imp_ident_ch1.xls new file mode 100644 index 0000000..3c0eb7e Binary files /dev/null and b/data/experiments/step_impulse/asotr02_ch1_03052025/cyclogram_imp_ident_ch1.xls differ diff --git a/data/experiments/step_impulse/asotr02_ch1_03052025/cyclogram_step_ident_ch1.xls b/data/experiments/step_impulse/asotr02_ch1_03052025/cyclogram_step_ident_ch1.xls new file mode 100644 index 0000000..cec3d6a Binary files /dev/null and b/data/experiments/step_impulse/asotr02_ch1_03052025/cyclogram_step_ident_ch1.xls differ diff --git a/decode_cmd_data.py b/decode_cmd_data.py index 2d96b9d..c841b3b 100644 --- a/decode_cmd_data.py +++ b/decode_cmd_data.py @@ -6,6 +6,7 @@ reload(asotr) import pandas as pd from datetime import datetime, timedelta +path_data = '/home/danila/Danila/work/MVN/Soft/asotr_csv/data/' fname = '/home/danila/Danila/work/MVN/Soft/PID/data/flight/cmd_asotr/all_flight_cmd_asotr.csv' fname_cmd_temp = './data/flight_cmd_temp.csv' @@ -23,8 +24,7 @@ with open(fname_cmd_temp, 'w') as file: file.write(f'{elem}\n') ## insert temperatures from flight commands file to main asotr temperatures data files -df_asotr_ = asotr.insert_temp_data_from_flight_cmd(fname_cmd_temp) - +df_asotr_ = asotr.insert_temp_data_from_flight_cmd(fname_cmd_temp, path_data) ## form timestamp file where minimum of temperatures were registered end_date = '' @@ -33,7 +33,6 @@ for i, data in enumerate(df_asotr_): data.to_csv(f'./data/asotr0{i+1}_data_T.csv', index=False, sep=';', encoding='utf-8-sig', decimal='.') -path_data = '/home/danila/Danila/work/MVN/Soft/asotr_csv/data/' timeformat = '%d.%m.%Y %H:%M:%S' prev_days = 14 diff --git a/plot_flight_borders.py b/plot_flight_borders.py index a824d98..5a42998 100644 --- a/plot_flight_borders.py +++ b/plot_flight_borders.py @@ -3,7 +3,7 @@ from matplotlib import dates import argparse import sys from importlib import reload -sys.path.append('/home/danila/Danila/work/MVN/Soft/PID/python/') +sys.path.append('./') import asotr reload(asotr) import pandas as pd @@ -16,18 +16,16 @@ def convert_to_str(lst): res += f"_{idx + 1}" return res -def plot_asotr_borders(ch, asotr_kit, begin, end, font=14, cmd=0): +def plot_asotr_borders(path_with_data, ch, asotr_kit, begin, end, font=14, cmd=0, show_flag=False): print_width = 20 print_height = 12 width = 1 plot_windows = 2 channels = list(map(int, ch)) - pict_name = (f'./reports/ASOTR{asotr_kit}_flight_T_P_{convert_to_str(channels)}_{begin[0:5].replace(".", "")}_{end[0:5].replace(".", "")}_{end[6:]}.png') plot_task = {"temp": 1, "temp_set": 1, "pow": 1} - ox_dtime_format = "%d.%m.%Y" legend = [ @@ -46,10 +44,12 @@ def plot_asotr_borders(ch, asotr_kit, begin, end, font=14, cmd=0): width_arr = [1, 0.5, 0.2, 0.1, 1, 1] # get from files and prepare data - path = "/home/danila/Danila/work/MVN/Soft/asotr_csv/data/" - start_date = begin + " 00:00:00" # Начальная граница - end_date = end + " 23:59:59" # Конечная граница - data, data_dict = asotr.get_data(path, asotr_kit, start_date, end_date, 'minutes') + start_date = begin + " 00:00:00" + end_date = end + " 23:59:59" + try: + data, data_dict = asotr.get_data(path_with_data, asotr_kit, start_date, end_date, 'minutes') + except Exception as e: + return if plot_windows == 1: fig, ax = plt.subplots(figsize=(print_width, print_height), dpi=200) @@ -82,10 +82,14 @@ def plot_asotr_borders(ch, asotr_kit, begin, end, font=14, cmd=0): ax1 = fig.add_subplot(2, 1, 1) ax2 = fig.add_subplot(2, 1, 2, sharex=ax1) - if cmd == '1': - cmd_human = pd.read_csv('./flight_cmd_human.txt', + fname = './flight_cmd_human.txt' + try: + cmd_human = pd.read_csv('./flight_cmd_human.txt', delimiter=';', names=['timestamp', 'cmd']) + except Exception as e: + print(f'Error parsing file: {e}') + return for i, row in cmd_human.iterrows(): if i > 20: @@ -97,7 +101,7 @@ def plot_asotr_borders(ch, asotr_kit, begin, end, font=14, cmd=0): linestyle='-.') ax1.text(data_dict['time_temp'][idx], 30, row['cmd'], rotation=45, va='bottom', fontsize=font) - + if plot_task["temp"] == 1: for i in range(len(channels)): if channels[i] == 1: @@ -149,11 +153,14 @@ def plot_asotr_borders(ch, asotr_kit, begin, end, font=14, cmd=0): fig.suptitle(title, fontsize=font) plt.tight_layout() fig.savefig(pict_name) - plt.show() + if show_flag == True: + plt.show() if __name__ == '__main__': argparser = argparse.ArgumentParser("plot_flight_borders.py") + argparser.add_argument('-s', '--source', required=True, + help='type path with asotr csv data') argparser.add_argument('-c', '--channel', required=True, help='type channel (example: 000011)') argparser.add_argument('-a', '--asotr', required=True, @@ -166,7 +173,9 @@ if __name__ == '__main__': help='type font size (from 1 to 30)') argparser.add_argument('-d', '--cmd', required=False, help='type display commands flag (0/1)') + argparser.add_argument('-p', '--plot', required=False, + help='display data in plot flag (0/1)') args = argparser.parse_args() - plot_asotr_borders(args.channel, args.asotr, args.begin, args.end, - args.font, args.cmd) + plot_asotr_borders(args.source, args.channel, args.asotr, args.begin, args.end, + args.font, args.cmd, args.plot) diff --git a/plot_flight_borders.sh b/plot_flight_borders.sh index a9dbc7a..4ae012f 100755 --- a/plot_flight_borders.sh +++ b/plot_flight_borders.sh @@ -1,19 +1,23 @@ #! /bin/bash -if [ $# != 2 ] +if [ $# != 5 ] then echo "erorr use $0. Right use this script: " - echo "$0 25.02.2025 10.03.2025" + echo "$0 path_to_csv_astor_data/ 25.02.2025 10.03.2025 14 0" else - begin=$1 - end=$2 - python3 plot_flight_borders.py -c 111100 -a 01 -b ${begin} -e ${end} - python3 plot_flight_borders.py -c 001000 -a 01 -b ${begin} -e ${end} - python3 plot_flight_borders.py -c 000011 -a 01 -b ${begin} -e ${end} - python3 plot_flight_borders.py -c 111100 -a 02 -b ${begin} -e ${end} - python3 plot_flight_borders.py -c 010100 -a 02 -b ${begin} -e ${end} - python3 plot_flight_borders.py -c 010000 -a 02 -b ${begin} -e ${end} - python3 plot_flight_borders.py -c 000100 -a 02 -b ${begin} -e ${end} - python3 plot_flight_borders.py -c 000011 -a 02 -b ${begin} -e ${end} + path_csv_data=$1 + begin=$2 + end=$3 + font=$4 + cmd_flag=$5 + + python3 plot_flight_borders.py -s ${path_csv_data} -c 111100 -a 01 -b ${begin} -e ${end} -f ${font} --cmd ${cmd_flag} + python3 plot_flight_borders.py -s ${path_csv_data} -c 001000 -a 01 -b ${begin} -e ${end} -f ${font} --cmd ${cmd_flag} + python3 plot_flight_borders.py -s ${path_csv_data} -c 000011 -a 01 -b ${begin} -e ${end} -f ${font} --cmd ${cmd_flag} + python3 plot_flight_borders.py -s ${path_csv_data} -c 111100 -a 02 -b ${begin} -e ${end} -f ${font} --cmd ${cmd_flag} + python3 plot_flight_borders.py -s ${path_csv_data} -c 010100 -a 02 -b ${begin} -e ${end} -f ${font} --cmd ${cmd_flag} + python3 plot_flight_borders.py -s ${path_csv_data} -c 010000 -a 02 -b ${begin} -e ${end} -f ${font} --cmd ${cmd_flag} + python3 plot_flight_borders.py -s ${path_csv_data} -c 000100 -a 02 -b ${begin} -e ${end} -f ${font} --cmd ${cmd_flag} + python3 plot_flight_borders.py -s ${path_csv_data} -c 000011 -a 02 -b ${begin} -e ${end} -f ${font} --cmd ${cmd_flag} fi diff --git a/prepare_flight_cyclo.py b/prepare_flight_cyclo.py index 5c327e7..065c5ba 100644 --- a/prepare_flight_cyclo.py +++ b/prepare_flight_cyclo.py @@ -1,17 +1,19 @@ import sys from importlib import reload -sys.path.append('/home/danila/Danila/work/MVN/Soft/PID/python/') +sys.path.append('./') import asotr reload(asotr) import pandas as pd from datetime import datetime, timedelta -timestamp = '25.04.2025 01:18:00' +path = './data/experiments/' + +timestamp = '03.05.2025 01:27:00' cyclogram_file = 'cyclogram_step_ident_ch1.xls' -asotr.insert_data_cyclo(timestamp, cyclogram_file) +asotr.insert_data_cyclo(timestamp, cyclogram_file, path) -timestamp = '25.04.2025 04:24:00' +timestamp = '03.05.2025 04:33:00' cyclogram_file = 'cyclogram_imp_ident_ch1.xls' -asotr.insert_data_cyclo(timestamp, cyclogram_file) +asotr.insert_data_cyclo(timestamp, cyclogram_file, path) diff --git a/temp_peaks_forecast.py b/temp_peaks_forecast.py index cac510d..d0f0ab1 100644 --- a/temp_peaks_forecast.py +++ b/temp_peaks_forecast.py @@ -12,7 +12,7 @@ from datetime import timedelta path = '/home/danila/Danila/work/MVN/Soft/asotr_csv/data/' channel = 'ch1' asotr_kit = '01' -start_date = '20.04.2025 00:00:00' +start_date = '23.04.2025 00:00:00' forecast_days = 20 # end_date = '26.03.2025 01:20:00'