Refactor: 1. add print into console data, 2. rewrite CLI parsing, 4. the output format csv has been reworked. 5. Add information to README. For addition create python code for plot csv data

This commit is contained in:
Danila Gamkov 2025-01-29 13:47:36 +03:00
parent d3cf1281dd
commit b44fc029fc
10 changed files with 458 additions and 74 deletions

5
.gitignore vendored
View File

@ -1,2 +1,7 @@
/target
*.asotr*
*.csv
*.png
*.swp
*.zip
asotr_csv

13
Cargo.lock generated
View File

@ -143,6 +143,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "769b0145982b4b48713e01ec42d61614425f27b7058bda7180a3a41f30104796"
dependencies = [
"clap_builder",
"clap_derive",
]
[[package]]
@ -157,6 +158,18 @@ dependencies = [
"strsim",
]
[[package]]
name = "clap_derive"
version = "4.5.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54b755194d6389280185988721fffba69495eed5ee9feeee9a599b53db80318c"
dependencies = [
"heck",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "clap_lex"
version = "0.7.4"

View File

@ -2,12 +2,14 @@
name = "asotr_csv"
version = "0.1.0"
edition = "2021"
authors = ["Danila Gamkov <danila_gamkov@cosmos.ru"]
description = "The parser for converting data from the ASOTR MVN control channels into the CSV format (see files in asotr.tar.gz)"
[dependencies]
byteorder = "1.4.3"
chrono = "0.4"
strum = { version = "0.26", features = ["derive"] }
clap = "4.*"
clap = { version = "4.*", features = ["derive"] }
lazy_static = { version = "1.5.0", features = ["spin_no_std"] }
regex = "1.7.0"
walkdir = "2.3.2"

View File

@ -8,6 +8,7 @@ The parser for converting data from the ASOTR MVN control channels into the CSV
- **Using**
- parsing ASOTR MVN data files in specified directory
- parsing all ASOTR MVN data files in specified directory and subdirectories
- plot data in python
- **Output asotr_csv data files description**
- **Contacts**
@ -82,6 +83,9 @@ cd <PATH_TO_ASOTR_CSV>/target/release/
csv data are ready to use in directory:
\<PATH_TO_ASOTR_CSV\>/target/release/
### Plot csv data in Python
If you want to plot csv data you might use \<PATH_TO_ASOTR_CSV\>/**asotr_all_unzip_auto.sh** shell script or use \<PATH_TO_ASOTR_CSV\>/data/**plot_flight_all.py** python-script directly for set different configurations.
## Output asotr_csv data files description
**description:**
asotr01_data_T.csv - ASOTR1 temperature data in channels 1-6 (in Celsius)
@ -92,10 +96,9 @@ asotr02_data_P.csv - ASOTR2 power data in channels 1-6 (in %)
asotr02_data_TSET.csv - ASOTR2 temperature sets in channels 1-6 (in Celsius)
**file data csv fromat:**
column 1: Unix timestamp
column 2: date
column 3: time
columns 4-9 - data from control channels (power, temperature or temperature set)
column 1: Unix timestamp in seconds
column 2: timestamp (data and time)
columns 3-8 - data from control channels (power, temperature or temperature set)
## Contatcs

16
asotr_all_unzip_auto.sh Executable file
View File

@ -0,0 +1,16 @@
#! /bin/bash
if [ $# != 1 ]
then
echo "erorr use $0. Right use this script: "
echo "$0 path"
else
cp ./target/release/asotr_csv ./data
path_=$1
find ${path_} -maxdepth 1 -type d | xargs -I {} ./asotr_unzip.sh {}
cd ./data
./asotr_csv -d ${path_}
python3 ./plot_flight_all.py
fi

122
data/plot_flight_all.py Normal file
View File

@ -0,0 +1,122 @@
import matplotlib.pyplot as plt
from matplotlib import dates
import pandas as pd
from datetime import datetime
import sys
font = 10
print_width = 20
print_height = 12
width = 1
plot_windows = 2
channels = [1, 1, 1, 1, 1, 1]
xborders=False
begin=0;
end=0;
path = '/home/danila/Danila/work/MVN/Soft/asotr_csv/data/'
fname = 'asotr01_data_T.csv'
fname_pow = 'asotr01_data_P.csv'
pict_name = path + "ASOTR1_flight_T_P_all"
ox_dtime_format = '%d.%m.%Y %H:%M'
legend=['ch1', 'ch2', 'ch3', 'ch4', 'ch5', 'ch6', 'ch7']
width=[1, 1, 1, 1, 1, 1]
marker = ['-', '-', '-', '-', '-', '-'];
width_arr = [1, 0.5, 0.2, 0.1, 1, 1]
fname = [path + fname, path + fname_pow]
dateparse = lambda x: datetime.strptime(x, "%d.%m.%Y %H:%M:%S.%f")
data = [pd.read_csv(fname[0], sep=';', parse_dates=['timestamp'], date_parser=dateparse),
pd.read_csv(fname[1], sep=';', parse_dates=['timestamp'], date_parser=dateparse)]
ch= [[], [], [], [], [], []]
ch_signs = ["temp", "pow"]
data_dict = {"temp": ch, "pow": ch, "time": []}
data_dict["time"] = data[0]['timestamp']
col=['ch1', 'ch2', 'ch3', 'ch4', 'ch5', 'ch6', 'ch7']
for j in range(2):
for index, row, in data[j].iterrows():
for i in range(6):
ch[i].append(float(row[col[i]]))
data_dict[ch_signs[j]] = ch
ch= [[], [], [], [], [], []]
len_data = [len(data_dict['temp'][0]), len(data_dict['pow'][0])]
len_ = min(len_data)
if xborders == False:
begin = 0
end = len_ - 1
if plot_windows == 1:
fig, ax = plt.subplots(figsize=(print_width, print_height), dpi=200)
i = 0
for elem in data_dict['temp']:
if channels[i] == 1:
plt.plot(data_dict['time'][begin:end], elem[begin:end], marker[i], linewidth=width[i], label=legend[i])
i += 1
# ax.axvline(x = data['timestamp'][300], color='r', linestyle='-.', label="power=100")
# ax.axvline(x = data['timestamp'][1500], color='b', linestyle='dotted', label="power=0")
ax.tick_params(axis="both", width=1, labelsize=font)
ax.grid(visible=True, linestyle = 'dotted')
ax.set_ylabel('Температура, $^\circ$C', fontsize=font)
ax.set_xlabel('Время', fontsize=font)
ax.legend(fontsize=font)
date_formatter = dates.DateFormatter(ox_dtime_format)
ax.xaxis.set_major_formatter(date_formatter)
plt.tight_layout()
fig.savefig(pict_name)
plt.show()
elif plot_windows == 2:
fig = plt.figure(figsize=(print_width, print_height), dpi=200)
ax1 = fig.add_subplot(2, 1, 1)
ax2 = fig.add_subplot(2, 1, 2, sharex=ax1)
i = 0
for elem in data_dict['temp']:
if channels[i] == 1:
ax1.plot(data_dict['time'][begin:end], elem[begin:end], marker[i], linewidth=width[i], label=legend[i])
i += 1
i = 0
for elem in data_dict['pow']:
if channels[i] == 1:
ax2.plot(data_dict['time'][begin:end], elem[begin:end], marker[i], linewidth=width[i], label=legend[i])
i += 1
# ax1.axvline(x = data['timestamp'][300], color='r', linestyle='-.', label="power=100")
# ax.axvline(x = data['timestamp'][1500], color='b', linestyle='dotted', label="power=0")
ax1.tick_params(axis="both", width=1, labelsize=font)
ax1.grid(visible=True, linestyle = 'dotted')
ax1.set_ylabel('Температура, $^\circ$C', fontsize=font)
ax1.set_xlabel('Время', fontsize=font)
ax1.legend(fontsize=font)
date_formatter = dates.DateFormatter('%d.%m.%Y %H')
ax1.xaxis.set_major_formatter(date_formatter)
ax2.tick_params(axis="both", width=1, labelsize=font)
ax2.grid(visible=True, linestyle = 'dotted')
ax2.set_ylabel('Мощность, %', fontsize=font)
ax2.set_xlabel('Время', fontsize=font)
ax2.legend(fontsize=font)
date_formatter = dates.DateFormatter(ox_dtime_format)
ax2.xaxis.set_major_formatter(date_formatter)
plt.tight_layout()
fig.savefig(pict_name)
plt.show()

30
data/prepare_csv.sh Executable file
View File

@ -0,0 +1,30 @@
#! /bin/bash
if [ $# != 2 ]
then
echo "error use $0. Right use this script: "
echo "$0 path_to_file data_type (flight or KDI)"
echo "example 1: $0 ./data/flight/30_12_2024/ASOTR_1_SOTR_T flight"
else
data_file=$1
data_type=$2
if [ "$data_type" == "flight" ]
then
cat ${data_file}.csv | grep -Eo '[0-9]{2}\.[0-9]{2}\.[0-9]{4}' > file1
cat ${data_file}.csv | grep -Eo [0-9]{2}:.* > file2
elif [ "$data_type" == "KDI" ]
then
cat ${data_file}.csv | grep -Eo [0-9]{2}.[0-9]{2}.[0-9]{4} > file1
cat ${data_file}.csv | grep -Eo [0-9]{2}:.* > file2
else
echo "error argument of data_type: write \"flight\" or \"KDI\" in second argument"
exit 1
fi
paste --delimiter=' ' file1 file2 > file.csv
echo "timestamp;ch1;ch2;ch3;ch4;ch5;ch6" > ${data_file}_clear.csv
cat file.csv >> ${data_file}_clear.csv
rm file1 file2 file.csv
fi

30
data/trash_prepare_csv.sh Executable file
View File

@ -0,0 +1,30 @@
#! /bin/bash
if [ $# != 2 ]
then
echo "error use $0. Right use this script: "
echo "$0 path_to_file data_type (flight or KDI)"
echo "example 1: $0 ./data/flight/30_12_2024/ASOTR_1_SOTR_T flight"
else
data_file=$1
data_type=$2
if [ "$data_type" == "flight" ]
then
cat ${data_file}.csv | grep -Eo '[0-9]{2}\.[0-9]{2}\.[0-9]{4}' > file1
cat ${data_file}.csv | grep -Eo [0-9]{2}:.* > file2
elif [ "$data_type" == "KDI" ]
then
cat ${data_file}.csv | grep -Eo [0-9]{2}.[0-9]{2}.[0-9]{4} > file1
cat ${data_file}.csv | grep -Eo [0-9]{2}:.* > file2
else
echo "error argument of data_type: write \"flight\" or \"KDI\" in second argument"
exit 1
fi
paste --delimiter=' ' file1 file2 > file.csv
echo "timestamp;ch1;ch2;ch3;ch4;ch5;ch6" > ${data_file}_clear.csv
cat file.csv >> ${data_file}_clear.csv
rm file1 file2 file.csv
fi

132
src/.vimrc Normal file
View File

@ -0,0 +1,132 @@
set tabstop=4
set softtabstop=4
set shiftwidth=4
set noexpandtab
set colorcolumn=90
highlight ColorColumnt ctermbg=darkgray
augroup project
autocmd!
autocmd BufRead,BufNewFile *.h,*.c set filetype=c.doxygen
augroup END
let &path.="src/include, src/source,"
" Включаем использование системного буфера
set clipboard=unnamedplus
" Работа с текстом
" Python использует 4 пробела для отступов
autocmd FileType python setlocal tabstop=4 shiftwidth=4
" Кодировка текста
set encoding=utf-8
set fileencoding=utf-8
set fileencodings=utf-8,cp1251,koi8-r,cp866
" Поиск по тексту
set hlsearch " подсвечивать результаты поиска
" Перемещение по тексту
" Когда достигаем границ строки, то перемещаемся на предыдующую/следующую
set whichwrap+=h,l,<,>,[,]
set number
" Настройки автодополнения
set completeopt=menu,menuone,noselect
" Разделение экрана
set splitbelow " разбивать вниз
set splitright " разбивать вправо
" сочетание клавиш
" Использование h, j, k, l для перемещения с зажатым Ctrl в режиме
" редактирования
inoremap <C-h> <Left>
inoremap <C-j> <Down>
inoremap <C-k> <Up>
inoremap <C-l> <Right>
let g:mapleader = "\<Space>"
" Переключение между вкладками
nnoremap <leader>t :tabnext<CR>
nnoremap <leader>T :tabprevious<CR>
" Список вкладок
nnoremap <leader>tl :tabs<CR>
" nnoremap <leader>tn :tabnew<CR>
nnoremap <leader>tc :tabclose<CR>
nnoremap <leader>to :tabonly<CR>
nnoremap <leader>tm :tabmove<CR>
" Редактировать файл в новой вкладке
nnoremap <leader>te :tabedit |
" Выбор вкладки
nnoremap <leader>1 1gt
nnoremap <leader>2 2gt
nnoremap <leader>3 3gt
nnoremap <leader>4 4gt
nnoremap <leader>5 5gt
nnoremap <leader>6 6gt
nnoremap <leader>7 7gt
nnoremap <leader>8 8gt
nnoremap <leader>9 9gt
nnoremap <leader>0 :tablast<CR>
" Разбиение окон
nnoremap <leader>s :split<CR>
nnoremap <leader>v :vsplit<CR>
" Выбор окна
nnoremap <C-h> <C-w>h
nnoremap <C-j> <C-w>j
nnoremap <C-k> <C-w>k
nnoremap <C-l> <C-w>l
" Размер окна
nnoremap <C-u> <C-w>+
nnoremap <C-d> <C-w>-
nnoremap <C-p> <C-w><
nnoremap <C-n> <C-w>>
" Vimspector
" nnoremap <leader><F2> <F10>
" nnoremap <leader>q <F11>
nmap <Leader><Right> <Plug>VimspectorStepOver
nmap <Leader><Down> <Plug>VimspectorStepInto
nmap <Leader><Up> <Plug>VimspectorStepOut
nmap <Leader><Tab> <Plug>VimspectorDisassemble
" Сделать окна одного размера
nnoremap <leader>= <C-w>=
" Переключения между буферами
" nnoremap <leader>b :bnext<CR>
" nnoremap <leader>B :bprevious<CR>
" nnoremap <leader>l :ls<CR>
" nnoremap <leader>d :bd<CR>
" " Скрыть/раскрыть блок кода
" nnoremap <leader>z za
" настройка плагинов
" настройки для отступов
" let g:indent_guides_enable_on_vim_startup = 1
" Настройки для разноцветной подсветки скобок
let g:rainbow_active = 1
" Настройки для vim-airline
let g:airline#extensions#tabline#enabled = 1
let g:airline#extensions#tabline#buffer_nr_show = 1
let g:airline#extensions#tabline#formatter = 'unique_tail'
let g:airline_powerline_fonts = 1
let g:airline_solarized_bg = 'luna'
let g:vimspector_enable_mappings = 'HUMAN'

View File

@ -1,4 +1,4 @@
use clap::{Arg, Command};
use clap::{Parser};
pub mod asotr_data {
use std::{fs::File, io::Read};
@ -31,6 +31,17 @@ pub mod asotr_data {
patterns
};
pub static ref patterns_disp: Vec<String> = {
let mut patterns: Vec<String> = Vec::new();
patterns.push(String::from("ASOTR01 temperature"));
patterns.push(String::from("ASOTR01 power"));
patterns.push(String::from("ASOTR01 temperature setpoint"));
patterns.push(String::from("ASOTR02 temperature"));
patterns.push(String::from("ASOTR02 power"));
patterns.push(String::from("ASOTR02 temperature setpoint"));
patterns
};
}
#[derive(Debug, FromRepr, PartialEq)]
@ -42,8 +53,7 @@ pub mod asotr_data {
struct AsotrDataDesc {
time_s: u64,
// time_mks: u32,
// datetime: String,
time_mks: u32,
date: String,
time: String,
data_type: AsotrDataType,
@ -51,9 +61,9 @@ pub mod asotr_data {
}
impl AsotrDataDesc {
pub fn new(time_s: u64, date: String, time: String,
pub fn new(time_s: u64, time_mks: u32, date: String, time: String,
data_type: AsotrDataType) -> AsotrDataDesc {
AsotrDataDesc { time_s, date, time, data_type }
AsotrDataDesc { time_s, time_mks, date, time, data_type }
}
}
@ -77,12 +87,11 @@ pub mod asotr_data {
Err(msg) => { return Err(format!("Error reading data file {}: {}", filename_full, msg)) }
};
out.push_str(&asotr_head.time_s.to_string());
out.push(';');
out.push_str(&asotr_head.date);
out.push(';');
out.push_str(&asotr_head.time);
out.push(';');
out.push_str(&format!("{};{} {}.{:02};",
&asotr_head.time_s,
&asotr_head.date,
&asotr_head.time,
asotr_head.time_mks));
if asotr_head.data_type == AsotrDataType::Temp ||
asotr_head.data_type == AsotrDataType::TempSet {
@ -104,11 +113,11 @@ pub mod asotr_data {
return Ok(out);
}
pub fn parse_data_dir(dir: &str) -> Result<(), String> {
pub fn parse_data_dir(dir: &str, disp: bool) -> Result<(), String> {
let mut data: Vec<String> = Vec::new();
println!("parse data from directory: {}", dir);
for (pattern, fname) in patterns_fnames_csv_data.iter() {
for (i, (pattern, fname)) in patterns_fnames_csv_data.iter().enumerate() {
let files = find_files_regex(dir, pattern)?;
for elem in files {
@ -117,6 +126,8 @@ pub mod asotr_data {
data.sort();
if disp { disp_data(&data, &patterns_disp[i])?; }
println!("save csv data to file: {}", fname);
save_data_csv(data.clone(), fname)?;
@ -159,19 +170,19 @@ pub mod asotr_data {
}
fn parse_filename(filename_full: String) -> Result<AsotrDataDesc, String> {
let mut filename = String::new();
let mut fname = String::new();
let msg_prev = format!("Error parsing filename {}:", filename_full);
match filename_full.rfind('/') {
Some(val) => { filename = (filename_full[val+1..filename_full.len()]).to_string(); }
_ => { filename = filename_full.clone(); }
Some(val) => { fname = (filename_full[val+1..filename_full.len()]).to_string(); }
_ => { fname = filename_full.clone(); }
}
if filename.len() != 32 {
if fname.len() != 32 {
return Err(format!("{} unsupported file", msg_prev));
}
let time_unix_ = filename[0..10].parse::<u64>();
let time_unix_ = fname[0..10].parse::<u64>();
let time_unix = match &time_unix_ {
Ok(data) => data,
Err(msg) => {
@ -180,7 +191,7 @@ pub mod asotr_data {
}
};
let data_type_ = filename[22..24].parse::<u8>();
let data_type_ = fname[22..24].parse::<u8>();
let data_type_u8 = match &data_type_ {
Ok(data) => data,
Err(msg) => {
@ -208,19 +219,19 @@ pub mod asotr_data {
// msg_prev, msg)); }
// };
// let _time_str_mks = filename[11..17].parse::<u32>();
// let time_mks = match &_time_str_mks {
// Ok(data) => data,
// Err(msg) => { return Err(format!("{}: expected digits in timestamp mks part ({})",
// msg_prev, msg)); }
// };
let _time_str_mks = fname[11..14].parse::<u32>();
let time_mks = match &_time_str_mks {
Ok(data) => data,
Err(msg) => { return Err(format!("{}: expected digits in timestamp mks part ({})",
msg_prev, msg)); }
};
let time: SystemTime = UNIX_EPOCH + Duration::from_secs(*time_unix);
let date_time = DateTime::<Utc>::from(time);
let date_s = date_time.format("%d.%m.%Y").to_string();
let time_s = date_time.format("%H:%M:%S").to_string();
let head = AsotrDataDesc::new(*time_unix, date_s, time_s, data_type);
let head = AsotrDataDesc::new(*time_unix, *time_mks, date_s, time_s, data_type);
return Ok(head);
}
@ -255,8 +266,31 @@ pub mod asotr_data {
return Ok(path_vec);
}
fn disp_data(data: &Vec<String>, about: &str) -> Result<(), String> {
println!("{}", about);
println!("{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}",
"timestamp_sec", "timestamp",
"ch1", "ch2", "ch3", "ch4", "ch5", "ch6");
for elem in data {
println!("{}", elem.replace(';', "\t"));
}
return Ok(())
}
fn save_data_csv(data: Vec<String>, fname: &str) -> Result<(), String> {
match std::fs::write(fname, data.join("\n")){
let mut data_w: Vec<String> = Vec::new();
data_w.push(format!("{};{};{};{};{};{};{};{}",
"timestamp_sec", "timestamp",
"ch1", "ch2", "ch3", "ch4", "ch5", "ch6"));
for elem in data {
data_w.push(elem);
}
match std::fs::write(fname, data_w.join("\n")) {
Ok(f) => f,
Err(msg) => {
return Err(format!("Error write data to csv file {}: {}", fname, msg))
@ -267,50 +301,47 @@ pub mod asotr_data {
}
}
#[derive(Parser, Debug)]
#[command(version, author, about, long_about = None)]
struct Cli {
/// file with ASOTR MVN data (.data01.asotr01(02), data02.asotr01(02), data06.asotr01(02))
#[arg(long, short = 'f', value_name = "FILE_DATA")]
filename: Option<String>,
/// directory with ASOTR MVN data
#[arg(long, short = 'd', value_name = "DIRECTORY_DATA")]
directory: Option<String>,
/// show data in console
#[arg(short = 's')]
show: bool,
}
fn main() {
use crate::asotr_data::*;
let arguments = Command::new("asotr_csv")
.version("0.1.0")
.author("Danila Gamkov <danila_gamkov@cosmos.ru>")
.about("raw data ASOTR MVN parser")
.arg(
Arg::new("file")
.short('f')
.long("filename")
.help(format!("file with ASOTR MVN data ({})", support_dtypes.to_string()))
)
.arg(
Arg::new("dir")
.short('d')
.long("directory")
.help(format!("directory with ASOTR MVN data"))
)
.get_matches();
let cli = Cli::parse();
let show = cli.show;
match arguments.get_one::<String>("file") {
Some(fname) => {
let s = match read_data(fname.clone()) {
Ok(elem) => elem,
Err(msg) => { println!("{}", msg); return; }
};
if let Some(fname) = &cli.filename {
println!("{}", s);
return;
},
_ => { }
};
let s = match read_data(fname.clone()) {
Ok(elem) => elem,
Err(msg) => { println!("{}", msg); return; }
};
match arguments.get_one::<String>("dir") {
Some(path) => {
match parse_data_dir(path) {
Ok(_) => {}
Err(msg) => { println!("{}", msg); return; }
}
return;
},
_ => { }
println!("{}", s);
return;
}
if let Some(dir) = &cli.directory {
match parse_data_dir(&dir.clone(), show) {
Ok(elem) => elem,
Err(msg) => { println!("{}", msg); return; }
}
return;
}
println!("Unexpected command. Type --help for more iformation");