1#[cfg(feature="tof-liftof")]
7pub mod ipbus;
8pub mod parsers;
9pub mod serialization;
10pub use serialization::Serialization;
11pub mod caraspace;
12#[cfg(feature="root")]
13pub mod root_reader;
14#[cfg(feature="root")]
15pub use root_reader::read_example;
16pub mod tof_reader;
17pub use tof_reader::TofPacketReader;
18pub mod tof_writer;
19pub use tof_writer::TofPacketWriter;
20pub mod telemetry_reader;
21pub use telemetry_reader::TelemetryPacketReader;
22pub mod telemetry_writer;
23pub use telemetry_writer::TelemetryPacketWriter;
24pub mod data_source;
25pub use data_source::DataSource;
26pub mod streamers;
27pub use streamers::*;
28
29
30#[cfg(feature="pybindings")]
31use std::path::PathBuf;
32
33use flate2::Compression;
34use flate2::write::GzEncoder;
35use flate2::read::GzDecoder;
36use diffy::{
37 apply_bytes,
38 Patch,
39 create_patch
40};
41use crate::prelude::*;
42
43#[derive(Debug, Clone)]
47pub enum FileType {
48 Unknown,
49 CalibrationFile(u8),
51 RunFile(u32),
53 SummaryFile(String),
56}
57
58pub fn list_path_contents_sorted(input: &str, pattern: Option<Regex>) -> Result<Vec<String>, io::Error> {
71 let path = Path::new(input);
72 match fs::metadata(path) {
73 Ok(metadata) => {
74 if metadata.is_file() {
75 let fname = String::from(input);
76 return Ok(vec![fname]);
77 }
78 if metadata.is_dir() {
79 let re : Regex;
80 match pattern {
81 None => {
82 re = Regex::new(GENERIC_ONLINE_FILE_PATTERH).unwrap();
85 }
86 Some(_re) => {
87 re = _re;
88 }
89 }
90 let mut entries: Vec<(u32, u32, String)> = fs::read_dir(path)?
91 .filter_map(Result::ok) .filter_map(|entry| {
93 let filename = format!("{}/{}", path.display(), entry.file_name().into_string().ok()?);
94 re.captures(&filename.clone()).map(|caps| {
95 let date = caps.get(1)?.as_str().parse::<u32>().ok()?;
96 let time = caps.get(2)?.as_str().parse::<u32>().ok()?;
97 Some((date, time, filename))
98 })?
99 })
100 .collect();
101
102 entries.sort_by(|a, b| (a.0, a.1).cmp(&(b.0, b.1)));
104 return Ok(entries.into_iter().map(|(_, _, name)| name).collect());
106 }
107 Err(io::Error::new(io::ErrorKind::Other, "Path exists but is neither a file nor a directory"))
108 }
109 Err(e) => Err(e),
110 }
111}
112
113#[cfg(feature="pybindings")]
126#[pyfunction]
127#[pyo3(name="list_path_contents_sorted")]
128#[pyo3(signature = ( input, pattern = None ))]
129pub fn list_path_contents_sorted_py(input: &str, pattern: Option<String>) -> PyResult<Option<Vec<String>>> {
130 let mut regex_pattern : Option<Regex> = None;
131 if let Some(pat_str) = pattern {
132 match Regex::new(&pat_str) {
133 Err(err) => {
134 let msg = format!("Unable to compile regex {}! {}. Check your regex syntax! Also try a raw string.", &pat_str, err);
135 return Err(PyValueError::new_err(msg));
136 }
137 Ok(re) => {
138 regex_pattern = Some(re);
139 }
140 }
141 }
142 match list_path_contents_sorted(input, regex_pattern) {
143 Err(err) => {
144 error!("Unable to get files! {err}");
145 return Err(PyValueError::new_err(err.to_string()));
146 }
147 Ok(files) => {
148 return Ok(Some(files));
149 }
150 }
151}
152
153#[cfg_attr(feature="pybindings", pyfunction)]
157pub fn get_utc_timestamp() -> String {
158 let now: DateTime<Utc> = Utc::now();
159 let timestamp_str = now.format(HUMAN_TIMESTAMP_FORMAT).to_string();
161 timestamp_str
162}
163
164#[cfg_attr(feature="pybindings", pyfunction)]
168pub fn get_utc_timestamp_from_unix(unix_time : f64) -> Option<String> {
169 let seconds = unix_time.trunc() as i64;
171 let nanoseconds = (unix_time.fract() * 1_000_000_000.0) as u32;
172
173 if let Some(dt) = Utc.timestamp_opt(seconds, nanoseconds).single() {
176 Some(dt.format("%y%m%d_%H%M%S").to_string())
177 } else {
178 None
179 }
180}
181
182#[cfg_attr(feature="pybindings", pyfunction)]
186pub fn get_unix_timestamp_from_telemetry(fname : &str) -> Option<u64> {
187 let tformat_re = Regex::new(GENERIC_TELEMETRY_FILE_PATTERN_CAPUTRE).unwrap();
188 let res = tformat_re.captures(fname).and_then(|caps| {
189 let map : HashMap<String, String> = tformat_re.capture_names()
190 .filter_map(|name| name)
191 .filter_map(|name| {
192 caps.name(name).map(|m| (name.to_string(), m.as_str().to_string()))
194 })
195 .collect();
196 Some(map)
197 });
198 return get_unix_timestamp(&res.unwrap()["utctime"], None);
199}
200
201#[cfg_attr(feature="pybindings", pyfunction)]
205pub fn get_utc_date() -> String {
206 let now: DateTime<Utc> = Utc::now();
207 let timestamp_str = now.format("%y%m%d").to_string();
209 timestamp_str
210}
211
212#[cfg_attr(feature="pybindings", pyfunction)]
225pub fn get_califilename(rb_id : u8, latest : bool) -> String {
226 let ts = get_utc_timestamp();
227 if latest {
228 format!("RB{rb_id:02}_latest.cali.tof.gaps")
229 } else {
230 format!("RB{rb_id:02}_{ts}.cali.tof.gaps")
231 }
232}
233
234#[cfg_attr(feature="pybindings", pyfunction)]
250pub fn get_runfilename(run : u32, subrun : u64, rb_id : Option<u8>, timestamp : Option<String>, tof_only : bool) -> String {
251 let ts : String;
252 match timestamp {
253 Some(_ts) => {
254 ts = _ts;
255 }
256 None => {
257 ts = get_utc_timestamp();
258 }
259 }
260 let fname : String;
261 match rb_id {
262 None => {
263 if tof_only {
264 fname = format!("Run{run}_{subrun}.{ts}.tof.gaps");
265 } else {
266 fname = format!("Run{run}_{subrun}.{ts}.gaps");
267 }
268 }
269 Some(rbid) => {
270 fname = format!("Run{run}_{subrun}.{ts}.RB{rbid:02}.tof.gaps");
271 }
272 }
273 fname
274}
275
276#[cfg_attr(feature="pybindings", pyfunction)]
283#[cfg_attr(feature="pybindings", pyo3(signature = (fname , pattern = None)))]
284pub fn get_rundata_from_file(fname : &str, pattern : Option<String>) -> Option<HashMap<String,String>> {
285 let regex_pattern : Regex;
286 if let Some(pat_str) = pattern {
287 match Regex::new(&pat_str) {
288 Err(err) => {
289 let msg = format!("Unable to compile regex {}! {}. Check your regex syntax! Also try a raw string.", &pat_str, err);
290 error!("{}",msg);
293 return None;
294 }
295 Ok(re) => {
296 regex_pattern = re;
297 }
298 }
299 } else {
300 regex_pattern = Regex::new(GENERIC_ONLINE_FILE_PATTERH_CAPTURE).unwrap();
301 }
302 let res : Option<HashMap<String,String>>;
303 res = regex_pattern.captures(fname).and_then(|caps| {
304 let map : HashMap<String, String> = regex_pattern.capture_names()
305 .filter_map(|name| name)
306 .filter_map(|name| {
307 caps.name(name).map(|m| (name.to_string(), m.as_str().to_string()))
309 })
310 .collect();
311 Some(map)
312 });
313 res
319}
320
321#[cfg_attr(feature="pybindings", pyfunction)]
333#[cfg_attr(feature="pybindings", pyo3(signature = (input , tformat = None )))]
334pub fn get_datetime(input : &str, tformat : Option<String>) -> Option<DateTime<Utc>> {
335 let mut date_time_format = String::from("%y%m%d_%H%M%S");
337 if let Some(tform) = tformat {
338 date_time_format = tform.to_string();
339 }
340 if let Ok(ndtime) = NaiveDateTime::parse_from_str(input, &date_time_format) {
341 let dt_utc : DateTime<Utc> = DateTime::<Utc>::from_naive_utc_and_offset(ndtime, Utc);
343 return Some(dt_utc);
344 } else {
345 error!("Unable to parse {} for format {}! You can specify formats trhough the tformat keyword", input, date_time_format);
346 return None;
347 }
348}
349
350#[cfg_attr(feature="pybindings", pyfunction)]
362#[cfg_attr(feature="pybindings", pyo3(signature = (input , tformat = None )))]
363pub fn get_unix_timestamp(input : &str, tformat : Option<String>) -> Option<u64> {
364 let dt = get_datetime(input, tformat);
365 if let Some(dt_) = dt {
366 return Some(dt_.timestamp() as u64);
369 } else {
370 return None;
371 }
372}
373
374#[derive(Debug, Copy, Clone, PartialEq,FromRepr, AsRefStr, EnumIter)]
378#[cfg_attr(feature = "pybindings", pyclass(eq, eq_int))]
379#[repr(u8)]
380pub enum DataSourceKind {
381 Unknown = 0,
382 TofFiles = 10,
385 TofStream = 11,
387 TelemetryFiles = 20,
390 TelemetryStream = 21,
393 CaraspaceFiles = 30,
397 CaraspaceStream = 31,
400 ROOTFiles = 40,
402}
403
404expand_and_test_enum!(DataSourceKind, test_datasourcekind_repr);
405
406#[cfg(feature = "pybindings")]
412#[pymethods]
413impl DataSourceKind {
414
415 #[getter]
416 fn __hash__(&self) -> usize {
417 (*self as u8) as usize
418 }
419}
420
421#[cfg(feature="pybindings")]
422pythonize_display!(DataSourceKind);
423
424#[macro_export]
429macro_rules! reader {
430 ($struct_name:ident, $element_type:ident) => {
431
432 use crate::io::DataReader;
433 use crate::io::Serialization;
434
435 impl Iterator for $struct_name {
436 type Item = $element_type;
437 fn next(&mut self) -> Option<Self::Item> {
438 self.read_next()
439 }
440 }
441
442 impl DataReader<$element_type> for $struct_name {
443 fn get_header0(&self) -> u8 {
444 ($element_type::HEAD & 0x1) as u8
445 }
446
447 fn get_header1(&self) -> u8 {
448 ($element_type::HEAD & 0x2) as u8
449 }
450
451 fn get_file_idx(&self) -> usize {
452 self.file_idx }
454
455 fn set_file_idx(&mut self, file_idx : usize) {
456 self.file_idx = file_idx;
457 }
458
459 fn get_filenames(&self) -> &Vec<String> {
460 &self.filenames
461 }
462
463 fn set_cursor(&mut self, pos : usize) {
464 self.cursor = pos;
465 }
466
467 fn set_file_reader(&mut self, reader : BufReader<File>) {
468 self.file_reader = reader;
469 }
470
471 fn read_next(&mut self) -> Option<$element_type> {
472 self.read_next_item()
473 }
474
475 fn prime_next_file(&mut self) -> Option<usize> {
477 if self.file_idx == self.filenames.len() -1 {
478 return None;
479 } else {
480 self.file_idx += 1;
481 let nextfilename : &str = self.filenames[self.file_idx].as_str();
482 let nextfile = OpenOptions::new().create(false).append(false).read(true).open(nextfilename).expect("Unable to open file {nextfilename}");
483 self.file_reader = BufReader::new(nextfile);
484 self.cursor = 0;
485 return Some(self.file_idx);
486 }
487 }
488 }
489 }
490}
491
492pub trait DataReader<T>
495 where T : Default + Serialization {
496 fn get_header0(&self) -> u8;
502 fn get_header1(&self) -> u8;
503
504 fn get_filenames(&self) -> &Vec<String>;
506
507 fn get_file_idx(&self) -> usize;
510
511 fn set_file_idx(&mut self, idx : usize);
514
515 fn set_file_reader(&mut self, freader : BufReader<File>);
517
518 fn prime_next_file(&mut self) -> Option<usize>;
520
521 fn get_current_filename(&self) -> Option<&str> {
524 if self.get_filenames().len() <= self.get_file_idx() {
526 return None;
527 }
528 Some(self.get_filenames()[self.get_file_idx()].as_str())
529 }
530
531 fn set_cursor(&mut self, pos : usize);
533
534 fn read_next(&mut self) -> Option<T>;
537
538 fn first(&mut self) -> Option<T> {
541 match self.rewind() {
542 Err(err) => {
543 error!("Error when rewinding files! {err}");
544 return None;
545 }
546 Ok(_) => ()
547 }
548 let pack = self.read_next();
549 match self.rewind() {
550 Err(err) => {
551 error!("Error when rewinding files! {err}");
552 }
553 Ok(_) => ()
554 }
555 return pack;
556 }
557
558 fn last(&mut self) -> Option<T> {
561 self.set_file_idx(self.get_filenames().len() - 1);
562 let lastfilename = self.get_filenames()[self.get_file_idx()].as_str();
563 let lastfile = OpenOptions::new().create(false).append(false).read(true).open(lastfilename).expect("Unable to open file {nextfilename}");
564 self.set_file_reader(BufReader::new(lastfile));
565 self.set_cursor(0);
566 let mut tp = T::default();
567 let mut idx = 0;
568 loop {
569 match self.read_next() {
570 None => {
571 match self.rewind() {
572 Err(err) => {
573 error!("Error when rewinding files! {err}");
574 }
575 Ok(_) => ()
576 }
577 if idx == 0 {
578 return None;
579 } else {
580 return Some(tp);
581 }
582 }
583 Some(pack) => {
584 idx += 1;
585 tp = pack;
586 continue;
587 }
588 }
589 }
590 }
591
592 fn rewind(&mut self) -> io::Result<()> {
596 let firstfile = &self.get_filenames()[0];
597 let file = OpenOptions::new().create(false).append(false).read(true).open(&firstfile)?;
598 self.set_file_reader(BufReader::new(file));
599 self.set_cursor(0);
600 self.set_file_idx(0);
601 Ok(())
602 }
603}
604
605pub fn compress_toml(file_path: &Path) -> Result<Vec<u8>, io::Error> {
608 let mut input_file = File::open(file_path)?;
609 let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
610 io::copy(&mut input_file, &mut encoder)?;
611 encoder.finish()
612}
613
614#[cfg(feature="pybindings")]
617#[pyfunction]
618#[pyo3(name="compress_toml")]
619pub fn compress_toml_py(file_path: String) -> Result<Vec<u8>, io::Error> {
620 let path_buff = PathBuf::from(file_path);
621 compress_toml(&path_buff)
622}
623
624pub fn decompress_toml(compressed_data: &[u8], output_path: &Path) -> Result<(), io::Error> {
626 let mut decoder = GzDecoder::new(compressed_data);
627 let mut output_file = File::create(output_path)?;
628 io::copy(&mut decoder, &mut output_file)?;
629 Ok(())
630}
631
632#[cfg(feature="pybindings")]
634#[pyfunction]
635#[pyo3(name="decompress_toml")]
636pub fn decompress_toml_py(compressed_data: &[u8], output_path: String) -> Result<(), io::Error> {
637 let path_buff = PathBuf::from(output_path);
638 decompress_toml(compressed_data, &path_buff)
639}
640
641
642pub fn create_compressed_diff(old_path: &Path, new_path: &Path) -> Result<Vec<u8>, io::Error> {
648 let old_text = fs::read_to_string(old_path)?;
649 let new_text = fs::read_to_string(new_path)?;
650 let diff = create_patch(&old_text, &new_text);
651 let diff_bytes = diff.to_bytes();
652 let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
653 io::copy(&mut diff_bytes.as_slice(), &mut encoder)?;
654 encoder.finish()
655}
656
657#[cfg(feature="pybindings")]
663#[pyfunction]
664#[pyo3(name="create_compressed_diff")]
665pub fn create_compressed_diff_py(old_path: String, new_path: String) -> Result<Vec<u8>, io::Error> {
666 let old_file = PathBuf::from(old_path);
667 let new_file = PathBuf::from(new_path);
668 create_compressed_diff(&old_file, &new_file)
669}
670
671
672pub fn apply_diff_to_file(compressed_bytes : Vec<u8>, original_file_path: &str) -> io::Result<()> {
682 let mut decoder = GzDecoder::new(&compressed_bytes[..]);
683 let mut uncompressed_data = Vec::new();
684 match decoder.read_to_end(&mut uncompressed_data) {
685 Ok(_) => (),
686 Err(e) => {
687 error!("Unable to decompress the received bytes!");
688 return Err(e);
689 }
690 }
691
692 let mut original_file = fs::File::open(original_file_path)?;
694 let mut original_content = String::new();
695 original_file.read_to_string(&mut original_content)?;
696 match Patch::from_bytes(&uncompressed_data.as_slice()) {
697 Ok(patch) => {
698 info!("Got patch {:?}", patch);
699 match apply_bytes(&original_content.as_bytes(), &patch) {
700 Ok(modified_content) => {
701 let mut output_file = fs::File::create(original_file_path)?;
702 output_file.write_all(&modified_content.as_slice())?;
703 }
704 Err(err) => {
705 error!("Unable to apply the patch {err}");
706 }
707 }
708 }
709 Err(err) => {
710 error!("Unable to apply the patch! {err}");
711 }
712 }
713 Ok(())
714}
715
716#[cfg(feature="pybindings")]
726#[pyfunction]
727#[pyo3(name="apply_diff_to_file")]
728pub fn apply_diff_to_file_py(compressed_bytes : Vec<u8>, original_file_path: &str) -> io::Result<()> {
729 apply_diff_to_file(compressed_bytes, original_file_path)
730}
731