1#[cfg(feature="tof-liftof")]
7pub mod ipbus;
8pub mod parsers;
9pub mod serialization;
10pub use serialization::Serialization;
11pub mod caraspace;
12#[cfg(feature="root")]
13pub mod root_reader;
14#[cfg(feature="root")]
15pub use root_reader::read_example;
16pub mod tof_reader;
17pub use tof_reader::TofPacketReader;
18pub mod tof_writer;
19pub use tof_writer::TofPacketWriter;
20pub mod telemetry_reader;
21pub use telemetry_reader::TelemetryPacketReader;
22pub mod data_source;
23pub use data_source::DataSource;
24pub mod streamers;
25pub use streamers::*;
26
27#[cfg(feature="pybindings")]
28use std::path::PathBuf;
29
30use flate2::Compression;
31use flate2::write::GzEncoder;
32use flate2::read::GzDecoder;
33use diffy::{
34 apply_bytes,
35 Patch,
36 create_patch
37};
38use crate::prelude::*;
39
40#[derive(Debug, Clone)]
44pub enum FileType {
45 Unknown,
46 CalibrationFile(u8),
48 RunFile(u32),
50 SummaryFile(String),
53}
54
55pub fn list_path_contents_sorted(input: &str, pattern: Option<Regex>) -> Result<Vec<String>, io::Error> {
68 let path = Path::new(input);
69 match fs::metadata(path) {
70 Ok(metadata) => {
71 if metadata.is_file() {
72 let fname = String::from(input);
73 return Ok(vec![fname]);
74 }
75 if metadata.is_dir() {
76 let re : Regex;
77 match pattern {
78 None => {
79 re = Regex::new(GENERIC_ONLINE_FILE_PATTERH).unwrap();
82 }
83 Some(_re) => {
84 re = _re;
85 }
86 }
87 let mut entries: Vec<(u32, u32, String)> = fs::read_dir(path)?
88 .filter_map(Result::ok) .filter_map(|entry| {
90 let filename = format!("{}/{}", path.display(), entry.file_name().into_string().ok()?);
91 re.captures(&filename.clone()).map(|caps| {
92 let date = caps.get(1)?.as_str().parse::<u32>().ok()?;
93 let time = caps.get(2)?.as_str().parse::<u32>().ok()?;
94 Some((date, time, filename))
95 })?
96 })
97 .collect();
98
99 entries.sort_by(|a, b| (a.0, a.1).cmp(&(b.0, b.1)));
101 return Ok(entries.into_iter().map(|(_, _, name)| name).collect());
103 }
104 Err(io::Error::new(io::ErrorKind::Other, "Path exists but is neither a file nor a directory"))
105 }
106 Err(e) => Err(e),
107 }
108}
109
110#[cfg(feature="pybindings")]
123#[pyfunction]
124#[pyo3(name="list_path_contents_sorted")]
125#[pyo3(signature = ( input, pattern = None ))]
126pub fn list_path_contents_sorted_py(input: &str, pattern: Option<String>) -> PyResult<Option<Vec<String>>> {
127 let mut regex_pattern : Option<Regex> = None;
128 if let Some(pat_str) = pattern {
129 match Regex::new(&pat_str) {
130 Err(err) => {
131 let msg = format!("Unable to compile regex {}! {}. Check your regex syntax! Also try a raw string.", &pat_str, err);
132 return Err(PyValueError::new_err(msg));
133 }
134 Ok(re) => {
135 regex_pattern = Some(re);
136 }
137 }
138 }
139 match list_path_contents_sorted(input, regex_pattern) {
140 Err(err) => {
141 error!("Unable to get files! {err}");
142 return Err(PyValueError::new_err(err.to_string()));
143 }
144 Ok(files) => {
145 return Ok(Some(files));
146 }
147 }
148}
149
150#[cfg_attr(feature="pybindings", pyfunction)]
154pub fn get_utc_timestamp() -> String {
155 let now: DateTime<Utc> = Utc::now();
156 let timestamp_str = now.format(HUMAN_TIMESTAMP_FORMAT).to_string();
158 timestamp_str
159}
160
161#[cfg_attr(feature="pybindings", pyfunction)]
163pub fn get_unix_timestamp_from_telemetry(fname : &str) -> Option<u64> {
164 let tformat_re = Regex::new(GENERIC_TELEMETRY_FILE_PATTERN_CAPUTRE).unwrap();
165 let res = tformat_re.captures(fname).and_then(|caps| {
166 let map : HashMap<String, String> = tformat_re.capture_names()
167 .filter_map(|name| name)
168 .filter_map(|name| {
169 caps.name(name).map(|m| (name.to_string(), m.as_str().to_string()))
171 })
172 .collect();
173 Some(map)
174 });
175 return get_unix_timestamp(&res.unwrap()["utctime"], None);
176}
177
178#[cfg_attr(feature="pybindings", pyfunction)]
182pub fn get_utc_date() -> String {
183 let now: DateTime<Utc> = Utc::now();
184 let timestamp_str = now.format("%y%m%d").to_string();
186 timestamp_str
187}
188
189#[cfg_attr(feature="pybindings", pyfunction)]
202pub fn get_califilename(rb_id : u8, latest : bool) -> String {
203 let ts = get_utc_timestamp();
204 if latest {
205 format!("RB{rb_id:02}_latest.cali.tof.gaps")
206 } else {
207 format!("RB{rb_id:02}_{ts}.cali.tof.gaps")
208 }
209}
210
211#[cfg_attr(feature="pybindings", pyfunction)]
227pub fn get_runfilename(run : u32, subrun : u64, rb_id : Option<u8>, timestamp : Option<String>, tof_only : bool) -> String {
228 let ts : String;
229 match timestamp {
230 Some(_ts) => {
231 ts = _ts;
232 }
233 None => {
234 ts = get_utc_timestamp();
235 }
236 }
237 let fname : String;
238 match rb_id {
239 None => {
240 if tof_only {
241 fname = format!("Run{run}_{subrun}.{ts}.tof.gaps");
242 } else {
243 fname = format!("Run{run}_{subrun}.{ts}.gaps");
244 }
245 }
246 Some(rbid) => {
247 fname = format!("Run{run}_{subrun}.{ts}.RB{rbid:02}.tof.gaps");
248 }
249 }
250 fname
251}
252
253#[cfg_attr(feature="pybindings", pyfunction)]
260#[cfg_attr(feature="pybindings", pyo3(signature = (fname , pattern = None)))]
261pub fn get_rundata_from_file(fname : &str, pattern : Option<String>) -> Option<HashMap<String,String>> {
262 let regex_pattern : Regex;
263 if let Some(pat_str) = pattern {
264 match Regex::new(&pat_str) {
265 Err(err) => {
266 let msg = format!("Unable to compile regex {}! {}. Check your regex syntax! Also try a raw string.", &pat_str, err);
267 error!("{}",msg);
270 return None;
271 }
272 Ok(re) => {
273 regex_pattern = re;
274 }
275 }
276 } else {
277 regex_pattern = Regex::new(GENERIC_ONLINE_FILE_PATTERH_CAPTURE).unwrap();
278 }
279 let res : Option<HashMap<String,String>>;
280 res = regex_pattern.captures(fname).and_then(|caps| {
281 let map : HashMap<String, String> = regex_pattern.capture_names()
282 .filter_map(|name| name)
283 .filter_map(|name| {
284 caps.name(name).map(|m| (name.to_string(), m.as_str().to_string()))
286 })
287 .collect();
288 Some(map)
289 });
290 res
296}
297
298#[cfg_attr(feature="pybindings", pyfunction)]
310#[cfg_attr(feature="pybindings", pyo3(signature = (input , tformat = None )))]
311pub fn get_datetime(input : &str, tformat : Option<String>) -> Option<DateTime<Utc>> {
312 let mut date_time_format = String::from("%y%m%d_%H%M%S");
314 if let Some(tform) = tformat {
315 date_time_format = tform.to_string();
316 }
317 if let Ok(ndtime) = NaiveDateTime::parse_from_str(input, &date_time_format) {
318 let dt_utc : DateTime<Utc> = DateTime::<Utc>::from_naive_utc_and_offset(ndtime, Utc);
320 return Some(dt_utc);
321 } else {
322 error!("Unable to parse {} for format {}! You can specify formats trhough the tformat keyword", input, date_time_format);
323 return None;
324 }
325}
326
327#[cfg_attr(feature="pybindings", pyfunction)]
339#[cfg_attr(feature="pybindings", pyo3(signature = (input , tformat = None )))]
340pub fn get_unix_timestamp(input : &str, tformat : Option<String>) -> Option<u64> {
341 let dt = get_datetime(input, tformat);
342 if let Some(dt_) = dt {
343 return Some(dt_.timestamp() as u64);
346 } else {
347 return None;
348 }
349}
350
351#[derive(Debug, Copy, Clone, PartialEq,FromRepr, AsRefStr, EnumIter)]
355#[cfg_attr(feature = "pybindings", pyclass(eq, eq_int))]
356#[repr(u8)]
357pub enum DataSourceKind {
358 Unknown = 0,
359 TofFiles = 10,
362 TofStream = 11,
364 TelemetryFiles = 20,
367 TelemetryStream = 21,
370 CaraspaceFiles = 30,
374 CaraspaceStream = 31,
377 ROOTFiles = 40,
379}
380
381expand_and_test_enum!(DataSourceKind, test_datasourcekind_repr);
382
383#[cfg(feature = "pybindings")]
389#[pymethods]
390impl DataSourceKind {
391
392 #[getter]
393 fn __hash__(&self) -> usize {
394 (*self as u8) as usize
395 }
396}
397
398#[cfg(feature="pybindings")]
399pythonize_display!(DataSourceKind);
400
401#[macro_export]
406macro_rules! reader {
407 ($struct_name:ident, $element_type:ident) => {
408
409 use crate::io::DataReader;
410 use crate::io::Serialization;
411
412 impl Iterator for $struct_name {
413 type Item = $element_type;
414 fn next(&mut self) -> Option<Self::Item> {
415 self.read_next()
416 }
417 }
418
419 impl DataReader<$element_type> for $struct_name {
420 fn get_header0(&self) -> u8 {
421 ($element_type::HEAD & 0x1) as u8
422 }
423
424 fn get_header1(&self) -> u8 {
425 ($element_type::HEAD & 0x2) as u8
426 }
427
428 fn get_file_idx(&self) -> usize {
429 self.file_idx }
431
432 fn set_file_idx(&mut self, file_idx : usize) {
433 self.file_idx = file_idx;
434 }
435
436 fn get_filenames(&self) -> &Vec<String> {
437 &self.filenames
438 }
439
440 fn set_cursor(&mut self, pos : usize) {
441 self.cursor = pos;
442 }
443
444 fn set_file_reader(&mut self, reader : BufReader<File>) {
445 self.file_reader = reader;
446 }
447
448 fn read_next(&mut self) -> Option<$element_type> {
449 self.read_next_item()
450 }
451
452 fn prime_next_file(&mut self) -> Option<usize> {
454 if self.file_idx == self.filenames.len() -1 {
455 return None;
456 } else {
457 self.file_idx += 1;
458 let nextfilename : &str = self.filenames[self.file_idx].as_str();
459 let nextfile = OpenOptions::new().create(false).append(false).read(true).open(nextfilename).expect("Unable to open file {nextfilename}");
460 self.file_reader = BufReader::new(nextfile);
461 self.cursor = 0;
462 return Some(self.file_idx);
463 }
464 }
465 }
466 }
467}
468
469pub trait DataReader<T>
472 where T : Default + Serialization {
473 fn get_header0(&self) -> u8;
479 fn get_header1(&self) -> u8;
480
481 fn get_filenames(&self) -> &Vec<String>;
483
484 fn get_file_idx(&self) -> usize;
487
488 fn set_file_idx(&mut self, idx : usize);
491
492 fn set_file_reader(&mut self, freader : BufReader<File>);
494
495 fn prime_next_file(&mut self) -> Option<usize>;
497
498 fn get_current_filename(&self) -> Option<&str> {
501 if self.get_filenames().len() <= self.get_file_idx() {
503 return None;
504 }
505 Some(self.get_filenames()[self.get_file_idx()].as_str())
506 }
507
508 fn set_cursor(&mut self, pos : usize);
510
511 fn read_next(&mut self) -> Option<T>;
514
515 fn first(&mut self) -> Option<T> {
518 match self.rewind() {
519 Err(err) => {
520 error!("Error when rewinding files! {err}");
521 return None;
522 }
523 Ok(_) => ()
524 }
525 let pack = self.read_next();
526 match self.rewind() {
527 Err(err) => {
528 error!("Error when rewinding files! {err}");
529 }
530 Ok(_) => ()
531 }
532 return pack;
533 }
534
535 fn last(&mut self) -> Option<T> {
538 self.set_file_idx(self.get_filenames().len() - 1);
539 let lastfilename = self.get_filenames()[self.get_file_idx()].as_str();
540 let lastfile = OpenOptions::new().create(false).append(false).read(true).open(lastfilename).expect("Unable to open file {nextfilename}");
541 self.set_file_reader(BufReader::new(lastfile));
542 self.set_cursor(0);
543 let mut tp = T::default();
544 let mut idx = 0;
545 loop {
546 match self.read_next() {
547 None => {
548 match self.rewind() {
549 Err(err) => {
550 error!("Error when rewinding files! {err}");
551 }
552 Ok(_) => ()
553 }
554 if idx == 0 {
555 return None;
556 } else {
557 return Some(tp);
558 }
559 }
560 Some(pack) => {
561 idx += 1;
562 tp = pack;
563 continue;
564 }
565 }
566 }
567 }
568
569 fn rewind(&mut self) -> io::Result<()> {
573 let firstfile = &self.get_filenames()[0];
574 let file = OpenOptions::new().create(false).append(false).read(true).open(&firstfile)?;
575 self.set_file_reader(BufReader::new(file));
576 self.set_cursor(0);
577 self.set_file_idx(0);
578 Ok(())
579 }
580}
581
582pub fn compress_toml(file_path: &Path) -> Result<Vec<u8>, io::Error> {
585 let mut input_file = File::open(file_path)?;
586 let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
587 io::copy(&mut input_file, &mut encoder)?;
588 encoder.finish()
589}
590
591#[cfg(feature="pybindings")]
594#[pyfunction]
595#[pyo3(name="compress_toml")]
596pub fn compress_toml_py(file_path: String) -> Result<Vec<u8>, io::Error> {
597 let path_buff = PathBuf::from(file_path);
598 compress_toml(&path_buff)
599}
600
601pub fn decompress_toml(compressed_data: &[u8], output_path: &Path) -> Result<(), io::Error> {
603 let mut decoder = GzDecoder::new(compressed_data);
604 let mut output_file = File::create(output_path)?;
605 io::copy(&mut decoder, &mut output_file)?;
606 Ok(())
607}
608
609#[cfg(feature="pybindings")]
611#[pyfunction]
612#[pyo3(name="decompress_toml")]
613pub fn decompress_toml_py(compressed_data: &[u8], output_path: String) -> Result<(), io::Error> {
614 let path_buff = PathBuf::from(output_path);
615 decompress_toml(compressed_data, &path_buff)
616}
617
618
619pub fn create_compressed_diff(old_path: &Path, new_path: &Path) -> Result<Vec<u8>, io::Error> {
625 let old_text = fs::read_to_string(old_path)?;
626 let new_text = fs::read_to_string(new_path)?;
627 let diff = create_patch(&old_text, &new_text);
628 let diff_bytes = diff.to_bytes();
629 let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
630 io::copy(&mut diff_bytes.as_slice(), &mut encoder)?;
631 encoder.finish()
632}
633
634#[cfg(feature="pybindings")]
640#[pyfunction]
641#[pyo3(name="create_compressed_diff")]
642pub fn create_compressed_diff_py(old_path: String, new_path: String) -> Result<Vec<u8>, io::Error> {
643 let old_file = PathBuf::from(old_path);
644 let new_file = PathBuf::from(new_path);
645 create_compressed_diff(&old_file, &new_file)
646}
647
648
649pub fn apply_diff_to_file(compressed_bytes : Vec<u8>, original_file_path: &str) -> io::Result<()> {
659 let mut decoder = GzDecoder::new(&compressed_bytes[..]);
660 let mut uncompressed_data = Vec::new();
661 match decoder.read_to_end(&mut uncompressed_data) {
662 Ok(_) => (),
663 Err(e) => {
664 error!("Unable to decompress the received bytes!");
665 return Err(e);
666 }
667 }
668
669 let mut original_file = fs::File::open(original_file_path)?;
671 let mut original_content = String::new();
672 original_file.read_to_string(&mut original_content)?;
673 match Patch::from_bytes(&uncompressed_data.as_slice()) {
674 Ok(patch) => {
675 info!("Got patch {:?}", patch);
676 match apply_bytes(&original_content.as_bytes(), &patch) {
677 Ok(modified_content) => {
678 let mut output_file = fs::File::create(original_file_path)?;
679 output_file.write_all(&modified_content.as_slice())?;
680 }
681 Err(err) => {
682 error!("Unable to apply the patch {err}");
683 }
684 }
685 }
686 Err(err) => {
687 error!("Unable to apply the patch! {err}");
688 }
689 }
690 Ok(())
691}
692
693#[cfg(feature="pybindings")]
703#[pyfunction]
704#[pyo3(name="apply_diff_to_file")]
705pub fn apply_diff_to_file_py(compressed_bytes : Vec<u8>, original_file_path: &str) -> io::Result<()> {
706 apply_diff_to_file(compressed_bytes, original_file_path)
707}
708