gondola_core/
io.rs

1//! gaps-online-software i/o system
2//!
3// This file is part of gaps-online-software and published 
4// under the GPLv3 license
5
6#[cfg(feature="tof-liftof")]
7pub mod ipbus;
8pub mod parsers;
9pub mod serialization;
10pub use serialization::Serialization;
11pub mod caraspace;
12#[cfg(feature="root")]
13pub mod root_reader;
14#[cfg(feature="root")]
15pub use root_reader::read_example;
16pub mod tof_reader;
17pub use tof_reader::TofPacketReader;
18pub mod tof_writer;
19pub use tof_writer::TofPacketWriter;
20pub mod telemetry_reader;
21pub use telemetry_reader::TelemetryPacketReader;
22pub mod data_source;
23pub use data_source::DataSource;
24pub mod streamers;
25pub use streamers::*;
26use crate::prelude::*;
27
28//----------------------------------------------------------
29
30/// Types of files
31#[derive(Debug, Clone)]
32pub enum FileType {
33  Unknown,
34  /// Calibration file for specific RB with id
35  CalibrationFile(u8),
36  /// A regular run file with TofEvents
37  RunFile(u32),
38  /// A file created from a file with TofEvents which 
39  /// contains only TofEventSummary
40  SummaryFile(String),
41}
42
43//----------------------------------------------------------
44
45/// Get all filenames in the current path sorted by timestamp if available
46/// If the given path is a file and not a directory, return only that 
47/// file instead
48///
49/// # Arguments:
50///
51///    * input   : name of the target directory
52///    * pattern : the regex pattern to look for. That the sorting works,
53///                the pattern needs to return a date for the first
54///                captured argument and a time for the second captured argument
55pub fn list_path_contents_sorted(input: &str, pattern: Option<Regex>) -> Result<Vec<String>, io::Error> {
56  let path = Path::new(input);
57  match fs::metadata(path) {
58    Ok(metadata) => {
59      if metadata.is_file() {
60        let fname = String::from(input);
61        return Ok(vec![fname]);
62      } 
63      if metadata.is_dir() {
64        let re : Regex;
65        match pattern {
66          None => {
67            // use a default pattern which matches mmost cases  
68            //re = Regex::new(r"Run\d+_\d+\.(\d{6})_(\d{6})UTC(\.tof)?\.gaps$").unwrap();
69            re = Regex::new(GENERIC_ONLINE_FILE_PATTERH).unwrap();
70          }
71          Some(_re) => {
72            re = _re;
73          }
74        }
75        let mut entries: Vec<(u32, u32, String)> = fs::read_dir(path)?
76          .filter_map(Result::ok) // Ignore unreadable entries
77          .filter_map(|entry| {
78            let filename = format!("{}/{}", path.display(), entry.file_name().into_string().ok()?);
79            re.captures(&filename.clone()).map(|caps| {
80              let date = caps.get(1)?.as_str().parse::<u32>().ok()?;
81              let time = caps.get(2)?.as_str().parse::<u32>().ok()?;
82              Some((date, time, filename))
83            })?
84          })
85          .collect();
86
87        // Sort by (date, time)
88        entries.sort_by(|a, b| (a.0, a.1).cmp(&(b.0, b.1)));
89        // Return only filenames
90        return Ok(entries.into_iter().map(|(_, _, name)| name).collect());
91      } 
92      Err(io::Error::new(io::ErrorKind::Other, "Path exists but is neither a file nor a directory"))
93    }
94    Err(e) => Err(e),
95  }
96}
97
98//----------------------------------------------------------
99
100/// Get all filenames in the current path sorted by timestamp if available
101/// If the given path is a file and not a directory, return only that 
102/// file instead
103///
104/// # Arguments:
105///
106///    * input   : name of the target directory
107///    * pattern : the regex pattern to look for. That the sorting works,
108///                the pattern needs to return a date for the first
109///                captured argument and a time for the second captured argument
110#[cfg(feature="pybindings")]
111#[pyfunction]
112#[pyo3(name="list_path_contents_sorted")]
113#[pyo3(signature = ( input, pattern = None ))]
114pub fn list_path_contents_sorted_py(input: &str, pattern: Option<String>) -> PyResult<Option<Vec<String>>> {
115  let mut regex_pattern : Option<Regex> = None;
116  if let Some(pat_str) = pattern {
117    match Regex::new(&pat_str) {
118      Err(err) => {
119        let msg = format!("Unable to compile regex {}! {}. Check your regex syntax! Also try a raw string.", &pat_str, err); 
120        return Err(PyValueError::new_err(msg));
121      }
122      Ok(re) => {
123        regex_pattern = Some(re);
124      }
125    }
126  }
127  match list_path_contents_sorted(input, regex_pattern) {
128    Err(err) => {  
129      error!("Unable to get files! {err}");
130      return Err(PyValueError::new_err(err.to_string()));
131    }
132    Ok(files) => {
133      return Ok(Some(files));
134    }
135  }
136}
137
138//----------------------------------------------------------
139
140/// Get a human readable timestamp for NOW
141#[cfg_attr(feature="pybindings", pyfunction)]
142pub fn get_utc_timestamp() -> String {
143  let now: DateTime<Utc> = Utc::now();
144  //let timestamp_str = now.format("%Y_%m_%d-%H_%M_%S").to_string();
145  let timestamp_str = now.format(HUMAN_TIMESTAMP_FORMAT).to_string();
146  timestamp_str
147}
148
149/// Retrieve the utc timestamp from any telemetry (binary) file 
150#[cfg_attr(feature="pybindings", pyfunction)]
151pub fn get_unix_timestamp_from_telemetry(fname : &str) -> Option<u64> { 
152  let tformat_re = Regex::new(GENERIC_TELEMETRY_FILE_PATTERN_CAPUTRE).unwrap();
153  let res = tformat_re.captures(fname).and_then(|caps| {
154    let map : HashMap<String, String> = tformat_re.capture_names()
155      .filter_map(|name| name)
156      .filter_map(|name| { 
157        //caps.name(name).map(|m| (m.as_str().to_string(), m.as_str().to_string()))
158        caps.name(name).map(|m| (name.to_string(), m.as_str().to_string()))
159      })
160      .collect();
161    Some(map)
162  });
163  return get_unix_timestamp(&res.unwrap()["utctime"], None); 
164}
165
166//----------------------------------------------------------
167
168/// Create date string in YYMMDD format
169#[cfg_attr(feature="pybindings", pyfunction)]
170pub fn get_utc_date() -> String {
171  let now: DateTime<Utc> = Utc::now();
172  //let timestamp_str = now.format("%Y_%m_%d-%H_%M_%S").to_string();
173  let timestamp_str = now.format("%y%m%d").to_string();
174  timestamp_str
175}
176
177//----------------------------------------------------------
178
179/// A standardized name for calibration files saved by 
180/// the liftof suite
181///
182/// # Arguments
183///
184/// * rb_id   : unique identfier for the 
185///             Readoutboard (1-50)
186/// * default : if default, just add 
187///             "latest" instead of 
188///             a timestamp
189#[cfg_attr(feature="pybindings", pyfunction)]
190pub fn get_califilename(rb_id : u8, latest : bool) -> String {
191  let ts = get_utc_timestamp();
192  if latest {
193    format!("RB{rb_id:02}_latest.cali.tof.gaps")
194  } else {
195    format!("RB{rb_id:02}_{ts}.cali.tof.gaps")
196  }
197}
198
199//----------------------------------------------------------
200
201/// A standardized name for regular run files saved by
202/// the liftof suite
203///
204/// # Arguments
205///
206/// * run       : run id (identifier)
207/// * subrun    :  subrun id (identifier of file # within
208///                the run
209/// * rb_id     :  in case this should be used on the rb, 
210///                a rb id can be specified as well
211/// * timestamp :  substitute the current time with this timestamp
212///                (or basically any other string) instead.
213#[cfg_attr(feature="pybindings", pyfunction)]
214pub fn get_runfilename(run : u32, subrun : u64, rb_id : Option<u8>, timestamp : Option<String>) -> String {
215  let ts : String;
216  match timestamp {
217    Some(_ts) => {
218      ts = _ts;
219    }
220    None => {
221      ts = get_utc_timestamp();
222    }
223  }
224  let fname : String;
225  match rb_id {
226    None => {
227      fname = format!("Run{run}_{subrun}.{ts}.gaps");
228    }
229    Some(rbid) => {
230      fname = format!("Run{run}_{subrun}.{ts}.RB{rbid:02}.gaps");
231    }
232  }
233  fname
234}
235
236//----------------------------------------------------------
237    
238/// Get the timestamp from a .tof.gaps file
239///
240/// # Arguments:
241///   * fname : Filename of .tof.gaps file
242#[cfg_attr(feature="pybindings", pyfunction)]
243#[cfg_attr(feature="pybindings", pyo3(signature = (fname , pattern = None)))]
244pub fn get_rundata_from_file(fname : &str, pattern : Option<String>) -> Option<HashMap<String,String>> {
245  let regex_pattern : Regex;
246  if let Some(pat_str) = pattern {
247    match Regex::new(&pat_str) {
248      Err(err) => {
249        let msg = format!("Unable to compile regex {}! {}. Check your regex syntax! Also try a raw string.", &pat_str, err); 
250        //return Err(PyValueError::new_err(msg));
251        //return Err(err);
252        error!("{}",msg);
253        return None;
254      }
255      Ok(re) => {
256        regex_pattern = re;
257      }
258    }
259  } else {
260    regex_pattern = Regex::new(GENERIC_ONLINE_FILE_PATTERH_CAPTURE).unwrap();
261  }
262  let res : Option<HashMap<String,String>>;
263  res = regex_pattern.captures(fname).and_then(|caps| {
264    let map : HashMap<String, String> = regex_pattern.capture_names()
265      .filter_map(|name| name)
266      .filter_map(|name| { 
267        //caps.name(name).map(|m| (m.as_str().to_string(), m.as_str().to_string()))
268        caps.name(name).map(|m| (name.to_string(), m.as_str().to_string()))
269      })
270      .collect();
271    Some(map)
272  });
273  //ts = pattern.search(str(fname)).groupdict()['tdate']
274  //#print (ts)
275  //ts = datetime.strptime(ts, '%y%m%d_%H%M%S')
276  //ts = ts.replace(tzinfo=timezone.utc)
277  //return ts
278  res
279}
280
281//----------------------------------------------------------
282
283/// Retrieve the DateTime object from a string as used in 
284/// the names of the run files
285///
286/// # Arguments:
287///
288///   * input  : The input string the datetime shall be extracted
289///              from 
290///   * format : The format of the date string. Something like 
291///              %y%m%d_%H%M%S
292#[cfg_attr(feature="pybindings", pyfunction)]
293#[cfg_attr(feature="pybindings", pyo3(signature = (input , tformat = None )))]
294pub fn get_datetime(input : &str, tformat : Option<String>) -> Option<DateTime<Utc>> {
295  // this is the default format 
296  let mut date_time_format = String::from("%y%m%d_%H%M%S");
297  if let Some(tform) = tformat {
298    date_time_format = tform.to_string(); 
299  }
300  if let Ok(ndtime) = NaiveDateTime::parse_from_str(input, &date_time_format) {
301    //let dt_utc : DateTime<Utc> = DateTime::<Utc>::from_utc(ndtime, Utc); 
302    let dt_utc : DateTime<Utc> = DateTime::<Utc>::from_naive_utc_and_offset(ndtime, Utc); 
303    return Some(dt_utc);
304  } else { 
305    error!("Unable to parse {} for format {}! You can specify formats trhough the tformat keyword", input, date_time_format);
306    return None;
307  }
308}
309
310//--------------------------------------------------------------
311
312/// Retrieve the UNIX timestamp from a string as used in 
313/// the names of the run files
314///
315/// # Arguments:
316///
317///   * input  : The input string the datetime shall be extracted
318///              from 
319///   * format : The format of the date string. Something like 
320///              %y%m%d_%H%M%S
321#[cfg_attr(feature="pybindings", pyfunction)]
322#[cfg_attr(feature="pybindings", pyo3(signature = (input , tformat = None )))]
323pub fn get_unix_timestamp(input : &str, tformat : Option<String>) -> Option<u64> {
324  let dt = get_datetime(input, tformat);
325  if let Some(dt_) = dt {
326    // FIXME - we are only supporting times later than 
327    //         the UNIX epoch!
328    return Some(dt_.timestamp() as u64);
329  } else {
330    return None;
331  }
332}
333
334//--------------------------------------------------------------
335
336/// Identifier for different data sources
337#[derive(Debug, Copy, Clone, PartialEq,FromRepr, AsRefStr, EnumIter)]
338#[cfg_attr(feature = "pybindings", pyclass(eq, eq_int))]
339#[repr(u8)]
340pub enum DataSourceKind {
341  Unknown            = 0,
342  /// The "classic" written to the TOF-CPU on disk in flight
343  /// season 24/25 style
344  TofFiles           = 10,
345  /// As TofFiles, but sent over the network
346  TofStream          = 11,
347  /// The files as written on disk when received by a GSE 
348  /// system
349  TelemetryFiles     = 20,
350  /// Flight telemetry stream as sent out directly by the 
351  /// instrument
352  TelemetryStream    = 21,
353  /// Caraspace is a comprehensive, highly efficient data 
354  /// format which is used to combine Telemetry + TofStream 
355  /// data. Data written in this format as stored on disk.
356  CaraspaceFiles     = 30,
357  /// The same as above, however, represented as a network
358  /// stream
359  CaraspaceStream    = 31,
360  /// Philip's SimpleDet ROOT files
361  ROOTFiles          = 40,
362}
363
364expand_and_test_enum!(DataSourceKind, test_datasourcekind_repr);
365
366//--------------------------------------------------------------
367
368// in case we have pybindings for this type, 
369// expand it so that it can be used as keys
370// in dictionaries
371#[cfg(feature = "pybindings")]
372#[pymethods]
373impl DataSourceKind {
374
375  #[getter]
376  fn __hash__(&self) -> usize {
377    (*self as u8) as usize
378  } 
379}
380
381#[cfg(feature="pybindings")]
382pythonize_display!(DataSourceKind);
383
384//--------------------------------------------------------------
385
386/// Implement the Reader trait and necessary getters/setters to 
387/// make a struct an actual reader
388#[macro_export]
389macro_rules! reader {
390  ($struct_name:ident, $element_type:ident) => {
391 
392    use crate::io::DataReader; 
393    use crate::io::Serialization;
394
395    impl Iterator for $struct_name {
396      type Item = $element_type;
397      fn next(&mut self) -> Option<Self::Item> {
398        self.read_next()
399      }
400    }
401
402    impl DataReader<$element_type> for $struct_name {
403      fn get_header0(&self) -> u8 {
404        ($element_type::HEAD & 0x1) as u8 
405      }
406
407      fn get_header1(&self) -> u8 {
408        ($element_type::HEAD & 0x2) as u8
409      }
410
411      fn get_file_idx(&self) -> usize {
412        self.file_idx // Setting the specified field
413      }
414    
415      fn set_file_idx(&mut self, file_idx : usize) {
416        self.file_idx = file_idx;
417      }
418      
419      fn get_filenames(&self) -> &Vec<String> {
420          &self.filenames
421      }
422      
423      fn set_cursor(&mut self, pos : usize) {
424        self.cursor = pos;
425      }
426 
427      fn set_file_reader(&mut self, reader : BufReader<File>) {
428        self.file_reader = reader;
429      }
430    
431      fn read_next(&mut self) -> Option<$element_type> {
432        self.read_next_item()
433      }
434    
435      /// Get the next file ready
436      fn prime_next_file(&mut self) -> Option<usize> {
437        if self.file_idx == self.filenames.len() -1 {
438          return None;
439        } else {
440          self.file_idx += 1;
441          let nextfilename : &str = self.filenames[self.file_idx].as_str();
442          let nextfile     = OpenOptions::new().create(false).append(false).read(true).open(nextfilename).expect("Unable to open file {nextfilename}");
443          self.file_reader = BufReader::new(nextfile);
444          self.cursor      = 0;
445          return Some(self.file_idx);
446        }
447      }
448    }
449  }
450}
451
452/// Generics for packet reading (TofPacket, Telemetry packet,...)
453/// FIXME - not implemented yet
454pub trait DataReader<T> 
455  where T : Default + Serialization {
456  ///// header bytes, e.g. 0xAAAA for TofPackets, first byte
457  //const HEADER0 : u8 = 0;
458  ///// header bytes, e.g. 0xAAAA for TofPackets, second byte
459  //const HEADER1 : u8 = 0;
460
461  fn get_header0(&self) -> u8;
462  fn get_header1(&self) -> u8;
463
464  /// Return all filenames the reader is primed with   
465  fn get_filenames(&self) -> &Vec<String>;
466
467  /// The current index corresponding to the file the 
468  /// reader is currently working on
469  fn get_file_idx(&self) -> usize;
470
471  /// Set a new file idx corresponding to a file the reader 
472  /// is currently working on
473  fn set_file_idx(&mut self, idx : usize);
474
475  /// reset a new reader
476  fn set_file_reader(&mut self, freader : BufReader<File>);
477  
478  /// Get the next file ready
479  fn prime_next_file(&mut self) -> Option<usize>;
480
481  /// The name of the file the reader is currently 
482  /// working on
483  fn get_current_filename(&self) -> Option<&str> {
484    // should only happen when it is empty
485    if self.get_filenames().len() <= self.get_file_idx() {
486      return None;
487    }
488    Some(self.get_filenames()[self.get_file_idx()].as_str())
489  }
490
491  /// Manage the internal cursor attribute
492  fn set_cursor(&mut self, pos : usize);
493
494  /// Get the next frame/packet from the stream. Can be used to 
495  /// implement iterators
496  fn read_next(&mut self) -> Option<T>; 
497
498  /// Get the first entry in all of the files the reader is 
499  /// primed with
500  fn first(&mut self)     -> Option<T> {
501      match self.rewind() {
502      Err(err) => {
503        error!("Error when rewinding files! {err}");
504        return None;
505      }
506      Ok(_) => ()
507    }
508    let pack = self.read_next();
509    match self.rewind() {
510      Err(err) => {
511        error!("Error when rewinding files! {err}");
512      }
513      Ok(_) => ()
514    }
515    return pack;
516  }
517
518  /// Get the last entry in all of the files the reader is 
519  /// primed with
520  fn last(&mut self)      -> Option<T> {
521    self.set_file_idx(self.get_filenames().len() - 1);
522    let lastfilename = self.get_filenames()[self.get_file_idx()].as_str();
523    let lastfile     = OpenOptions::new().create(false).append(false).read(true).open(lastfilename).expect("Unable to open file {nextfilename}");
524    self.set_file_reader(BufReader::new(lastfile));
525    self.set_cursor(0);
526    let mut tp    = T::default();
527    let mut idx = 0;
528    loop {
529      match self.read_next() {
530        None => {
531          match self.rewind() {
532            Err(err) => {
533              error!("Error when rewinding files! {err}");
534            }
535            Ok(_) => ()
536          }
537          if idx == 0 {
538            return None;
539          } else {
540            return Some(tp);
541          }
542        }
543        Some(pack) => {
544          idx += 1;
545          tp = pack;
546          continue;
547        }
548      }
549    }
550  }
551
552  /// Rewind the current file and set the file index to the 
553  /// first file, so data can be read again from the 
554  /// beginning
555  fn rewind(&mut self) -> io::Result<()> {
556    let firstfile = &self.get_filenames()[0];
557    let file = OpenOptions::new().create(false).append(false).read(true).open(&firstfile)?;
558    self.set_file_reader(BufReader::new(file));
559    self.set_cursor(0);
560    self.set_file_idx(0);
561    Ok(())
562  }
563}
564
565//// blanket implementation: every `T` that implements Reader also implements Iterator
566//impl<T:std::default::Default + Serialization> Iterator for DataReader<T>  { 
567//  type Item = T;
568//  fn next(&mut self) -> Option<Self::Item> {
569//    self.read_next()
570//  }
571//}
572
gondola_core/io.rs

gondola_core/
io.rs
