tof_dataclasses/
serialization.rs

1//! Serialization/Deserialization helpers
2//!
3//!
4
5use half::f16;
6
7// re-exports
8pub use crate::errors::SerializationError;
9
10use std::error::Error;
11use std::path::Path;
12
13use std::collections::VecDeque;
14
15use crate::packets::{
16  TofPacket,
17  PacketType,
18};
19
20/// Convert a vector of u16 into a vector of u8
21///
22/// The resulting vector has twice the number
23/// of entries of the original vector.
24/// This is useful, when serializing data 
25/// represented as u16, e.g. the waveforms.
26pub fn u16_to_u8(vec_u16: &[u16]) -> Vec<u8> {
27    vec_u16.iter()
28        .flat_map(|&n| n.to_le_bytes().to_vec())
29        .collect()
30}
31
32/// Restore a vector of u16 from a vector of u8
33///
34/// This interpretes two following u8 as an u16
35/// Useful for deserialization of waveforms.
36pub fn u8_to_u16(vec_u8: &[u8]) -> Vec<u16> {
37    vec_u8.chunks_exact(2)
38        .map(|chunk| u16::from_le_bytes([chunk[0], chunk[1]]))
39        .collect()
40}
41
42/// Restore a vector of u16 from a vector of u8
43///
44/// This interpretes two following u8 as an u16
45/// Useful for deserialization of waveforms.
46/// Additionally it masks the first 2 bits 
47/// binary adding 0x3ff to each u16.
48pub fn u8_to_u16_14bit(vec_u8: &[u8]) -> Vec<u16> {
49    vec_u8.chunks_exact(2)
50        .map(|chunk| 0x3fff & u16::from_le_bytes([chunk[0], chunk[1]]))
51        .collect()
52}
53
54/// Restore a vector of u16 from a vector of u8, using the first 2 bits of each u16 
55/// to get channel/cell error bit information
56///
57/// This interpretes two following u8 as an u16
58/// Useful for deserialization of waveforms.
59/// Additioanlly, it preserves the error bits
60///
61/// # Arguments:
62///
63/// # Returns:
64///
65///   `Vec<u16>`, ch_sync_err, cell_sync_err : if one of the error bits is
66///                                            set, ch_sync_err or cell_sync_err
67///                                            will be set to true
68pub fn u8_to_u16_err_check(vec_u8: &[u8]) -> (Vec<u16>, bool, bool) {
69    let mut ch_sync_err   = true;
70    let mut cell_sync_err = true;
71    let vec_u16 = vec_u8.chunks_exact(2)
72        .map(|chunk| {
73          let value     =  u16::from_le_bytes([chunk[0], chunk[1]]);
74          ch_sync_err   = ch_sync_err   && (((0x8000 & value) >> 15) == 0x1); 
75          cell_sync_err = cell_sync_err && (((0x4000 & value) >> 14) == 0x1) ;
76          return 0x3fff & value;
77        })
78        .collect();
79    (vec_u16, ch_sync_err, cell_sync_err)
80}
81
82pub fn parse_u8(bs : &Vec::<u8>, pos : &mut usize) -> u8 {
83  let value = u8::from_le_bytes([bs[*pos]]);
84  *pos += 1;
85  value
86}
87
88pub fn parse_u8_deque(bs : &VecDeque::<u8>, pos : &mut usize) -> u8 {
89  let value = u8::from_le_bytes([bs[*pos]]);
90  *pos += 1;
91  value
92}
93
94
95/// Get u32 from a bytestream and move on the position marker
96///
97/// # Arguments 
98///
99/// * bs
100/// * pos 
101pub fn parse_u16(bs : &Vec::<u8>, pos : &mut usize) -> u16 {
102  let value = u16::from_le_bytes([bs[*pos], bs[*pos+1]]);
103  *pos += 2;
104  value
105}
106
107pub fn parse_u16_be(bs : &Vec::<u8>, pos : &mut usize) -> u16 {
108  let value = u16::from_be_bytes([bs[*pos], bs[*pos+1]]);
109  *pos += 2;
110  value
111}
112
113
114// FIXME - make this a generic
115pub fn parse_u16_deque(bs : &VecDeque::<u8>, pos : &mut usize) -> u16 {
116  let value = u16::from_le_bytes([bs[*pos], bs[*pos+1]]);
117  *pos += 2;
118  value
119}
120
121/// BIG Endian version of parse_u32. NOT for botched event id decoding!
122/// Used for network communications
123pub fn parse_u32_be(bs : &Vec::<u8>, pos : &mut usize) -> u32 {
124  let value = u32::from_be_bytes([bs[*pos], bs[*pos+1], bs[*pos+2], bs[*pos+3]]);
125  *pos += 4;
126  value
127}
128
129pub fn parse_u32(bs : &Vec::<u8>, pos : &mut usize) -> u32 {
130  let value = u32::from_le_bytes([bs[*pos], bs[*pos+1], bs[*pos+2], bs[*pos+3]]);
131  *pos += 4;
132  value
133}
134
135pub fn parse_u64(bs : &Vec::<u8>, pos : &mut usize) -> u64 {
136  let value = u64::from_le_bytes([bs[*pos],   bs[*pos+1], bs[*pos+2], bs[*pos+3],
137                                  bs[*pos+4], bs[*pos+5], bs[*pos+6], bs[*pos+7]]);
138  *pos += 8;
139  value
140}
141
142#[cfg(not(target_arch="arm"))]
143pub fn parse_usize(bs: &Vec::<u8>, pos: &mut usize) -> usize {
144  let value: usize = usize::from_le_bytes([bs[*pos],bs[*pos + 1], bs[*pos + 2], bs[*pos + 3], 
145    bs[*pos + 4], bs[*pos + 5], bs[*pos + 6], bs[*pos + 7],]);
146  *pos += std::mem::size_of::<usize>();
147  value
148}
149
150#[cfg(target_arch="arm")]
151pub fn parse_usize(bs: &Vec::<u8>, pos: &mut usize) -> usize {
152  parse_u32(bs, pos) as usize
153}
154
155/// Get an u32 from a bytestream 
156///
157/// This assumes an underlying representation of 
158/// an atomic unit of 16bit instead of 8.
159/// This is realized for the raw data stream
160/// from the readoutboards.
161pub fn parse_u32_for_16bit_words(bs  : &Vec::<u8>,
162                                 pos : &mut usize) -> u32 {
163  
164  let raw_bytes_4  = [bs[*pos + 2],
165                      bs[*pos + 3],
166                      bs[*pos    ],
167                      bs[*pos + 1]];
168  *pos += 4;
169  u32::from_le_bytes(raw_bytes_4)
170}
171
172/// Get an 48bit number from a bytestream 
173///
174/// This assumes an underlying representation of 
175/// an atomic unit of 16bit instead of 8.
176/// This is realized for the raw data stream
177/// from the readoutboards.
178pub fn parse_u48_for_16bit_words(bs  : &Vec::<u8>,
179                                 pos : &mut usize) -> u64 {
180  
181  let raw_bytes_8  = [0u8,
182                      0u8,
183                      bs[*pos + 4],
184                      bs[*pos + 5],
185                      bs[*pos + 2],
186                      bs[*pos + 3],
187                      bs[*pos    ],
188                      bs[*pos + 1]];
189  *pos += 6;
190  u64::from_le_bytes(raw_bytes_8)
191}
192
193//pub fn parse_f8(bs: &Vec<u8>, pos: &mut usize) -> f8 {
194//  let value = f8::from_le_bytes([bs[*pos]]);
195//  *pos += 1;
196//  value
197//}
198
199pub fn parse_f16(bs : &Vec::<u8>, pos : &mut usize) -> f16 {
200  let value = f16::from_le_bytes([bs[*pos], bs[*pos+1]]);
201  *pos += 2;
202  value
203}
204
205pub fn parse_f32(bs : &Vec::<u8>, pos : &mut usize) -> f32 {
206  let value = f32::from_le_bytes([bs[*pos],   bs[*pos+1],  
207                                  bs[*pos+2], bs[*pos+3]]);
208  *pos += 4;
209  value
210}
211
212pub fn parse_f64(bs : &Vec::<u8>, pos : &mut usize) -> f64 {
213  let value = f64::from_le_bytes([bs[*pos],   bs[*pos+1],  
214                                  bs[*pos+2], bs[*pos+3],
215                                  bs[*pos+4], bs[*pos+5],
216                                  bs[*pos+6], bs[*pos+7]]);
217  *pos += 8;
218  value
219}
220
221pub fn parse_bool(bs : &Vec::<u8>, pos : &mut usize) -> bool {
222  let value = u8::from_le_bytes([bs[*pos]]); 
223  *pos += 1;
224  value > 0
225}
226
227pub fn get_json_from_file(filename : &Path)
228    -> Result<String, Box<dyn Error>> {
229  let file_content = std::fs::read_to_string(filename)?;
230  let config = serde_json::from_str(&file_content)?;
231  Ok(config)
232}
233
234
235/// Can be wrapped within a TofPacket. To do, we just have
236/// to define a packet type
237pub trait Packable {
238  const PACKET_TYPE : PacketType;
239
240  /// Wrap myself in a TofPacket
241  fn pack(&self) -> TofPacket 
242    where Self: Serialization {
243    let mut tp     = TofPacket::new();
244    tp.payload     = self.to_bytestream();
245    tp.packet_type = Self::PACKET_TYPE;
246    tp
247  }
248}
249
250
251/// Encode/decode structs to `Vec::<u8>` to write to a file or
252/// send over the network
253///
254pub trait Serialization {
255
256  const HEAD: u16;
257  const TAIL: u16;
258  /// The SIZE is the size of the serialized 
259  /// bytestream INCLUDING 4 bytes for head
260  /// and tail bytes. In case the struct does 
261  /// NOT HAVE a fixed size, SIZE will be 0
262  /// (so default value of the trait
263  const SIZE: usize = 0;
264
265  /// Verify that the serialized representation of the struct has the 
266  /// correct size, including header + footer.
267  ///
268  /// Will panic for variable sized structs.
269  fn verify_fixed(stream : &Vec<u8>, 
270                  pos    : &mut usize) -> Result<(), SerializationError> {
271    if !Self::SIZE == 0 {
272      // we can panic here, since this is a conceptional logic error. If we
273      // don't panic, monsters will arise downstream.
274      panic!("Self::verify_fixed can be only used for structs with a fixed size! In case you are convinced, that your struct has indeed a fixed size, please implement trait Serialization::SIZE with the serialized size in bytes including 4 bytes for header and footer!");
275    }
276    let head_pos = search_for_u16(Self::HEAD, stream, *pos)?; 
277    let tail_pos = search_for_u16(Self::TAIL, stream, head_pos + Self::SIZE-2)?;
278    // At this state, this can be a header or a full event. Check here and
279    // proceed depending on the options
280    if tail_pos + 2 - head_pos != Self::SIZE {
281      error!("Seing {} bytes, but expecting {}", tail_pos + 2 - head_pos, Self::SIZE);
282      *pos = head_pos + 2; 
283      return Err(SerializationError::WrongByteSize);
284    }
285    *pos = head_pos + 2;
286    Ok(())
287  } 
288
289  /// Decode a serializable from a bytestream  
290  fn from_bytestream(bytestream : &Vec<u8>, 
291                     pos        : &mut usize)
292    -> Result<Self, SerializationError>
293    where Self : Sized;
294
295  /// Decode a serializable directly from a TofPacket
296  fn from_tofpacket(packet : &TofPacket)
297    -> Result<Self, SerializationError>
298    where Self: Sized {
299    let unpacked = Self::from_bytestream(&packet.payload, &mut 0)?;
300    Ok(unpacked)
301  }
302
303  /// Encode a serializable to a bytestream  
304  fn to_bytestream(&self) -> Vec<u8> {
305    println!("There can't be a default implementation for this trait!");
306    todo!();
307  }
308
309  fn from_slice(_slice     : &[u8],
310                _start_pos : usize)
311    -> Result<Self, SerializationError>
312    where Self : Sized {
313    println!("There can't be a default implementation for this trait!");
314    todo!();
315    }
316
317  /// Construct byte slice out of self.
318  ///
319  /// Can not fail.
320  fn to_slice(&self) 
321    -> &[u8]
322    where Self : Sized {
323    println!("There can't be a default implementation for this trait!");
324    todo!();
325  }
326}
327
328/// Search for a certain number of type `u16` in a bytestream
329pub fn search_for_u16(number : u16, bytestream : &Vec<u8>, start_pos : usize) 
330  -> Result<usize, SerializationError> {
331  // -2 bc later on we are looking for 2 bytes!
332  if bytestream.len() == 0 {
333    error!("Stream empty!");
334    return Err(SerializationError::StreamTooShort);
335  }
336  if start_pos  > bytestream.len() - 2 {
337    error!("Start position {} beyond stream capacity {}!", start_pos, bytestream.len() -2);
338    return Err(SerializationError::StreamTooShort);
339  }
340  let mut pos = start_pos;
341  let mut two_bytes : [u8;2]; 
342  // will find the next header
343  two_bytes = [bytestream[pos], bytestream[pos + 1]];
344  // FIXME - this should be little endian?
345  if u16::from_le_bytes(two_bytes) == number {
346    return Ok(pos);
347  }
348  // if it is not at start pos, then traverse 
349  // the stream
350  pos += 1;
351  let mut found = false;
352  // we search for the next packet
353  for n in pos..bytestream.len() - 1 {
354    two_bytes = [bytestream[n], bytestream[n + 1]];
355    if (u16::from_le_bytes(two_bytes)) == number {
356      pos = n;
357      found = true;
358      break;
359    }
360  }
361  if !found {
362    let delta = bytestream.len() - start_pos;
363    warn!("Can not find {} in bytestream [-{}:{}]!", number, delta ,bytestream.len());
364    return Err(SerializationError::ValueNotFound);
365  }
366  trace!("Found {number} at {pos}");
367  Ok(pos)
368}
369
370#[cfg(test)]
371mod test_serialization {
372  use crate::serialization::{search_for_u16,
373                             u16_to_u8};
374
375  #[test]
376  fn test_u16_to_u8_size_doubled() {
377    let size = 1000usize;
378    let data = vec![42u16;size];
379    let data_u8 = u16_to_u8(data.as_slice());
380    let data_u8_size = data_u8.len();
381    let double_size  = 2*size;
382    assert_eq!(data_u8_size, double_size);
383    
384  }
385
386  #[test]
387  fn test_search_for_2_bytemarker() {
388    // just test it two times - FIXME - use a better method
389    let mut bytestream = vec![1,2,3,0xAA, 0xAA, 5, 7];
390    let mut pos = search_for_u16(0xAAAA, &bytestream, 0).unwrap();
391    assert_eq!(pos, 3);
392    
393    bytestream = vec![1,2,3,244, 16, 32, 0xaa, 0xff, 5, 7];
394    pos = search_for_u16(65450, &bytestream, 1).unwrap();
395    assert_eq!(pos, 6);
396    
397    bytestream = vec![0xaa,0xaa,3,244, 16, 32, 0xAA, 0xFF, 5, 7];
398    pos = search_for_u16(0xaaaa, &bytestream, 0).unwrap();
399    assert_eq!(pos, 0);
400  }
401}
tof_dataclasses/serialization.rs

tof_dataclasses/
serialization.rs
