gondola_core/tof/
alerts.rs

1//! The alerts are just clues to trigger an individual to respond to a certain situation
2//!
3// This file is part of gaps-online-software and published 
4// under the GPLv3 license
5
6// kudos Grace!! (who else ^^ ) 
7//so these are the alerts relating to the tof:
8//-- RB rate = 0 (and if its two we'll know a RAT crashed)
9//-- RB temp (from RBMoni)  too cold < -40C
10//-- RB temp (from RBMoni) too warm > 80C
11//-- RAT labjack temp too cold < -40C
12//-- RAT labjack temp too warm > 80C
13//-- CAT labjack temp too cold < ?
14//-- CAT labjack temp too warm > ?
15//-- tofcpu temp too cold (from db?) < -40C
16//-- tofcpu temp too warm (from db?) > 95C
17//-- tofcpu temp too cold (CPUMoni) < -40C
18//-- tofcpu temp too warm (CPUMoni) > 95C
19//-- MTB FPGA temp too cold (MTBMoni) < -40C
20//-- MTB FPGA temp too warm (MTBMoni) > 80C
21//-- tofcpu cpu usage (from db?) > 90%
22//-- MTB rate (db) > depends
23//-- MTB rate (db) == 0
24//-- MTB lost rate (db) > MTB rate
25
26use std::fmt;
27use std::collections::HashMap;
28
29use std::time::Instant;
30use std::fs::File;
31use std::io::{
32  Read,
33  Write
34};
35
36use serde_json::json;
37
38//use crate::serialization::SerializationError;
39use crate::prelude::*;
40
41/// helper function to parse output for TofBot
42fn remove_from_word(s: String, word: &str) -> String {
43  if let Some(index) = s.find(word) {
44    // Keep everything up to the found index (not including the word itself)
45    s[..index].to_string()
46  } else {
47    // If the word isn't found, return the original string
48    s
49  }
50}
51
52/// How did whatever went wrong,
53/// go bad?
54#[derive(Debug, Copy, Clone, PartialEq, serde::Deserialize, serde::Serialize)]
55pub enum OutOfBound {
56  TooHigh,
57  TooLow,
58  TooLowOrTooHigh,
59  TooOld,
60  Zero,
61  Unknown,
62}
63
64impl fmt::Display for OutOfBound {
65  fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
66    let repr : &str;
67    match self {
68      OutOfBound::Unknown         => repr = "Unknown",
69      OutOfBound::TooHigh         => repr = "TooHigh",
70      OutOfBound::TooLowOrTooHigh => repr = "TooLowOrTooHigh",
71      OutOfBound::TooLow          => repr = "TooLow",
72      OutOfBound::TooOld          => repr = "TooOld",
73      OutOfBound::Zero            => repr = "Zero",
74    };
75    write!(f, "{}", repr)
76  }
77}
78
79#[derive(Debug, Copy, Clone, PartialEq, serde::Deserialize, serde::Serialize)]
80pub enum Shifters {
81  Unknown,
82  Grace,
83  Kazu,
84  Achim,
85  TofBot,
86}
87
88pub trait Pageable {
89  fn page(&self, content : String);
90
91  fn resolve(&self) {
92  }
93}
94
95impl fmt::Display for Shifters {
96  fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
97    let repr : &str;
98    match self {
99      Shifters::Unknown => repr = "Unknown",
100      Shifters::Grace   => repr = "Grace",
101      Shifters::Kazu    => repr = "Kazu",
102      Shifters::Achim   => repr = "Achim",
103      Shifters::TofBot  => repr = "TofBot",
104    };
105    write!(f, "{}", repr)
106  }
107}
108
109impl Pageable for Shifters {
110
111  fn page(&self, content : String) {
112    match self {
113      Shifters::Grace => {
114      }
115      Shifters::Kazu  => {
116      }
117      Shifters::Achim => {
118      }
119      Shifters::TofBot => {
120        // currently silence TofBot
121        // Achim's channel
122        //let url     = "https://hooks.slack.com/services/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";
123        // TofBot channel
124        //let url     = "https://hooks.slack.com/services/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";
125        let message = format!("\u{1F916}\u{1F680}\u{1F388} [LIFTOF (Bot)]\n {}",content);
126        let clean_message = remove_from_word(message, "tofbot_webhook");
127        let data = json!({
128          "text" : clean_message
129        });
130        match serde_json::to_string(&data) {
131          Err(err) => {
132            error!("Can not convert .json to string! {err}");
133          }
134          Ok(data_string) => {
135            warn!("Alert system disabled! Not paging TofBot with {}", data_string);
136        //    match ureq::post(url)
137        //        .set("Content-Type", "application/json")
138        //        .send_string(&data_string) {
139        //      Err(err) => { 
140        //        error!("Unable to send {} to TofBot! {err}", data_string);
141        //      }
142        //      Ok(response) => {
143        //        match response.into_string() {
144        //          Err(err) => {
145        //            error!("Not able to read response! {err}");
146        //          }
147        //          Ok(body) => {
148        //            info!("TofBot responded with {}", body);
149        //          }
150        //        }
151        //      }
152        //    }
153          }
154        }
155      }
156      _ => error!("Can't page unknown shifter!"),
157    }
158  }
159
160
161  fn resolve(&self) {
162    match self {
163      Shifters::Grace => {
164      }
165      Shifters::Kazu  => {
166      }
167      Shifters::Achim => {
168      }
169      Shifters::TofBot => {
170      }
171      _ => error!("Can't page unknown shifter!"),
172    }
173  }
174}
175
176/// How did whatever went wrong,
177/// go bad?
178#[derive(Debug, Copy, Clone, PartialEq, serde::Deserialize, serde::Serialize)]
179pub enum Variable {
180  Unknown,
181  TriggerRate,
182  Telemetry,
183  LostTriggerRate,
184  FPGATemp,
185  CoreTemp,
186  LabjackTemp,
187  AvailableDiskSpace,
188  DataMangling,
189  MoniData,
190}
191
192impl fmt::Display for Variable {
193  fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
194    let repr : &str;
195    match self {
196      Variable::Unknown            => repr = "Unknown",
197      Variable::TriggerRate        => repr = "TriggerRate",
198      Variable::Telemetry          => repr = "Telemetry",
199      Variable::LostTriggerRate    => repr = "LostTriggerRate",
200      Variable::CoreTemp           => repr = "CoreTemp",
201      Variable::FPGATemp           => repr = "FPGATemp",
202      Variable::LabjackTemp        => repr = "LabjackTemp",
203      Variable::AvailableDiskSpace => repr = "AvailableDiskSpace",
204      Variable::DataMangling       => repr = "DataMangling",
205      Variable::MoniData           => repr = "MoniData",
206    };
207    write!(f, "{}", repr)
208  }
209}
210
211/// How did whatever went wrong,
212/// go bad?
213#[derive(Debug, Copy, Clone, PartialEq, serde::Deserialize, serde::Serialize)]
214pub enum Component {
215  MTB,
216  CPU,
217  CAT,
218  System,
219  RAT(u8),
220  RB(u8),
221}
222
223impl fmt::Display for Component {
224  fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
225    let repr : String;
226    match self {
227      Component::MTB      => repr = String::from("MTB"),
228      Component::CPU      => repr = String::from("CPU"),
229      Component::CAT      => repr = String::from("CAT"),
230      Component::RAT(rid) => repr = format!("RAT{:02}", rid),
231      Component::RB(rid)  => repr = format!("RB{:02}", rid),
232      Component::System   => repr = String::from("System"),
233    };
234    write!(f, "{}", repr)
235  }
236}
237
238
239
240/// Alerts are clues to trigger a 
241/// reaction of an individual, so 
242/// they are designed for humans, 
243/// but should be issued by a 
244/// machine
245///
246/// ISSUES: The design choice of going with 
247/// a &str here and carry the livetime around
248/// is admittedly a bit questionable, especially
249/// since we might want to create similar 
250/// alerts automatically (which I did not think 
251/// of earlier)
252#[derive(Debug, Clone)]
253pub struct TofAlert<'a> {
254  /// a unique identifier so that we can 
255  /// look this up in settings
256  pub key          : &'a str,
257  /// description of what had happened
258  pub descr        : &'a str,
259  /// recommended course of action
260  pub whattodo     : Vec<&'a str>,
261  /// The configurable part of the alerm, 
262  /// subscirbers, bounds and is the alarm 
263  /// armed
264  pub config       : TofAlertConfig,
265  /// The variable which is an issue, e.g Rate
266  pub variable     : Variable,
267  /// The part of the TofSystem which caused the 
268  /// issue
269  pub component    : Component, 
270  /// How did the variable get out-of-bounds? 
271  /// E.g. too high, too low?
272  pub outofbound   : OutOfBound,
273  /// When did this issue occur?
274  pub triggered    : Option<Instant>,
275  /// How often did this alert page?
276  pub n_paged      : u32,
277}
278
279impl TofAlert<'_> {
280  
281  pub fn acknowledge(&mut self) {
282    self.triggered = None;
283  }
284
285  pub fn has_triggered(&self) -> bool {
286    self.triggered.is_some()
287  }
288
289  //pub fn annoy(&self) {
290  //  if self.triggered.is_some() && self.armed {
291  //    for person in &self.config.subscribers {
292  //      person.page();
293  //    }
294  //  }
295  //}
296  pub fn format_page(&self) -> String {
297    let mut page_text = format!("<< Alert {} triggered!\n", self.key);
298    page_text += &self.get_text();
299    page_text += ">>";
300    page_text
301  }
302
303  pub fn get_text(&self) -> String {
304    let mut repr = format!("< {} | {} | {}", self.component, self.variable, self.outofbound);
305    //repr    += &(format!(" -- descr : {}", self.descr));
306    //repr    += &(format!(" -- action: {}", self.whattodo));
307    //repr    += &(format!(" -- pages : "));
308    //for shifter in &self.subscribers {
309    //  repr += &(format!(" {} ", shifter)); 
310    //}
311    repr    += ">";
312    repr
313  }
314  //pub fn new() -> Self {
315  //  Self {
316  //    message      : String::from(""),
317  //    variable     :
318  //    outofbound   : OutOfBound::Unknown,
319  //    acknowledged : false
320  //  }
321  //}
322
323  // FIXME find a solution for the unwraps
324  pub fn trigger(&mut self, val : f32) {
325    let mut triggered = false;
326    match self.outofbound {
327      OutOfBound::TooHigh => {
328        match self.config.max_allowed {
329          None => {
330            error!("Set condition for {} as 'TooHigh', but 'max_allowed' is not specified", self.key);
331          }
332          Some(max_allowed) => {
333            if val > max_allowed {
334              warn!("Alarm {} triggered at {}!", self.key, max_allowed);
335              triggered = true;
336            }
337          }
338        }
339      }
340      OutOfBound::TooLow  => {
341        match self.config.min_allowed {
342          None => {
343            error!("Set condition for {} as 'TooLow', but 'min_allowed' is not specified", self.key);
344          }
345          Some(min_allowed) => {
346            if val < min_allowed {
347              warn!("Alarm {} triggered at {}!", self.key, min_allowed);
348              triggered = true;
349            }
350          }
351        }
352      }
353      OutOfBound::TooLowOrTooHigh => {
354        if val < self.config.min_allowed.unwrap() 
355        || val > self.config.max_allowed.unwrap() {
356          //warn!("Alarm {} triggered at {}!", self.key, max_allowed);
357          triggered = true;
358        }
359      }
360      OutOfBound::TooOld  => {
361        match self.config.max_allowed {
362          None => {
363            error!("Set condition for {} as 'TooOld', but 'max_allowed' is not specified", self.key);
364          }
365          Some(max_allowed) => {
366            if val > max_allowed {
367              warn!("Alarm {} triggered at {}!", self.key, max_allowed);
368              triggered = true;
369            }
370          }
371        }
372      }
373      OutOfBound::Zero    => {
374        if val == 0.0 {
375          triggered = true;
376        }
377      }
378      OutOfBound::Unknown => (),  
379    }
380    if triggered {
381      self.triggered = Some(Instant::now());
382    }
383  }
384 
385  pub fn page(&mut self) {
386    let content = self.format_page();
387    self.n_paged += 1;
388    for person in &self.config.subscribers {
389      match person {
390        Shifters::TofBot => {
391          // always page TofBot
392          person.page(content.clone());
393        }
394        _ => {
395          if self.config.armed {
396            // person who might need sleep, only 
397            // page them when the alert is armed
398            person.page(content.clone());
399          }
400        }
401      }
402    }
403  }
404
405
406  pub fn configure_from_manifest(&mut self, manifest : &TofAlertManifest)  {
407    match manifest.get(self.key) {
408      Some(cfg) => {
409        self.config = cfg;
410      }
411      None => {
412        error!("No entry for {} found in the alert manifest!", self.key);
413      }
414    }
415  }
416}
417
418impl PartialEq for TofAlert<'_> {
419  fn eq(&self, other: &Self) -> bool {
420    self.variable      == other.variable
421    && self.component  == other.component
422    && self.outofbound == other.outofbound
423  }
424}
425
426impl fmt::Display for TofAlert<'_> {
427  fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
428    let repr = self.get_text();
429    write!(f, "{}", repr)
430  }
431}
432
433///// Create the temperature and rate alerts for a RB
434//pub fn alert_factory_rb<'a>(rb_id      : u8) -> (TofAlert<'a>, TofAlert<'a>) {
435//  let rate_alert = TofAlert {
436//    key          : &(format!("RB{:02}_rate_zero", rb_id)),
437//    descr        : &(format!("RB{:02} is not triggering!", rb_id)),
438//    whattodo     : vec!["1) Run restart", "2) Restart liftof-rb", "3) Soft reboot RB", "4) Soft reboot MTB", "5) Hard reboot RAT",  "6) Hard reboot MTB"],
439//    config       : TofAlertConfig::new(),
440//    variable     : Variable::TriggerRate,
441//    outofbound   : OutOfBound::Zero,
442//    component    : Component::RB(rb_id),
443//    triggered    : None,
444//  };
445//  
446//  let temp_alert = TofAlert {
447//    key          : format!("RB{:02}_temp", rb_id).as_str(),
448//    descr        : format!("RB{:02} (FPGA) temperature is out of bounds!", rb_id).as_str(),
449//    whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
450//    config       : TofAlertConfig::new(),
451//    variable     : Variable::TriggerRate,
452//    outofbound   : OutOfBound::Zero,
453//    component    : Component::RB(rb_id),
454//    triggered    : None,
455//  };
456//  (rate_alert, temp_alert)
457//}
458
459
460
461//}
462//
463///// Create an alert for mtb temperature out of range
464//pub fn alert_factory_rb_temp<'a>(rb_id       : u8,
465//                                 subscribers : Vec<Shifters>,
466//                                 outofbound  : OutOfBound) -> TofAlert<'a> {
467//  TofAlert {
468//    key          : "rb_temp",
469//    descr        : "RB (FPGA) Temperature out of bounds!!",
470//    whattodo     : vec!["This is critical! Check with everyone and then shut off this RB!"],
471//    config       : TofAlertConfig::new(),
472//    variable     : Variable::FPGATemp,
473//    outofbound   : outofbound,
474//    component    : Component::RB(rb_id),
475//    acknowledged : false,
476//    created      : Instant::now(),
477//  }
478//}
479//
480///// Create an alert for the case where the MTB is not triggering
481//pub fn alert_factory_mtb_rate_zero<'a>(subscribers : Vec<Shifters>) -> TofAlert<'a> {
482//  TofAlert {
483//    key          : "mtb_rate_zero",
484//    descr        : "MTB is not triggering!",
485//    whattodo     : vec!["1) Check about TIU status", "2) Run restart", "3) If SSH available, debug with pybindings", "4) Soft reboot MTB", "5) Hard reboot MTB"],
486//    config       : TofAlertConfig::new(),
487//    variable     : Variable::TriggerRate,
488//    outofbound   : OutOfBound::Zero,
489//    component    : Component::MTB,
490//    acknowledged : false,
491//    created      : Instant::now(),
492//  }
493//}
494//
495///// Create an alert for mtb temperature out of range
496//pub fn alert_factory_mtb_temp<'a>(subscribers : Vec<Shifters>,
497//                                  outofbound  : OutOfBound) -> TofAlert<'a> {
498//  TofAlert {
499//    key          : "mtb_fpga_temp",
500//    descr        : "MTB Temperature out of bounds!!",
501//    whattodo     : vec!["This is critical! Check with everyone and then shut off the MTB!"],
502//    config       : TofAlertConfig::new(),
503//    variable     : Variable::FPGATemp,
504//    outofbound   : outofbound,
505//    component    : Component::MTB,
506//    acknowledged : false,
507//    created      : Instant::now(),
508//  }
509//}
510//
511///// Create an alert for mtb temperature out of range
512//pub fn alert_factory_mtb_lost_rate<'a>(subscribers : Vec<Shifters>,
513//                                       outofbound  : OutOfBound) -> TofAlert<'a> {
514//  TofAlert {
515//    key          : "mtb_lost_rate",
516//    descr        : "MTB Lost rate is whacky!!",
517//    whattodo     : vec!["Unclear course of action. Check with tracker, maybe nothing can be done!"],
518//    config       : TofAlertConfig::new(),
519//    variable     : Variable::LostTriggerRate,
520//    outofbound   : outofbound,
521//    component    : Component::MTB,
522//    acknowledged : false,
523//    created      : Instant::now(),
524//  }
525//}
526
527/// Configure alerts from a .toml file
528#[derive(Debug, Clone, PartialEq, serde::Deserialize, serde::Serialize)]
529pub struct TofAlertConfig {
530  pub subscribers : Vec<Shifters>,
531  pub armed       : bool,
532  pub min_allowed : Option<f32>,
533  pub max_allowed : Option<f32>,
534  pub non_zero    : Option<bool>,
535}
536
537impl TofAlertConfig {
538  pub fn new() -> Self {
539    Self {
540      subscribers : vec![Shifters::Grace,
541                         Shifters::Kazu,
542                         Shifters::Achim,
543                         Shifters::TofBot],
544      armed       : false,
545      min_allowed : Some(-40.0),
546      max_allowed : Some(80.0),
547      non_zero    : Some(false),
548    }
549  }
550}
551
552
553
554/// Describes a .toml file with alert. 
555/// settings
556#[derive(Debug, Clone, PartialEq, serde::Deserialize, serde::Serialize)]
557pub struct TofAlertManifest {
558  pub data_mangling   : TofAlertConfig,
559  pub miss_tofevid    : TofAlertConfig,
560  pub tofev_vs_merge  : TofAlertConfig,
561  pub notof_vs_merge  : TofAlertConfig,
562  pub frac_interest   : TofAlertConfig,
563  pub mtb_lost_rate   : TofAlertConfig,
564  pub mtb_fpga_temp   : TofAlertConfig,
565  pub mtb_rate_zero   : TofAlertConfig,
566  pub mtb_hk_too_old  : TofAlertConfig,
567  pub cpu_core0_temp  : TofAlertConfig,
568  pub cpu_core1_temp  : TofAlertConfig,
569  pub cpu_hk_too_old  : TofAlertConfig,
570  pub cpu_disk        : TofAlertConfig,
571  pub rb01_temp       : TofAlertConfig,
572  pub rb02_temp       : TofAlertConfig,  
573  pub rb03_temp       : TofAlertConfig,  
574  pub rb04_temp       : TofAlertConfig, 
575  pub rb05_temp       : TofAlertConfig, 
576  pub rb06_temp       : TofAlertConfig, 
577  pub rb07_temp       : TofAlertConfig,
578  pub rb08_temp       : TofAlertConfig, 
579  pub rb09_temp       : TofAlertConfig, 
580  pub rb11_temp       : TofAlertConfig, 
581  pub rb13_temp       : TofAlertConfig,
582  pub rb14_temp       : TofAlertConfig, 
583  pub rb15_temp       : TofAlertConfig, 
584  pub rb16_temp       : TofAlertConfig, 
585  pub rb17_temp       : TofAlertConfig, 
586  pub rb18_temp       : TofAlertConfig, 
587  pub rb19_temp       : TofAlertConfig, 
588  pub rb20_temp       : TofAlertConfig, 
589  pub rb21_temp       : TofAlertConfig, 
590  pub rb22_temp       : TofAlertConfig, 
591  pub rb23_temp       : TofAlertConfig, 
592  pub rb24_temp       : TofAlertConfig, 
593  pub rb25_temp       : TofAlertConfig, 
594  pub rb26_temp       : TofAlertConfig, 
595  pub rb27_temp       : TofAlertConfig, 
596  pub rb28_temp       : TofAlertConfig, 
597  pub rb29_temp       : TofAlertConfig, 
598  pub rb30_temp       : TofAlertConfig, 
599  pub rb31_temp       : TofAlertConfig, 
600  pub rb32_temp       : TofAlertConfig, 
601  pub rb33_temp       : TofAlertConfig, 
602  pub rb34_temp       : TofAlertConfig, 
603  pub rb35_temp       : TofAlertConfig, 
604  pub rb36_temp       : TofAlertConfig, 
605  pub rb39_temp       : TofAlertConfig, 
606  pub rb40_temp       : TofAlertConfig, 
607  pub rb41_temp       : TofAlertConfig, 
608  pub rb42_temp       : TofAlertConfig, 
609  pub rb44_temp       : TofAlertConfig, 
610  pub rb46_temp       : TofAlertConfig, 
611  pub rb01_rate_zero  : TofAlertConfig, 
612  pub rb02_rate_zero  : TofAlertConfig,  
613  pub rb03_rate_zero  : TofAlertConfig,  
614  pub rb04_rate_zero  : TofAlertConfig, 
615  pub rb05_rate_zero  : TofAlertConfig, 
616  pub rb06_rate_zero  : TofAlertConfig, 
617  pub rb07_rate_zero  : TofAlertConfig, 
618  pub rb08_rate_zero  : TofAlertConfig, 
619  pub rb09_rate_zero  : TofAlertConfig, 
620  pub rb11_rate_zero  : TofAlertConfig, 
621  pub rb13_rate_zero  : TofAlertConfig, 
622  pub rb14_rate_zero  : TofAlertConfig, 
623  pub rb15_rate_zero  : TofAlertConfig, 
624  pub rb16_rate_zero  : TofAlertConfig, 
625  pub rb17_rate_zero  : TofAlertConfig, 
626  pub rb18_rate_zero  : TofAlertConfig, 
627  pub rb19_rate_zero  : TofAlertConfig, 
628  pub rb20_rate_zero  : TofAlertConfig, 
629  pub rb21_rate_zero  : TofAlertConfig, 
630  pub rb22_rate_zero  : TofAlertConfig, 
631  pub rb23_rate_zero  : TofAlertConfig, 
632  pub rb24_rate_zero  : TofAlertConfig, 
633  pub rb25_rate_zero  : TofAlertConfig, 
634  pub rb26_rate_zero  : TofAlertConfig, 
635  pub rb27_rate_zero  : TofAlertConfig, 
636  pub rb28_rate_zero  : TofAlertConfig, 
637  pub rb29_rate_zero  : TofAlertConfig, 
638  pub rb30_rate_zero  : TofAlertConfig, 
639  pub rb31_rate_zero  : TofAlertConfig, 
640  pub rb32_rate_zero  : TofAlertConfig, 
641  pub rb33_rate_zero  : TofAlertConfig, 
642  pub rb34_rate_zero  : TofAlertConfig, 
643  pub rb35_rate_zero  : TofAlertConfig, 
644  pub rb36_rate_zero  : TofAlertConfig, 
645  pub rb39_rate_zero  : TofAlertConfig, 
646  pub rb40_rate_zero  : TofAlertConfig, 
647  pub rb41_rate_zero  : TofAlertConfig, 
648  pub rb42_rate_zero  : TofAlertConfig, 
649  pub rb44_rate_zero  : TofAlertConfig, 
650  pub rb46_rate_zero  : TofAlertConfig, 
651  pub rb01_hk_too_old : TofAlertConfig, 
652  pub rb02_hk_too_old : TofAlertConfig,  
653  pub rb03_hk_too_old : TofAlertConfig,  
654  pub rb04_hk_too_old : TofAlertConfig, 
655  pub rb05_hk_too_old : TofAlertConfig, 
656  pub rb06_hk_too_old : TofAlertConfig, 
657  pub rb07_hk_too_old : TofAlertConfig, 
658  pub rb08_hk_too_old : TofAlertConfig, 
659  pub rb09_hk_too_old : TofAlertConfig, 
660  pub rb11_hk_too_old : TofAlertConfig, 
661  pub rb13_hk_too_old : TofAlertConfig, 
662  pub rb14_hk_too_old : TofAlertConfig, 
663  pub rb15_hk_too_old : TofAlertConfig, 
664  pub rb16_hk_too_old : TofAlertConfig, 
665  pub rb17_hk_too_old : TofAlertConfig, 
666  pub rb18_hk_too_old : TofAlertConfig, 
667  pub rb19_hk_too_old : TofAlertConfig, 
668  pub rb20_hk_too_old : TofAlertConfig, 
669  pub rb21_hk_too_old : TofAlertConfig, 
670  pub rb22_hk_too_old : TofAlertConfig, 
671  pub rb23_hk_too_old : TofAlertConfig, 
672  pub rb24_hk_too_old : TofAlertConfig, 
673  pub rb25_hk_too_old : TofAlertConfig, 
674  pub rb26_hk_too_old : TofAlertConfig, 
675  pub rb27_hk_too_old : TofAlertConfig, 
676  pub rb28_hk_too_old : TofAlertConfig, 
677  pub rb29_hk_too_old : TofAlertConfig, 
678  pub rb30_hk_too_old : TofAlertConfig, 
679  pub rb31_hk_too_old : TofAlertConfig, 
680  pub rb32_hk_too_old : TofAlertConfig, 
681  pub rb33_hk_too_old : TofAlertConfig, 
682  pub rb34_hk_too_old : TofAlertConfig, 
683  pub rb35_hk_too_old : TofAlertConfig, 
684  pub rb36_hk_too_old : TofAlertConfig, 
685  pub rb39_hk_too_old : TofAlertConfig, 
686  pub rb40_hk_too_old : TofAlertConfig, 
687  pub rb41_hk_too_old : TofAlertConfig, 
688  pub rb42_hk_too_old : TofAlertConfig, 
689  pub rb44_hk_too_old : TofAlertConfig, 
690  pub rb46_hk_too_old : TofAlertConfig, 
691}
692
693impl TofAlertManifest {
694  pub fn new() -> Self {
695    Self {
696      data_mangling   : TofAlertConfig::new(),
697      miss_tofevid    : TofAlertConfig::new(),
698      tofev_vs_merge  : TofAlertConfig::new(),
699      notof_vs_merge  : TofAlertConfig::new(),
700      frac_interest   : TofAlertConfig::new(),
701      mtb_lost_rate   : TofAlertConfig::new(),
702      mtb_fpga_temp   : TofAlertConfig::new(),
703      mtb_rate_zero   : TofAlertConfig::new(),
704      mtb_hk_too_old  : TofAlertConfig::new(),
705      cpu_core0_temp  : TofAlertConfig::new(),
706      cpu_core1_temp  : TofAlertConfig::new(),
707      cpu_hk_too_old  : TofAlertConfig::new(),
708      cpu_disk        : TofAlertConfig::new(),
709      rb01_temp       : TofAlertConfig::new(),
710      rb02_temp       : TofAlertConfig::new(),  
711      rb03_temp       : TofAlertConfig::new(),  
712      rb04_temp       : TofAlertConfig::new(), 
713      rb05_temp       : TofAlertConfig::new(), 
714      rb06_temp       : TofAlertConfig::new(), 
715      rb07_temp       : TofAlertConfig::new(),
716      rb08_temp       : TofAlertConfig::new(), 
717      rb09_temp       : TofAlertConfig::new(), 
718      rb11_temp       : TofAlertConfig::new(), 
719      rb13_temp       : TofAlertConfig::new(),
720      rb14_temp       : TofAlertConfig::new(), 
721      rb15_temp       : TofAlertConfig::new(), 
722      rb16_temp       : TofAlertConfig::new(), 
723      rb17_temp       : TofAlertConfig::new(), 
724      rb18_temp       : TofAlertConfig::new(), 
725      rb19_temp       : TofAlertConfig::new(), 
726      rb20_temp       : TofAlertConfig::new(), 
727      rb21_temp       : TofAlertConfig::new(), 
728      rb22_temp       : TofAlertConfig::new(), 
729      rb23_temp       : TofAlertConfig::new(), 
730      rb24_temp       : TofAlertConfig::new(), 
731      rb25_temp       : TofAlertConfig::new(), 
732      rb26_temp       : TofAlertConfig::new(), 
733      rb27_temp       : TofAlertConfig::new(), 
734      rb28_temp       : TofAlertConfig::new(), 
735      rb29_temp       : TofAlertConfig::new(), 
736      rb30_temp       : TofAlertConfig::new(), 
737      rb31_temp       : TofAlertConfig::new(), 
738      rb32_temp       : TofAlertConfig::new(), 
739      rb33_temp       : TofAlertConfig::new(), 
740      rb34_temp       : TofAlertConfig::new(), 
741      rb35_temp       : TofAlertConfig::new(), 
742      rb36_temp       : TofAlertConfig::new(), 
743      rb39_temp       : TofAlertConfig::new(), 
744      rb40_temp       : TofAlertConfig::new(), 
745      rb41_temp       : TofAlertConfig::new(), 
746      rb42_temp       : TofAlertConfig::new(), 
747      rb44_temp       : TofAlertConfig::new(), 
748      rb46_temp       : TofAlertConfig::new(), 
749      rb01_rate_zero  : TofAlertConfig::new(), 
750      rb02_rate_zero  : TofAlertConfig::new(),  
751      rb03_rate_zero  : TofAlertConfig::new(),  
752      rb04_rate_zero  : TofAlertConfig::new(), 
753      rb05_rate_zero  : TofAlertConfig::new(), 
754      rb06_rate_zero  : TofAlertConfig::new(), 
755      rb07_rate_zero  : TofAlertConfig::new(), 
756      rb08_rate_zero  : TofAlertConfig::new(), 
757      rb09_rate_zero  : TofAlertConfig::new(), 
758      rb11_rate_zero  : TofAlertConfig::new(), 
759      rb13_rate_zero  : TofAlertConfig::new(), 
760      rb14_rate_zero  : TofAlertConfig::new(), 
761      rb15_rate_zero  : TofAlertConfig::new(), 
762      rb16_rate_zero  : TofAlertConfig::new(), 
763      rb17_rate_zero  : TofAlertConfig::new(), 
764      rb18_rate_zero  : TofAlertConfig::new(), 
765      rb19_rate_zero  : TofAlertConfig::new(), 
766      rb20_rate_zero  : TofAlertConfig::new(), 
767      rb21_rate_zero  : TofAlertConfig::new(), 
768      rb22_rate_zero  : TofAlertConfig::new(), 
769      rb23_rate_zero  : TofAlertConfig::new(), 
770      rb24_rate_zero  : TofAlertConfig::new(), 
771      rb25_rate_zero  : TofAlertConfig::new(), 
772      rb26_rate_zero  : TofAlertConfig::new(), 
773      rb27_rate_zero  : TofAlertConfig::new(), 
774      rb28_rate_zero  : TofAlertConfig::new(), 
775      rb29_rate_zero  : TofAlertConfig::new(), 
776      rb30_rate_zero  : TofAlertConfig::new(), 
777      rb31_rate_zero  : TofAlertConfig::new(), 
778      rb32_rate_zero  : TofAlertConfig::new(), 
779      rb33_rate_zero  : TofAlertConfig::new(), 
780      rb34_rate_zero  : TofAlertConfig::new(), 
781      rb35_rate_zero  : TofAlertConfig::new(), 
782      rb36_rate_zero  : TofAlertConfig::new(), 
783      rb39_rate_zero  : TofAlertConfig::new(), 
784      rb40_rate_zero  : TofAlertConfig::new(), 
785      rb41_rate_zero  : TofAlertConfig::new(), 
786      rb42_rate_zero  : TofAlertConfig::new(), 
787      rb44_rate_zero  : TofAlertConfig::new(), 
788      rb46_rate_zero  : TofAlertConfig::new(), 
789      rb01_hk_too_old : TofAlertConfig::new(), 
790      rb02_hk_too_old : TofAlertConfig::new(),  
791      rb03_hk_too_old : TofAlertConfig::new(),  
792      rb04_hk_too_old : TofAlertConfig::new(), 
793      rb05_hk_too_old : TofAlertConfig::new(), 
794      rb06_hk_too_old : TofAlertConfig::new(), 
795      rb07_hk_too_old : TofAlertConfig::new(), 
796      rb08_hk_too_old : TofAlertConfig::new(), 
797      rb09_hk_too_old : TofAlertConfig::new(), 
798      rb11_hk_too_old : TofAlertConfig::new(), 
799      rb13_hk_too_old : TofAlertConfig::new(), 
800      rb14_hk_too_old : TofAlertConfig::new(), 
801      rb15_hk_too_old : TofAlertConfig::new(), 
802      rb16_hk_too_old : TofAlertConfig::new(), 
803      rb17_hk_too_old : TofAlertConfig::new(), 
804      rb18_hk_too_old : TofAlertConfig::new(), 
805      rb19_hk_too_old : TofAlertConfig::new(), 
806      rb20_hk_too_old : TofAlertConfig::new(), 
807      rb21_hk_too_old : TofAlertConfig::new(), 
808      rb22_hk_too_old : TofAlertConfig::new(), 
809      rb23_hk_too_old : TofAlertConfig::new(), 
810      rb24_hk_too_old : TofAlertConfig::new(), 
811      rb25_hk_too_old : TofAlertConfig::new(), 
812      rb26_hk_too_old : TofAlertConfig::new(), 
813      rb27_hk_too_old : TofAlertConfig::new(), 
814      rb28_hk_too_old : TofAlertConfig::new(), 
815      rb29_hk_too_old : TofAlertConfig::new(), 
816      rb30_hk_too_old : TofAlertConfig::new(), 
817      rb31_hk_too_old : TofAlertConfig::new(), 
818      rb32_hk_too_old : TofAlertConfig::new(), 
819      rb33_hk_too_old : TofAlertConfig::new(), 
820      rb34_hk_too_old : TofAlertConfig::new(), 
821      rb35_hk_too_old : TofAlertConfig::new(), 
822      rb36_hk_too_old : TofAlertConfig::new(), 
823      rb39_hk_too_old : TofAlertConfig::new(), 
824      rb40_hk_too_old : TofAlertConfig::new(), 
825      rb41_hk_too_old : TofAlertConfig::new(), 
826      rb42_hk_too_old : TofAlertConfig::new(), 
827      rb44_hk_too_old : TofAlertConfig::new(), 
828      rb46_hk_too_old : TofAlertConfig::new(), 
829    }
830  }
831
832  pub fn keys(&self) -> Vec<&'static str> {
833    let keys = vec!["data_mangling",  
834                    "miss_tofevid",   
835                    "tofev_vs_merge", 
836                    "notof_vs_merge", 
837                    "frac_interest",  
838                    "mtb_lost_rate",
839                    "mtb_fpga_temp",
840                    "mtb_rate_zero",
841                    "mtb_hk_too_old",
842                    "cpu_core0_temp",
843                    "cpu_core1_temp",
844                    "cpu_hk_too_old",
845                    "cpu_disk",
846                    "rb01_temp",
847                    "rb02_temp",  
848                    "rb03_temp",  
849                    "rb04_temp", 
850                    "rb05_temp", 
851                    "rb06_temp", 
852                    "rb07_temp",
853                    "rb08_temp", 
854                    "rb09_temp", 
855                    "rb11_temp", 
856                    "rb13_temp",
857                    "rb14_temp", 
858                    "rb15_temp", 
859                    "rb16_temp", 
860                    "rb17_temp", 
861                    "rb18_temp", 
862                    "rb19_temp", 
863                    "rb20_temp", 
864                    "rb21_temp", 
865                    "rb22_temp", 
866                    "rb23_temp", 
867                    "rb24_temp", 
868                    "rb25_temp", 
869                    "rb26_temp", 
870                    "rb27_temp", 
871                    "rb28_temp", 
872                    "rb29_temp", 
873                    "rb30_temp", 
874                    "rb31_temp", 
875                    "rb32_temp", 
876                    "rb33_temp", 
877                    "rb34_temp", 
878                    "rb35_temp", 
879                    "rb36_temp", 
880                    "rb39_temp", 
881                    "rb40_temp", 
882                    "rb41_temp", 
883                    "rb42_temp", 
884                    "rb44_temp", 
885                    "rb46_temp", 
886                    "rb01_rate_zero", 
887                    "rb02_rate_zero",  
888                    "rb03_rate_zero",  
889                    "rb04_rate_zero", 
890                    "rb05_rate_zero", 
891                    "rb06_rate_zero", 
892                    "rb07_rate_zero", 
893                    "rb08_rate_zero", 
894                    "rb09_rate_zero", 
895                    "rb11_rate_zero", 
896                    "rb13_rate_zero", 
897                    "rb14_rate_zero", 
898                    "rb15_rate_zero", 
899                    "rb16_rate_zero", 
900                    "rb17_rate_zero", 
901                    "rb18_rate_zero", 
902                    "rb19_rate_zero", 
903                    "rb20_rate_zero", 
904                    "rb21_rate_zero", 
905                    "rb22_rate_zero", 
906                    "rb23_rate_zero", 
907                    "rb24_rate_zero", 
908                    "rb25_rate_zero", 
909                    "rb26_rate_zero", 
910                    "rb27_rate_zero", 
911                    "rb28_rate_zero", 
912                    "rb29_rate_zero", 
913                    "rb30_rate_zero", 
914                    "rb31_rate_zero", 
915                    "rb32_rate_zero", 
916                    "rb33_rate_zero", 
917                    "rb34_rate_zero", 
918                    "rb35_rate_zero", 
919                    "rb36_rate_zero", 
920                    "rb39_rate_zero", 
921                    "rb40_rate_zero", 
922                    "rb41_rate_zero", 
923                    "rb42_rate_zero", 
924                    "rb44_rate_zero", 
925                    "rb46_rate_zero", 
926                    "rb01_hk_too_old", 
927                    "rb02_hk_too_old",  
928                    "rb03_hk_too_old",  
929                    "rb04_hk_too_old", 
930                    "rb05_hk_too_old", 
931                    "rb06_hk_too_old", 
932                    "rb07_hk_too_old", 
933                    "rb08_hk_too_old", 
934                    "rb09_hk_too_old", 
935                    "rb11_hk_too_old", 
936                    "rb13_hk_too_old", 
937                    "rb14_hk_too_old", 
938                    "rb15_hk_too_old", 
939                    "rb16_hk_too_old", 
940                    "rb17_hk_too_old", 
941                    "rb18_hk_too_old", 
942                    "rb19_hk_too_old", 
943                    "rb20_hk_too_old", 
944                    "rb21_hk_too_old", 
945                    "rb22_hk_too_old", 
946                    "rb23_hk_too_old", 
947                    "rb24_hk_too_old", 
948                    "rb25_hk_too_old", 
949                    "rb26_hk_too_old", 
950                    "rb27_hk_too_old", 
951                    "rb28_hk_too_old", 
952                    "rb29_hk_too_old", 
953                    "rb30_hk_too_old", 
954                    "rb31_hk_too_old", 
955                    "rb32_hk_too_old", 
956                    "rb33_hk_too_old", 
957                    "rb34_hk_too_old", 
958                    "rb35_hk_too_old", 
959                    "rb36_hk_too_old", 
960                    "rb39_hk_too_old", 
961                    "rb40_hk_too_old", 
962                    "rb41_hk_too_old", 
963                    "rb42_hk_too_old", 
964                    "rb44_hk_too_old", 
965                    "rb46_hk_too_old", 
966                    ];
967    return keys;
968  }
969
970  pub fn get(&self, key :  &str) -> Option<TofAlertConfig> {
971    match key {
972      "data_mangling"   => Some(self.data_mangling.clone()),  
973      "miss_tofevid"    => Some(self.miss_tofevid.clone()),   
974      "tofev_vs_merge"  => Some(self.tofev_vs_merge.clone()), 
975      "notof_vs_merge"  => Some(self.notof_vs_merge.clone()), 
976      "frac_interest"   => Some(self.frac_interest.clone()),  
977      "mtb_lost_rate"   => Some(self.mtb_lost_rate.clone()),
978      "mtb_fpga_temp"   => Some(self.mtb_fpga_temp.clone()),
979      "mtb_rate_zero"   => Some(self.mtb_rate_zero.clone()),
980      "mtb_hk_too_old"  => Some(self.mtb_hk_too_old.clone()),
981      "cpu_core0_temp"  => Some(self.cpu_core0_temp.clone()),
982      "cpu_core1_temp"  => Some(self.cpu_core1_temp.clone()),
983      "cpu_hk_too_old"  => Some(self.cpu_hk_too_old.clone()),
984      "cpu_disk"        => Some(self.cpu_disk.clone()),
985      "rb01_temp"       => Some(self.rb01_temp.clone()),
986      "rb02_temp"       => Some(self.rb02_temp.clone()),  
987      "rb03_temp"       => Some(self.rb03_temp.clone()),  
988      "rb04_temp"       => Some(self.rb04_temp.clone()), 
989      "rb05_temp"       => Some(self.rb05_temp.clone()), 
990      "rb06_temp"       => Some(self.rb06_temp.clone()), 
991      "rb07_temp"       => Some(self.rb07_temp.clone()),
992      "rb08_temp"       => Some(self.rb08_temp.clone()), 
993      "rb09_temp"       => Some(self.rb09_temp.clone()), 
994      "rb11_temp"       => Some(self.rb11_temp.clone()), 
995      "rb13_temp"       => Some(self.rb13_temp.clone()),
996      "rb14_temp"       => Some(self.rb14_temp.clone()), 
997      "rb15_temp"       => Some(self.rb15_temp.clone()), 
998      "rb16_temp"       => Some(self.rb16_temp.clone()), 
999      "rb17_temp"       => Some(self.rb17_temp.clone()), 
1000      "rb18_temp"       => Some(self.rb18_temp.clone()), 
1001      "rb19_temp"       => Some(self.rb19_temp.clone()), 
1002      "rb20_temp"       => Some(self.rb20_temp.clone()), 
1003      "rb21_temp"       => Some(self.rb21_temp.clone()), 
1004      "rb22_temp"       => Some(self.rb22_temp.clone()), 
1005      "rb23_temp"       => Some(self.rb23_temp.clone()), 
1006      "rb24_temp"       => Some(self.rb24_temp.clone()), 
1007      "rb25_temp"       => Some(self.rb25_temp.clone()), 
1008      "rb26_temp"       => Some(self.rb26_temp.clone()), 
1009      "rb27_temp"       => Some(self.rb27_temp.clone()), 
1010      "rb28_temp"       => Some(self.rb28_temp.clone()), 
1011      "rb29_temp"       => Some(self.rb29_temp.clone()), 
1012      "rb30_temp"       => Some(self.rb30_temp.clone()), 
1013      "rb31_temp"       => Some(self.rb31_temp.clone()), 
1014      "rb32_temp"       => Some(self.rb32_temp.clone()), 
1015      "rb33_temp"       => Some(self.rb33_temp.clone()), 
1016      "rb34_temp"       => Some(self.rb34_temp.clone()), 
1017      "rb35_temp"       => Some(self.rb35_temp.clone()), 
1018      "rb36_temp"       => Some(self.rb36_temp.clone()), 
1019      "rb39_temp"       => Some(self.rb39_temp.clone()), 
1020      "rb40_temp"       => Some(self.rb40_temp.clone()), 
1021      "rb41_temp"       => Some(self.rb41_temp.clone()), 
1022      "rb42_temp"       => Some(self.rb42_temp.clone()), 
1023      "rb44_temp"       => Some(self.rb44_temp.clone()), 
1024      "rb46_temp"       => Some(self.rb46_temp.clone()), 
1025      "rb01_rate_zero"  => Some(self.rb01_rate_zero.clone()), 
1026      "rb02_rate_zero"  => Some(self.rb02_rate_zero.clone()),  
1027      "rb03_rate_zero"  => Some(self.rb03_rate_zero.clone()),  
1028      "rb04_rate_zero"  => Some(self.rb04_rate_zero.clone()), 
1029      "rb05_rate_zero"  => Some(self.rb05_rate_zero.clone()), 
1030      "rb06_rate_zero"  => Some(self.rb06_rate_zero.clone()), 
1031      "rb07_rate_zero"  => Some(self.rb07_rate_zero.clone()), 
1032      "rb08_rate_zero"  => Some(self.rb08_rate_zero.clone()), 
1033      "rb09_rate_zero"  => Some(self.rb09_rate_zero.clone()), 
1034      "rb11_rate_zero"  => Some(self.rb11_rate_zero.clone()), 
1035      "rb13_rate_zero"  => Some(self.rb13_rate_zero.clone()), 
1036      "rb14_rate_zero"  => Some(self.rb14_rate_zero.clone()), 
1037      "rb15_rate_zero"  => Some(self.rb15_rate_zero.clone()), 
1038      "rb16_rate_zero"  => Some(self.rb16_rate_zero.clone()), 
1039      "rb17_rate_zero"  => Some(self.rb17_rate_zero.clone()), 
1040      "rb18_rate_zero"  => Some(self.rb18_rate_zero.clone()), 
1041      "rb19_rate_zero"  => Some(self.rb19_rate_zero.clone()), 
1042      "rb20_rate_zero"  => Some(self.rb20_rate_zero.clone()), 
1043      "rb21_rate_zero"  => Some(self.rb21_rate_zero.clone()), 
1044      "rb22_rate_zero"  => Some(self.rb22_rate_zero.clone()), 
1045      "rb23_rate_zero"  => Some(self.rb23_rate_zero.clone()), 
1046      "rb24_rate_zero"  => Some(self.rb24_rate_zero.clone()), 
1047      "rb25_rate_zero"  => Some(self.rb25_rate_zero.clone()), 
1048      "rb26_rate_zero"  => Some(self.rb26_rate_zero.clone()), 
1049      "rb27_rate_zero"  => Some(self.rb27_rate_zero.clone()), 
1050      "rb28_rate_zero"  => Some(self.rb28_rate_zero.clone()), 
1051      "rb29_rate_zero"  => Some(self.rb29_rate_zero.clone()), 
1052      "rb30_rate_zero"  => Some(self.rb30_rate_zero.clone()), 
1053      "rb31_rate_zero"  => Some(self.rb31_rate_zero.clone()), 
1054      "rb32_rate_zero"  => Some(self.rb32_rate_zero.clone()), 
1055      "rb33_rate_zero"  => Some(self.rb33_rate_zero.clone()), 
1056      "rb34_rate_zero"  => Some(self.rb34_rate_zero.clone()), 
1057      "rb35_rate_zero"  => Some(self.rb35_rate_zero.clone()), 
1058      "rb36_rate_zero"  => Some(self.rb36_rate_zero.clone()), 
1059      "rb39_rate_zero"  => Some(self.rb39_rate_zero.clone()), 
1060      "rb40_rate_zero"  => Some(self.rb40_rate_zero.clone()), 
1061      "rb41_rate_zero"  => Some(self.rb41_rate_zero.clone()), 
1062      "rb42_rate_zero"  => Some(self.rb42_rate_zero.clone()), 
1063      "rb44_rate_zero"  => Some(self.rb44_rate_zero.clone()), 
1064      "rb46_rate_zero"  => Some(self.rb46_rate_zero.clone()), 
1065      "rb01_hk_too_old" => Some(self.rb01_hk_too_old.clone()), 
1066      "rb02_hk_too_old" => Some(self.rb02_hk_too_old.clone()),  
1067      "rb03_hk_too_old" => Some(self.rb03_hk_too_old.clone()),  
1068      "rb04_hk_too_old" => Some(self.rb04_hk_too_old.clone()), 
1069      "rb05_hk_too_old" => Some(self.rb05_hk_too_old.clone()), 
1070      "rb06_hk_too_old" => Some(self.rb06_hk_too_old.clone()), 
1071      "rb07_hk_too_old" => Some(self.rb07_hk_too_old.clone()), 
1072      "rb08_hk_too_old" => Some(self.rb08_hk_too_old.clone()), 
1073      "rb09_hk_too_old" => Some(self.rb09_hk_too_old.clone()), 
1074      "rb11_hk_too_old" => Some(self.rb11_hk_too_old.clone()), 
1075      "rb13_hk_too_old" => Some(self.rb13_hk_too_old.clone()), 
1076      "rb14_hk_too_old" => Some(self.rb14_hk_too_old.clone()), 
1077      "rb15_hk_too_old" => Some(self.rb15_hk_too_old.clone()), 
1078      "rb16_hk_too_old" => Some(self.rb16_hk_too_old.clone()), 
1079      "rb17_hk_too_old" => Some(self.rb17_hk_too_old.clone()), 
1080      "rb18_hk_too_old" => Some(self.rb18_hk_too_old.clone()), 
1081      "rb19_hk_too_old" => Some(self.rb19_hk_too_old.clone()), 
1082      "rb20_hk_too_old" => Some(self.rb20_hk_too_old.clone()), 
1083      "rb21_hk_too_old" => Some(self.rb21_hk_too_old.clone()), 
1084      "rb22_hk_too_old" => Some(self.rb22_hk_too_old.clone()), 
1085      "rb23_hk_too_old" => Some(self.rb23_hk_too_old.clone()), 
1086      "rb24_hk_too_old" => Some(self.rb24_hk_too_old.clone()), 
1087      "rb25_hk_too_old" => Some(self.rb25_hk_too_old.clone()), 
1088      "rb26_hk_too_old" => Some(self.rb26_hk_too_old.clone()), 
1089      "rb27_hk_too_old" => Some(self.rb27_hk_too_old.clone()), 
1090      "rb28_hk_too_old" => Some(self.rb28_hk_too_old.clone()), 
1091      "rb29_hk_too_old" => Some(self.rb29_hk_too_old.clone()), 
1092      "rb30_hk_too_old" => Some(self.rb30_hk_too_old.clone()), 
1093      "rb31_hk_too_old" => Some(self.rb31_hk_too_old.clone()), 
1094      "rb32_hk_too_old" => Some(self.rb32_hk_too_old.clone()), 
1095      "rb33_hk_too_old" => Some(self.rb33_hk_too_old.clone()), 
1096      "rb34_hk_too_old" => Some(self.rb34_hk_too_old.clone()), 
1097      "rb35_hk_too_old" => Some(self.rb35_hk_too_old.clone()), 
1098      "rb36_hk_too_old" => Some(self.rb36_hk_too_old.clone()), 
1099      "rb39_hk_too_old" => Some(self.rb39_hk_too_old.clone()), 
1100      "rb40_hk_too_old" => Some(self.rb40_hk_too_old.clone()), 
1101      "rb41_hk_too_old" => Some(self.rb41_hk_too_old.clone()), 
1102      "rb42_hk_too_old" => Some(self.rb42_hk_too_old.clone()), 
1103      "rb44_hk_too_old" => Some(self.rb44_hk_too_old.clone()), 
1104      "rb46_hk_too_old" => Some(self.rb46_hk_too_old.clone()), 
1105      _                => None
1106    }
1107  }
1108  
1109  pub fn from_toml(filename : &str) -> Result<Self, SerializationError> {
1110    match File::open(filename) {
1111      Err(err) => {
1112        println!("Unable to open {}! {}", filename, err);
1113        return Err(SerializationError::TomlDecodingError);
1114      }
1115      Ok(mut file) => {
1116        let mut toml_string = String::from("");
1117        match file.read_to_string(&mut toml_string) {
1118          Err(err) => {
1119            println!("Unable to read {}! {}", filename, err);
1120            return Err(SerializationError::TomlDecodingError);
1121          }
1122          Ok(_) => {
1123            match toml::from_str::<TofAlertManifest>(&toml_string) {
1124              Err(err) => {
1125                println!("Can't interpret toml! {}", err);
1126                return Err(SerializationError::TomlDecodingError);
1127              }
1128              Ok(manifest) => {
1129                Ok(manifest)
1130              }
1131            }
1132          }
1133        }
1134      }
1135    }
1136  }
1137  
1138  pub fn to_toml(&self, mut filename : String) {
1139    if !filename.ends_with(".toml") {
1140      filename += ".toml";
1141    }
1142    info!("Will write to file {}!", filename);
1143    match File::create(&filename) {
1144      Err(err) => {
1145        error!("Unable to open file {}! {}", filename, err);
1146      }
1147      Ok(mut file) => {
1148        match toml::to_string_pretty(&self) {
1149          Err(err) => {
1150            error!("Unable to serialize toml! {err}");
1151          }
1152          Ok(toml_string) => {
1153            match file.write_all(toml_string.as_bytes()) {
1154              Err(err) => error!("Unable to write to file {}! {}", filename, err),
1155              Ok(_)    => debug!("Wrote settings to {}!", filename)
1156            }
1157          }
1158        }
1159      }
1160    }
1161  }
1162  
1163}
1164
1165
1166/// Load all alerts from the manifest
1167pub fn load_alerts<'a>(manifest : TofAlertManifest) -> HashMap<&'a str, TofAlert<'a>> {
1168  let mut alerts = HashMap::from([("mtb_rate_zero", TofAlert {
1169                                                        key          : "mtb_rate_zero",
1170                                                        descr        : "MTB is not triggering!",
1171                                                        whattodo     : vec!["1) Check about TIU status", "2) Run restart", "3) If SSH available, debug with pybindings", "4) Soft reboot MTB", "5) Hard reboot MTB"],
1172                                                        config       : TofAlertConfig::new(),
1173                                                        variable     : Variable::TriggerRate,
1174                                                        outofbound   : OutOfBound::Zero,
1175                                                        component    : Component::MTB,
1176                                                        n_paged      : 0,
1177                                                        triggered    : None}),
1178                                   ("mtb_lost_rate", TofAlert { 
1179                                                        key          : "mtb_lost_rate",
1180                                                        descr        : "MTB Lost rate is whacky!!",
1181                                                        whattodo     : vec!["Unclear course of action. Check with tracker, maybe nothing can be done!"],
1182                                                        config       : TofAlertConfig::new(),
1183                                                        variable     : Variable::LostTriggerRate,
1184                                                        outofbound   : OutOfBound::TooHigh,
1185                                                        component    : Component::MTB,
1186                                                        n_paged      : 0,
1187                                                        triggered    : None}),
1188                                    ("mtb_fpga_temp", TofAlert {
1189                                                        key          : "mtb_fpga_temp",
1190                                                        descr        : "MTB Temperature out of bounds!!",
1191                                                        whattodo     : vec!["This is critical! Check with everyone and then shut off the MTB!"],
1192                                                        config       : TofAlertConfig::new(),
1193                                                        variable     : Variable::FPGATemp,
1194                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
1195                                                        component    : Component::MTB,
1196                                                        n_paged      : 0,
1197                                                        triggered    : None}),
1198                                    ("mtb_hk_too_old" , TofAlert {
1199                                                        key          : "mtb_hk_too_old", 
1200                                                        descr        : "MTBMoniData is out-of-date!",
1201                                                        whattodo     : vec!["Critical! This most likely means we are not triggering!" ,  "1) Check with tracker", "2) Run restart", "3) Soft reboot MTB", "4) Powercycle CAT"],
1202                                                        config       : TofAlertConfig::new(),
1203                                                        variable     : Variable::MoniData,
1204                                                        outofbound   : OutOfBound::TooOld,
1205                                                        component    : Component::RB(40),
1206                                                        n_paged      : 0,
1207                                                        triggered    : None}),
1208                                    ("cpu_core0_temp", TofAlert {
1209                                                        key          : "cpu_core0_temp",
1210                                                        descr        : "CPU Core0 temperatrue is out of bounds!",
1211                                                        whattodo     : vec!["This is critical! Check with everyone and then stop the run and likely turn off the CAT!"],
1212                                                        config       : TofAlertConfig::new(),
1213                                                        variable     : Variable::CoreTemp,
1214                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
1215                                                        component    : Component::CPU,
1216                                                        n_paged      : 0,
1217                                                        triggered    : None}),
1218                                    ("cpu_core1_temp", TofAlert {
1219                                                        key          : "cpu_core1_temp",
1220                                                        descr        : "CPU Core1 temperatrue is out of bounds!",
1221                                                        whattodo     : vec!["This is critical! Check with everyone and then stop the run and likely turn off the CAT!"],
1222                                                        config       : TofAlertConfig::new(),
1223                                                        variable     : Variable::CoreTemp,
1224                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
1225                                                        component    : Component::CPU,
1226                                                        n_paged      : 0,
1227                                                        triggered    : None}),
1228                                    ("cpu_disk", TofAlert {
1229                                                        key          : "cpu_disk",
1230                                                        descr        : "The disks on the TOF CPU are getting full.",
1231                                                        whattodo     : vec!["This is a general issue and need to be discussed in ops","No immediate action necessary"],
1232                                                        config       : TofAlertConfig::new(),
1233                                                        variable     : Variable::AvailableDiskSpace,
1234                                                        outofbound   : OutOfBound::TooHigh,
1235                                                        component    : Component::CPU,
1236                                                        n_paged      : 0,
1237                                                        triggered    : None}),
1238                                    ("cpu_hk_too_old" , TofAlert {
1239                                                        key          : "cpu_hk_too_old", 
1240                                                        descr        : "CPUMoniData is out-of-date!",
1241                                                        whattodo     : vec!["If everything else is fine, this is non-critical", "Typicallly, if this is an issue, there might be other, more severe issues!"],
1242                                                        config       : TofAlertConfig::new(),
1243                                                        variable     : Variable::MoniData,
1244                                                        outofbound   : OutOfBound::TooOld,
1245                                                        component    : Component::RB(40),
1246                                                        n_paged      : 0,
1247                                                        triggered    : None}),
1248                                    ("data_mangling", TofAlert {
1249                                                        key          : "data_mangling",
1250                                                        descr        : "Data mangling (intermix of RB channels on one or several RBs) is excessive!",
1251                                                        whattodo     : vec!["This is a general issue!","No immediate action necessary","Options have to be discussed"],
1252                                                        config       : TofAlertConfig::new(),
1253                                                        variable     : Variable::DataMangling,
1254                                                        outofbound   : OutOfBound::TooHigh,
1255                                                        component    : Component::System,
1256                                                        n_paged      : 0,
1257                                                        triggered    : None}),
1258                                    ("miss_tofevid", TofAlert {
1259                                                        key          : "miss_tofevid",
1260                                                        descr        : "The Tof system is missing event ids in an excessive amount!",
1261                                                        whattodo     : vec!["This is nist likely related to ","No immediate action necessary","Options have to be discussed"],
1262                                                        config       : TofAlertConfig::new(),
1263                                                        variable     : Variable::DataMangling,
1264                                                        outofbound   : OutOfBound::TooHigh,
1265                                                        component    : Component::System,
1266                                                        n_paged      : 0,
1267                                                        triggered    : None}),
1268                                    ("tofev_vs_merge", TofAlert {
1269                                                        key          : "tofev_vs_merge",
1270                                                        descr        : "The rate of Tof event is larger thatn the rate of merged events!",
1271                                                        whattodo     : vec!["If you are getting the tof events from telemetry, this simply can't happen ","If this happens, something is utterly broken, most likely with our monitoring","In case we don't get TofEventSummary from telemetry but directlly from the TofCPU, we are on ground and this simply needs to be debugged"],
1272                                                        config       : TofAlertConfig::new(),
1273                                                        variable     : Variable::Telemetry,
1274                                                        outofbound   : OutOfBound::TooHigh,
1275                                                        component    : Component::System,
1276                                                        n_paged      : 0,
1277                                                        triggered    : None}),
1278                                    ("notof_vs_merge", TofAlert {
1279                                                        key          : "notof_vs_merge",
1280                                                        descr        : "The rate of merged events without TofEvents is excessive!",
1281                                                        whattodo     : vec!["The stream of TofEvents to the flight computer might have stopped","1) Run restart recommended","2) Soft Reboot MTB", "3) Hard reboot CAT"],
1282                                                        config       : TofAlertConfig::new(),
1283                                                        variable     : Variable::Telemetry,
1284                                                        outofbound   : OutOfBound::TooHigh,
1285                                                        component    : Component::System,
1286                                                        n_paged      : 0,
1287                                                        triggered    : None}),
1288                                    ("frac_interest", TofAlert {
1289                                                        key          : "frac_interest",
1290                                                        descr        : "The rate of interesting events is too low!",
1291                                                        whattodo     : vec!["The stream of TofEvents to the flight computer might have stopped","1) Run restart recommended","2) Soft Reboot MTB", "3) Hard reboot CAT"],
1292                                                        config       : TofAlertConfig::new(),
1293                                                        variable     : Variable::Telemetry,
1294                                                        outofbound   : OutOfBound::TooHigh,
1295                                                        component    : Component::System,
1296                                                        n_paged      : 0,
1297                                                        triggered    : None}),
1298
1299                                    // 40x 3 RB alerts. I am only doing this to celebrate me being
1300                                    // miserable on basically ny's eve. May the gods have mercy and 
1301                                    // strike me down.
1302                                    ("rb01_rate_zero" , TofAlert {
1303                                                        key          : "rb01_rate_zero", 
1304                                                        descr        : "RB01 is not triggering!", 
1305                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
1306                                                        config       : TofAlertConfig::new(),
1307                                                        variable     : Variable::TriggerRate,
1308                                                        outofbound   : OutOfBound::Zero,
1309                                                        component    : Component::RB(1),
1310                                                        n_paged      : 0,
1311                                                        triggered    : None}),
1312                                    ("rb01_temp" , TofAlert {
1313                                                        key          : "rb01_temp", 
1314                                                        descr        : "RB01 (FPGA) temperature is out of bounds!",
1315                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
1316                                                        config       : TofAlertConfig::new(),
1317                                                        variable     : Variable::FPGATemp,
1318                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
1319                                                        component    : Component::RB(1),
1320                                                        n_paged      : 0,
1321                                                        triggered    : None}),
1322                                    ("rb01_hk_too_old" , TofAlert {
1323                                                        key          : "rb01_hk_too_old", 
1324                                                        descr        : "RB01 RBMoniData is out-of-date!",
1325                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
1326                                                        config       : TofAlertConfig::new(),
1327                                                        variable     : Variable::MoniData,
1328                                                        outofbound   : OutOfBound::TooOld,
1329                                                        component    : Component::RB(1),
1330                                                        n_paged      : 0,
1331                                                        triggered    : None}),
1332                                    /////////////////////////////////////////////////////////
1333                                    ("rb02_rate_zero" , TofAlert {
1334                                                        key          : "rb02_rate_zero", 
1335                                                        descr        : "RB02 is not triggering!", 
1336                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
1337                                                        config       : TofAlertConfig::new(),
1338                                                        variable     : Variable::TriggerRate,
1339                                                        outofbound   : OutOfBound::Zero,
1340                                                        component    : Component::RB(2),
1341                                                        n_paged      : 0,
1342                                                        triggered    : None}),
1343                                    ("rb02_temp" , TofAlert {
1344                                                        key          : "rb02_temp", 
1345                                                        descr        : "RB02 (FPGA) temperature is out of bounds!",
1346                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
1347                                                        config       : TofAlertConfig::new(),
1348                                                        variable     : Variable::FPGATemp,
1349                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
1350                                                        component    : Component::RB(2),
1351                                                        n_paged      : 0,
1352                                                        triggered    : None}),
1353                                    ("rb02_hk_too_old" , TofAlert {
1354                                                        key          : "rb02_hk_too_old", 
1355                                                        descr        : "RB02 RBMoniData is out-of-date!",
1356                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
1357                                                        config       : TofAlertConfig::new(),
1358                                                        variable     : Variable::MoniData,
1359                                                        outofbound   : OutOfBound::TooOld,
1360                                                        n_paged      : 0,
1361                                                        component    : Component::RB(2),
1362                                                        triggered    : None}),
1363                                    ("rb03_rate_zero" , TofAlert {
1364                                                        key          : "rb03_rate_zero", 
1365                                                        descr        : "RB03 is not triggering!", 
1366                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
1367                                                        config       : TofAlertConfig::new(),
1368                                                        variable     : Variable::TriggerRate,
1369                                                        outofbound   : OutOfBound::Zero,
1370                                                        n_paged      : 0,
1371                                                        component    : Component::RB(3),
1372                                                        triggered    : None}),
1373                                    ("rb03_temp" , TofAlert {
1374                                                        key          : "rb03_temp", 
1375                                                        descr        : "RB03 (FPGA) temperature is out of bounds!",
1376                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
1377                                                        config       : TofAlertConfig::new(),
1378                                                        variable     : Variable::FPGATemp,
1379                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
1380                                                        component    : Component::RB(3),
1381                                                        n_paged      : 0,
1382                                                        triggered    : None}),
1383                                    ("rb03_hk_too_old" , TofAlert {
1384                                                        key          : "rb03_hk_too_old", 
1385                                                        descr        : "RB03 RBMoniData is out-of-date!",
1386                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
1387                                                        config       : TofAlertConfig::new(),
1388                                                        variable     : Variable::MoniData,
1389                                                        outofbound   : OutOfBound::TooOld,
1390                                                        component    : Component::RB(3),
1391                                                        n_paged      : 0,
1392                                                        triggered    : None}),
1393                                    ("rb04_rate_zero" , TofAlert {
1394                                                        key          : "rb04_rate_zero", 
1395                                                        descr        : "RB04 is not triggering!", 
1396                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
1397                                                        config       : TofAlertConfig::new(),
1398                                                        variable     : Variable::TriggerRate,
1399                                                        outofbound   : OutOfBound::Zero,
1400                                                        component    : Component::RB(4),
1401                                                        n_paged      : 0,
1402                                                        triggered    : None}),
1403                                    ("rb04_temp" , TofAlert {
1404                                                        key          : "rb04_temp", 
1405                                                        descr        : "RB04 (FPGA) temperature is out of bounds!",
1406                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
1407                                                        config       : TofAlertConfig::new(),
1408                                                        variable     : Variable::FPGATemp,
1409                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
1410                                                        component    : Component::RB(4),
1411                                                        n_paged      : 0,
1412                                                        triggered    : None}),
1413                                    ("rb04_hk_too_old" , TofAlert {
1414                                                        key          : "rb05_hk_too_old", 
1415                                                        descr        : "RB05 RBMoniData is out-of-date!",
1416                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
1417                                                        config       : TofAlertConfig::new(),
1418                                                        variable     : Variable::MoniData,
1419                                                        outofbound   : OutOfBound::TooOld,
1420                                                        component    : Component::RB(4),
1421                                                        n_paged      : 0,
1422                                                        triggered    : None}),
1423                                    ("rb05_rate_zero" , TofAlert {
1424                                                        key          : "rb05_rate_zero", 
1425                                                        descr        : "RB05 is not triggering!", 
1426                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
1427                                                        config       : TofAlertConfig::new(),
1428                                                        variable     : Variable::TriggerRate,
1429                                                        outofbound   : OutOfBound::Zero,
1430                                                        n_paged      : 0,
1431                                                        component    : Component::RB(5),
1432                                                        triggered    : None}),
1433                                    ("rb05_temp" , TofAlert {
1434                                                        key          : "rb05_temp", 
1435                                                        descr        : "RB01 (FPGA) temperature is out of bounds!",
1436                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
1437                                                        config       : TofAlertConfig::new(),
1438                                                        variable     : Variable::FPGATemp,
1439                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
1440                                                        component    : Component::RB(5),
1441                                                        n_paged      : 0,
1442                                                        triggered    : None}),
1443                                    ("rb05_hk_too_old" , TofAlert {
1444                                                        key          : "rb05_hk_too_old", 
1445                                                        descr        : "RB01 RBMoniData is out-of-date!",
1446                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
1447                                                        config       : TofAlertConfig::new(),
1448                                                        variable     : Variable::MoniData,
1449                                                        outofbound   : OutOfBound::TooOld,
1450                                                        component    : Component::RB(5),
1451                                                        n_paged      : 0,
1452                                                        triggered    : None}),
1453                                    ("rb06_rate_zero" , TofAlert {
1454                                                        key          : "rb06_rate_zero", 
1455                                                        descr        : "RB06 is not triggering!", 
1456                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
1457                                                        config       : TofAlertConfig::new(),
1458                                                        variable     : Variable::TriggerRate,
1459                                                        outofbound   : OutOfBound::Zero,
1460                                                        component    : Component::RB(6),
1461                                                        n_paged      : 0,
1462                                                        triggered    : None}),
1463                                    ("rb06_temp" , TofAlert {
1464                                                        key          : "rb06_temp", 
1465                                                        descr        : "RB06 (FPGA) temperature is out of bounds!",
1466                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
1467                                                        config       : TofAlertConfig::new(),
1468                                                        variable     : Variable::FPGATemp,
1469                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
1470                                                        component    : Component::RB(6),
1471                                                        n_paged      : 0,
1472                                                        triggered    : None}),
1473                                    ("rb06_hk_too_old" , TofAlert {
1474                                                        key          : "rb06_hk_too_old", 
1475                                                        descr        : "RB06 RBMoniData is out-of-date!",
1476                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
1477                                                        config       : TofAlertConfig::new(),
1478                                                        variable     : Variable::MoniData,
1479                                                        outofbound   : OutOfBound::TooOld,
1480                                                        component    : Component::RB(6),
1481                                                        n_paged      : 0,
1482                                                        triggered    : None}),
1483                                    ("rb07_rate_zero" , TofAlert {
1484                                                        key          : "rb07_rate_zero", 
1485                                                        descr        : "RB07 is not triggering!", 
1486                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
1487                                                        config       : TofAlertConfig::new(),
1488                                                        variable     : Variable::TriggerRate,
1489                                                        outofbound   : OutOfBound::Zero,
1490                                                        component    : Component::RB(7),
1491                                                        n_paged      : 0,
1492                                                        triggered    : None}),
1493                                    ("rb07_temp" , TofAlert {
1494                                                        key          : "rb07_temp", 
1495                                                        descr        : "RB07 (FPGA) temperature is out of bounds!",
1496                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
1497                                                        config       : TofAlertConfig::new(),
1498                                                        variable     : Variable::FPGATemp,
1499                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
1500                                                        component    : Component::RB(7),
1501                                                        n_paged      : 0,
1502                                                        triggered    : None}),
1503                                    ("rb07_hk_too_old" , TofAlert {
1504                                                        key          : "rb07_hk_too_old", 
1505                                                        descr        : "RB07 RBMoniData is out-of-date!",
1506                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
1507                                                        config       : TofAlertConfig::new(),
1508                                                        variable     : Variable::MoniData,
1509                                                        outofbound   : OutOfBound::TooOld,
1510                                                        component    : Component::RB(7),
1511                                                        n_paged      : 0,
1512                                                        triggered    : None}),
1513                                    ("rb08_rate_zero" , TofAlert {
1514                                                        key          : "rb08_rate_zero", 
1515                                                        descr        : "RB08 is not triggering!", 
1516                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
1517                                                        config       : TofAlertConfig::new(),
1518                                                        variable     : Variable::TriggerRate,
1519                                                        outofbound   : OutOfBound::Zero,
1520                                                        component    : Component::RB(8),
1521                                                        n_paged      : 0,
1522                                                        triggered    : None}),
1523                                    ("rb08_temp" , TofAlert {
1524                                                        key          : "rb08_temp", 
1525                                                        descr        : "RB08 (FPGA) temperature is out of bounds!",
1526                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
1527                                                        config       : TofAlertConfig::new(),
1528                                                        variable     : Variable::FPGATemp,
1529                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
1530                                                        component    : Component::RB(8),
1531                                                        n_paged      : 0,
1532                                                        triggered    : None}),
1533                                    ("rb08_hk_too_old" , TofAlert {
1534                                                        key          : "rb08_hk_too_old", 
1535                                                        descr        : "RB08 RBMoniData is out-of-date!",
1536                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
1537                                                        config       : TofAlertConfig::new(),
1538                                                        variable     : Variable::MoniData,
1539                                                        outofbound   : OutOfBound::TooOld,
1540                                                        component    : Component::RB(8),
1541                                                        n_paged      : 0,
1542                                                        triggered    : None}),
1543                                    ("rb09_rate_zero" , TofAlert {
1544                                                        key          : "rb09_rate_zero", 
1545                                                        descr        : "RB09 is not triggering!", 
1546                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
1547                                                        config       : TofAlertConfig::new(),
1548                                                        variable     : Variable::TriggerRate,
1549                                                        outofbound   : OutOfBound::Zero,
1550                                                        component    : Component::RB(9),
1551                                                        n_paged      : 0,
1552                                                        triggered    : None}),
1553                                    ("rb09_temp" , TofAlert {
1554                                                        key          : "rb09_temp", 
1555                                                        descr        : "RB09 (FPGA) temperature is out of bounds!",
1556                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
1557                                                        config       : TofAlertConfig::new(),
1558                                                        variable     : Variable::FPGATemp,
1559                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
1560                                                        component    : Component::RB(9),
1561                                                        n_paged      : 0,
1562                                                        triggered    : None}),
1563                                    ("rb09_hk_too_old" , TofAlert {
1564                                                        key          : "rb09_hk_too_old", 
1565                                                        descr        : "RB09 RBMoniData is out-of-date!",
1566                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
1567                                                        config       : TofAlertConfig::new(),
1568                                                        variable     : Variable::MoniData,
1569                                                        outofbound   : OutOfBound::TooOld,
1570                                                        component    : Component::RB(9),
1571                                                        n_paged      : 0,
1572                                                        triggered    : None}),
1573                                    ("rb11_rate_zero" , TofAlert {
1574                                                        key          : "rb11_rate_zero", 
1575                                                        descr        : "RB11 is not triggering!", 
1576                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
1577                                                        config       : TofAlertConfig::new(),
1578                                                        variable     : Variable::TriggerRate,
1579                                                        outofbound   : OutOfBound::Zero,
1580                                                        component    : Component::RB(11),
1581                                                        n_paged      : 0,
1582                                                        triggered    : None}),
1583                                    ("rb11_temp" , TofAlert {
1584                                                        key          : "rb11_temp", 
1585                                                        descr        : "RB11 (FPGA) temperature is out of bounds!",
1586                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
1587                                                        config       : TofAlertConfig::new(),
1588                                                        variable     : Variable::FPGATemp,
1589                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
1590                                                        component    : Component::RB(11),
1591                                                        n_paged      : 0,
1592                                                        triggered    : None}),
1593                                    ("rb11_hk_too_old" , TofAlert {
1594                                                        key          : "rb11_hk_too_old", 
1595                                                        descr        : "RB11 RBMoniData is out-of-date!",
1596                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
1597                                                        config       : TofAlertConfig::new(),
1598                                                        variable     : Variable::MoniData,
1599                                                        outofbound   : OutOfBound::TooOld,
1600                                                        component    : Component::RB(11),
1601                                                        n_paged      : 0,
1602                                                        triggered    : None}),
1603                                    /////////////////////////////////////////////////////////
1604                                    ("rb13_rate_zero" , TofAlert {
1605                                                        key          : "rb13_rate_zero", 
1606                                                        descr        : "RB13 is not triggering!", 
1607                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
1608                                                        config       : TofAlertConfig::new(),
1609                                                        variable     : Variable::TriggerRate,
1610                                                        outofbound   : OutOfBound::Zero,
1611                                                        component    : Component::RB(13),
1612                                                        n_paged      : 0,
1613                                                        triggered    : None}),
1614                                    ("rb13_temp" , TofAlert {
1615                                                        key          : "rb13_temp", 
1616                                                        descr        : "RB13 (FPGA) temperature is out of bounds!",
1617                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
1618                                                        config       : TofAlertConfig::new(),
1619                                                        variable     : Variable::FPGATemp,
1620                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
1621                                                        component    : Component::RB(13),
1622                                                        n_paged      : 0,
1623                                                        triggered    : None}),
1624                                    ("rb13_hk_too_old" , TofAlert {
1625                                                        key          : "rb13_hk_too_old", 
1626                                                        descr        : "RB13 RBMoniData is out-of-date!",
1627                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
1628                                                        config       : TofAlertConfig::new(),
1629                                                        variable     : Variable::MoniData,
1630                                                        outofbound   : OutOfBound::TooOld,
1631                                                        component    : Component::RB(13),
1632                                                        n_paged      : 0,
1633                                                        triggered    : None}),
1634                                    ("rb14_rate_zero" , TofAlert {
1635                                                        key          : "rb14_rate_zero", 
1636                                                        descr        : "RB14 is not triggering!", 
1637                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
1638                                                        config       : TofAlertConfig::new(),
1639                                                        variable     : Variable::TriggerRate,
1640                                                        outofbound   : OutOfBound::Zero,
1641                                                        component    : Component::RB(14),
1642                                                        n_paged      : 0,
1643                                                        triggered    : None}),
1644                                    ("rb14_temp" , TofAlert {
1645                                                        key          : "rb14_temp", 
1646                                                        descr        : "RB14 (FPGA) temperature is out of bounds!",
1647                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
1648                                                        config       : TofAlertConfig::new(),
1649                                                        variable     : Variable::FPGATemp,
1650                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
1651                                                        component    : Component::RB(14),
1652                                                        n_paged      : 0,
1653                                                        triggered    : None}),
1654                                    ("rb14_hk_too_old" , TofAlert {
1655                                                        key          : "rb14_hk_too_old", 
1656                                                        descr        : "RB14 RBMoniData is out-of-date!",
1657                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
1658                                                        config       : TofAlertConfig::new(),
1659                                                        variable     : Variable::MoniData,
1660                                                        outofbound   : OutOfBound::TooOld,
1661                                                        component    : Component::RB(14),
1662                                                        n_paged      : 0,
1663                                                        triggered    : None}),
1664                                    ("rb15_rate_zero" , TofAlert {
1665                                                        key          : "rb15_rate_zero", 
1666                                                        descr        : "RB15 is not triggering!", 
1667                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
1668                                                        config       : TofAlertConfig::new(),
1669                                                        variable     : Variable::TriggerRate,
1670                                                        outofbound   : OutOfBound::Zero,
1671                                                        component    : Component::RB(15),
1672                                                        n_paged      : 0,
1673                                                        triggered    : None}),
1674                                    ("rb15_temp" , TofAlert {
1675                                                        key          : "rb15_temp", 
1676                                                        descr        : "RB15 (FPGA) temperature is out of bounds!",
1677                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
1678                                                        config       : TofAlertConfig::new(),
1679                                                        variable     : Variable::FPGATemp,
1680                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
1681                                                        component    : Component::RB(15),
1682                                                        n_paged      : 0,
1683                                                        triggered    : None}),
1684                                    ("rb15_hk_too_old" , TofAlert {
1685                                                        key          : "rb15_hk_too_old", 
1686                                                        descr        : "RB15 RBMoniData is out-of-date!",
1687                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
1688                                                        config       : TofAlertConfig::new(),
1689                                                        variable     : Variable::MoniData,
1690                                                        outofbound   : OutOfBound::TooOld,
1691                                                        component    : Component::RB(15),
1692                                                        n_paged      : 0,
1693                                                        triggered    : None}),
1694                                    ("rb16_rate_zero" , TofAlert {
1695                                                        key          : "rb16_rate_zero", 
1696                                                        descr        : "RB16 is not triggering!", 
1697                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
1698                                                        config       : TofAlertConfig::new(),
1699                                                        variable     : Variable::TriggerRate,
1700                                                        outofbound   : OutOfBound::Zero,
1701                                                        component    : Component::RB(16),
1702                                                        n_paged      : 0,
1703                                                        triggered    : None}),
1704                                    ("rb16_temp" , TofAlert {
1705                                                        key          : "rb16_temp", 
1706                                                        descr        : "RB16 (FPGA) temperature is out of bounds!",
1707                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
1708                                                        config       : TofAlertConfig::new(),
1709                                                        variable     : Variable::FPGATemp,
1710                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
1711                                                        component    : Component::RB(16),
1712                                                        n_paged      : 0,
1713                                                        triggered    : None}),
1714                                    ("rb16_hk_too_old" , TofAlert {
1715                                                        key          : "rb16_hk_too_old", 
1716                                                        descr        : "RB16 RBMoniData is out-of-date!",
1717                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
1718                                                        config       : TofAlertConfig::new(),
1719                                                        variable     : Variable::MoniData,
1720                                                        outofbound   : OutOfBound::TooOld,
1721                                                        component    : Component::RB(16),
1722                                                        n_paged      : 0,
1723                                                        triggered    : None}),
1724                                    ("rb17_rate_zero" , TofAlert {
1725                                                        key          : "rb17_rate_zero", 
1726                                                        descr        : "RB17 is not triggering!", 
1727                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
1728                                                        config       : TofAlertConfig::new(),
1729                                                        variable     : Variable::TriggerRate,
1730                                                        outofbound   : OutOfBound::Zero,
1731                                                        component    : Component::RB(17),
1732                                                        n_paged      : 0,
1733                                                        triggered    : None}),
1734                                    ("rb17_temp" , TofAlert {
1735                                                        key          : "rb17_temp", 
1736                                                        descr        : "RB17 (FPGA) temperature is out of bounds!",
1737                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
1738                                                        config       : TofAlertConfig::new(),
1739                                                        variable     : Variable::FPGATemp,
1740                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
1741                                                        component    : Component::RB(17),
1742                                                        n_paged      : 0,
1743                                                        triggered    : None}),
1744                                    ("rb17_hk_too_old" , TofAlert {
1745                                                        key          : "rb17_hk_too_old", 
1746                                                        descr        : "RB17 RBMoniData is out-of-date!",
1747                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
1748                                                        config       : TofAlertConfig::new(),
1749                                                        variable     : Variable::MoniData,
1750                                                        outofbound   : OutOfBound::TooOld,
1751                                                        component    : Component::RB(17),
1752                                                        n_paged      : 0,
1753                                                        triggered    : None}),
1754                                    ("rb18_rate_zero" , TofAlert {
1755                                                        key          : "rb18_rate_zero", 
1756                                                        descr        : "RB18 is not triggering!", 
1757                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
1758                                                        config       : TofAlertConfig::new(),
1759                                                        variable     : Variable::TriggerRate,
1760                                                        outofbound   : OutOfBound::Zero,
1761                                                        component    : Component::RB(18),
1762                                                        n_paged      : 0,
1763                                                        triggered    : None}),
1764                                    ("rb18_temp" , TofAlert {
1765                                                        key          : "rb18_temp", 
1766                                                        descr        : "RB18 (FPGA) temperature is out of bounds!",
1767                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
1768                                                        config       : TofAlertConfig::new(),
1769                                                        variable     : Variable::FPGATemp,
1770                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
1771                                                        component    : Component::RB(18),
1772                                                        n_paged      : 0,
1773                                                        triggered    : None}),
1774                                    ("rb18_hk_too_old" , TofAlert {
1775                                                        key          : "rb18_hk_too_old", 
1776                                                        descr        : "RB18 RBMoniData is out-of-date!",
1777                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
1778                                                        config       : TofAlertConfig::new(),
1779                                                        variable     : Variable::MoniData,
1780                                                        outofbound   : OutOfBound::TooOld,
1781                                                        component    : Component::RB(18),
1782                                                        n_paged      : 0,
1783                                                        triggered    : None}),
1784                                    ("rb19_rate_zero" , TofAlert {
1785                                                        key          : "rb19_rate_zero", 
1786                                                        descr        : "RB19 is not triggering!", 
1787                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
1788                                                        config       : TofAlertConfig::new(),
1789                                                        variable     : Variable::TriggerRate,
1790                                                        outofbound   : OutOfBound::Zero,
1791                                                        component    : Component::RB(19),
1792                                                        n_paged      : 0,
1793                                                        triggered    : None}),
1794                                    ("rb19_temp" , TofAlert {
1795                                                        key          : "rb19_temp", 
1796                                                        descr        : "RB19 (FPGA) temperature is out of bounds!",
1797                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
1798                                                        config       : TofAlertConfig::new(),
1799                                                        variable     : Variable::FPGATemp,
1800                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
1801                                                        component    : Component::RB(19),
1802                                                        n_paged      : 0,
1803                                                        triggered    : None}),
1804                                    ("rb19_hk_too_old" , TofAlert {
1805                                                        key          : "rb19_hk_too_old", 
1806                                                        descr        : "RB19 RBMoniData is out-of-date!",
1807                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
1808                                                        config       : TofAlertConfig::new(),
1809                                                        variable     : Variable::MoniData,
1810                                                        outofbound   : OutOfBound::TooOld,
1811                                                        component    : Component::RB(19),
1812                                                        n_paged      : 0,
1813                                                        triggered    : None}),
1814                                    ("rb20_rate_zero" , TofAlert {
1815                                                        key          : "rb20_rate_zero", 
1816                                                        descr        : "RB20 is not triggering!", 
1817                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
1818                                                        config       : TofAlertConfig::new(),
1819                                                        variable     : Variable::TriggerRate,
1820                                                        outofbound   : OutOfBound::Zero,
1821                                                        component    : Component::RB(20),
1822                                                        n_paged      : 0,
1823                                                        triggered    : None}),
1824                                    ("rb20_temp" , TofAlert {
1825                                                        key          : "rb20_temp", 
1826                                                        descr        : "RB20 (FPGA) temperature is out of bounds!",
1827                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
1828                                                        config       : TofAlertConfig::new(),
1829                                                        variable     : Variable::FPGATemp,
1830                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
1831                                                        component    : Component::RB(20),
1832                                                        n_paged      : 0,
1833                                                        triggered    : None}),
1834                                    ("rb20_hk_too_old" , TofAlert {
1835                                                        key          : "rb20_hk_too_old", 
1836                                                        descr        : "RB20 RBMoniData is out-of-date!",
1837                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
1838                                                        config       : TofAlertConfig::new(),
1839                                                        variable     : Variable::MoniData,
1840                                                        outofbound   : OutOfBound::TooOld,
1841                                                        component    : Component::RB(20),
1842                                                        n_paged      : 0,
1843                                                        triggered    : None}),
1844                                    ("rb21_rate_zero" , TofAlert {
1845                                                        key          : "rb21_rate_zero", 
1846                                                        descr        : "RB21 is not triggering!", 
1847                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
1848                                                        config       : TofAlertConfig::new(),
1849                                                        variable     : Variable::TriggerRate,
1850                                                        outofbound   : OutOfBound::Zero,
1851                                                        component    : Component::RB(21),
1852                                                        n_paged      : 0,
1853                                                        triggered    : None}),
1854                                    ("rb21_temp" , TofAlert {
1855                                                        key          : "rb21_temp", 
1856                                                        descr        : "RB21 (FPGA) temperature is out of bounds!",
1857                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
1858                                                        config       : TofAlertConfig::new(),
1859                                                        variable     : Variable::FPGATemp,
1860                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
1861                                                        component    : Component::RB(21),
1862                                                        n_paged      : 0,
1863                                                        triggered    : None}),
1864                                    ("rb21_hk_too_old" , TofAlert {
1865                                                        key          : "rb21_hk_too_old", 
1866                                                        descr        : "RB21 RBMoniData is out-of-date!",
1867                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
1868                                                        config       : TofAlertConfig::new(),
1869                                                        variable     : Variable::MoniData,
1870                                                        outofbound   : OutOfBound::TooOld,
1871                                                        component    : Component::RB(21),
1872                                                        n_paged      : 0,
1873                                                        triggered    : None}),
1874                                    ////////////////////////////////////////////////////////
1875                                    ("rb22_rate_zero" , TofAlert {
1876                                                        key          : "rb22_rate_zero", 
1877                                                        descr        : "RB22 is not triggering!", 
1878                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
1879                                                        config       : TofAlertConfig::new(),
1880                                                        variable     : Variable::TriggerRate,
1881                                                        outofbound   : OutOfBound::Zero,
1882                                                        component    : Component::RB(22),
1883                                                        n_paged      : 0,
1884                                                        triggered    : None}),
1885                                    ("rb22_temp" , TofAlert {
1886                                                        key          : "rb22_temp", 
1887                                                        descr        : "RB22 (FPGA) temperature is out of bounds!",
1888                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
1889                                                        config       : TofAlertConfig::new(),
1890                                                        variable     : Variable::FPGATemp,
1891                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
1892                                                        component    : Component::RB(22),
1893                                                        n_paged      : 0,
1894                                                        triggered    : None}),
1895                                    ("rb22_hk_too_old" , TofAlert {
1896                                                        key          : "rb22_hk_too_old", 
1897                                                        descr        : "RB22 RBMoniData is out-of-date!",
1898                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
1899                                                        config       : TofAlertConfig::new(),
1900                                                        variable     : Variable::MoniData,
1901                                                        outofbound   : OutOfBound::TooOld,
1902                                                        component    : Component::RB(22),
1903                                                        n_paged      : 0,
1904                                                        triggered    : None}),
1905                                    ("rb23_rate_zero" , TofAlert {
1906                                                        key          : "rb23_rate_zero", 
1907                                                        descr        : "RB23 is not triggering!", 
1908                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
1909                                                        config       : TofAlertConfig::new(),
1910                                                        variable     : Variable::TriggerRate,
1911                                                        outofbound   : OutOfBound::Zero,
1912                                                        component    : Component::RB(23),
1913                                                        n_paged      : 0,
1914                                                        triggered    : None}),
1915                                    ("rb23_temp" , TofAlert {
1916                                                        key          : "rb23_temp", 
1917                                                        descr        : "RB23 (FPGA) temperature is out of bounds!",
1918                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
1919                                                        config       : TofAlertConfig::new(),
1920                                                        variable     : Variable::FPGATemp,
1921                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
1922                                                        component    : Component::RB(23),
1923                                                        n_paged      : 0,
1924                                                        triggered    : None}),
1925                                    ("rb23_hk_too_old" , TofAlert {
1926                                                        key          : "rb23_hk_too_old", 
1927                                                        descr        : "RB23 RBMoniData is out-of-date!",
1928                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
1929                                                        config       : TofAlertConfig::new(),
1930                                                        variable     : Variable::MoniData,
1931                                                        outofbound   : OutOfBound::TooOld,
1932                                                        component    : Component::RB(23),
1933                                                        n_paged      : 0,
1934                                                        triggered    : None}),
1935                                    ("rb24_rate_zero" , TofAlert {
1936                                                        key          : "rb24_rate_zero", 
1937                                                        descr        : "RB24 is not triggering!", 
1938                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
1939                                                        config       : TofAlertConfig::new(),
1940                                                        variable     : Variable::TriggerRate,
1941                                                        outofbound   : OutOfBound::Zero,
1942                                                        component    : Component::RB(24),
1943                                                        n_paged      : 0,
1944                                                        triggered    : None}),
1945                                    ("rb24_temp" , TofAlert {
1946                                                        key          : "rb24_temp", 
1947                                                        descr        : "RB24 (FPGA) temperature is out of bounds!",
1948                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
1949                                                        config       : TofAlertConfig::new(),
1950                                                        variable     : Variable::FPGATemp,
1951                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
1952                                                        component    : Component::RB(24),
1953                                                        n_paged      : 0,
1954                                                        triggered    : None}),
1955                                    ("rb24_hk_too_old" , TofAlert {
1956                                                        key          : "rb24_hk_too_old", 
1957                                                        descr        : "RB24 RBMoniData is out-of-date!",
1958                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
1959                                                        config       : TofAlertConfig::new(),
1960                                                        variable     : Variable::MoniData,
1961                                                        outofbound   : OutOfBound::TooOld,
1962                                                        component    : Component::RB(24),
1963                                                        n_paged      : 0,
1964                                                        triggered    : None}),
1965                                    ("rb25_rate_zero" , TofAlert {
1966                                                        key          : "rb25_rate_zero", 
1967                                                        descr        : "RB25 is not triggering!", 
1968                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
1969                                                        config       : TofAlertConfig::new(),
1970                                                        variable     : Variable::TriggerRate,
1971                                                        outofbound   : OutOfBound::Zero,
1972                                                        component    : Component::RB(25),
1973                                                        n_paged      : 0,
1974                                                        triggered    : None}),
1975                                    ("rb25_temp" , TofAlert {
1976                                                        key          : "rb25_temp", 
1977                                                        descr        : "RB25 (FPGA) temperature is out of bounds!",
1978                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
1979                                                        config       : TofAlertConfig::new(),
1980                                                        variable     : Variable::FPGATemp,
1981                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
1982                                                        component    : Component::RB(25),
1983                                                        n_paged      : 0,
1984                                                        triggered    : None}),
1985                                    ("rb25_hk_too_old" , TofAlert {
1986                                                        key          : "rb25_hk_too_old", 
1987                                                        descr        : "RB25 RBMoniData is out-of-date!",
1988                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
1989                                                        config       : TofAlertConfig::new(),
1990                                                        variable     : Variable::MoniData,
1991                                                        outofbound   : OutOfBound::TooOld,
1992                                                        component    : Component::RB(25),
1993                                                        n_paged      : 0,
1994                                                        triggered    : None}),
1995                                    ("rb26_rate_zero" , TofAlert {
1996                                                        key          : "rb26_rate_zero", 
1997                                                        descr        : "RB26 is not triggering!", 
1998                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
1999                                                        config       : TofAlertConfig::new(),
2000                                                        variable     : Variable::TriggerRate,
2001                                                        outofbound   : OutOfBound::Zero,
2002                                                        component    : Component::RB(26),
2003                                                        n_paged      : 0,
2004                                                        triggered    : None}),
2005                                    ("rb26_temp" , TofAlert {
2006                                                        key          : "rb26_temp", 
2007                                                        descr        : "RB26 (FPGA) temperature is out of bounds!",
2008                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
2009                                                        config       : TofAlertConfig::new(),
2010                                                        variable     : Variable::FPGATemp,
2011                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
2012                                                        component    : Component::RB(26),
2013                                                        n_paged      : 0,
2014                                                        triggered    : None}),
2015                                    ("rb26_hk_too_old" , TofAlert {
2016                                                        key          : "rb26_hk_too_old", 
2017                                                        descr        : "RB26 RBMoniData is out-of-date!",
2018                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
2019                                                        config       : TofAlertConfig::new(),
2020                                                        variable     : Variable::MoniData,
2021                                                        outofbound   : OutOfBound::TooOld,
2022                                                        component    : Component::RB(26),
2023                                                        n_paged      : 0,
2024                                                        triggered    : None}),
2025                                    ("rb27_rate_zero" , TofAlert {
2026                                                        key          : "rb27_rate_zero", 
2027                                                        descr        : "RB27 is not triggering!", 
2028                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
2029                                                        config       : TofAlertConfig::new(),
2030                                                        variable     : Variable::TriggerRate,
2031                                                        outofbound   : OutOfBound::Zero,
2032                                                        component    : Component::RB(27),
2033                                                        n_paged      : 0,
2034                                                        triggered    : None}),
2035                                    ("rb27_temp" , TofAlert {
2036                                                        key          : "rb27_temp", 
2037                                                        descr        : "RB27 (FPGA) temperature is out of bounds!",
2038                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
2039                                                        config       : TofAlertConfig::new(),
2040                                                        variable     : Variable::FPGATemp,
2041                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
2042                                                        component    : Component::RB(27),
2043                                                        n_paged      : 0,
2044                                                        triggered    : None}),
2045                                    ("rb27_hk_too_old" , TofAlert {
2046                                                        key          : "rb27_hk_too_old", 
2047                                                        descr        : "RB27 RBMoniData is out-of-date!",
2048                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
2049                                                        config       : TofAlertConfig::new(),
2050                                                        variable     : Variable::MoniData,
2051                                                        outofbound   : OutOfBound::TooOld,
2052                                                        component    : Component::RB(27),
2053                                                        n_paged      : 0,
2054                                                        triggered    : None}),
2055                                    ("rb28_rate_zero" , TofAlert {
2056                                                        key          : "rb28_rate_zero", 
2057                                                        descr        : "RB28 is not triggering!", 
2058                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
2059                                                        config       : TofAlertConfig::new(),
2060                                                        variable     : Variable::TriggerRate,
2061                                                        outofbound   : OutOfBound::Zero,
2062                                                        component    : Component::RB(28),
2063                                                        n_paged      : 0,
2064                                                        triggered    : None}),
2065                                    ("rb28_temp" , TofAlert {
2066                                                        key          : "rb28_temp", 
2067                                                        descr        : "RB28 (FPGA) temperature is out of bounds!",
2068                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
2069                                                        config       : TofAlertConfig::new(),
2070                                                        variable     : Variable::FPGATemp,
2071                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
2072                                                        component    : Component::RB(28),
2073                                                        n_paged      : 0,
2074                                                        triggered    : None}),
2075                                    ("rb28_hk_too_old" , TofAlert {
2076                                                        key          : "rb28_hk_too_old", 
2077                                                        descr        : "RB28 RBMoniData is out-of-date!",
2078                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
2079                                                        config       : TofAlertConfig::new(),
2080                                                        variable     : Variable::MoniData,
2081                                                        outofbound   : OutOfBound::TooOld,
2082                                                        component    : Component::RB(28),
2083                                                        n_paged      : 0,
2084                                                        triggered    : None}),
2085                                    ("rb29_rate_zero" , TofAlert {
2086                                                        key          : "rb29_rate_zero", 
2087                                                        descr        : "RB29 is not triggering!", 
2088                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
2089                                                        config       : TofAlertConfig::new(),
2090                                                        variable     : Variable::TriggerRate,
2091                                                        outofbound   : OutOfBound::Zero,
2092                                                        component    : Component::RB(29),
2093                                                        n_paged      : 0,
2094                                                        triggered    : None}),
2095                                    ("rb29_temp" , TofAlert {
2096                                                        key          : "rb29_temp", 
2097                                                        descr        : "RB29 (FPGA) temperature is out of bounds!",
2098                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
2099                                                        config       : TofAlertConfig::new(),
2100                                                        variable     : Variable::FPGATemp,
2101                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
2102                                                        component    : Component::RB(29),
2103                                                        n_paged      : 0,
2104                                                        triggered    : None}),
2105                                    ("rb29_hk_too_old" , TofAlert {
2106                                                        key          : "rb29_hk_too_old", 
2107                                                        descr        : "RB29 RBMoniData is out-of-date!",
2108                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
2109                                                        config       : TofAlertConfig::new(),
2110                                                        variable     : Variable::MoniData,
2111                                                        outofbound   : OutOfBound::TooOld,
2112                                                        component    : Component::RB(29),
2113                                                        n_paged      : 0,
2114                                                        triggered    : None}),
2115                                    ////////////////////////////////////////////////////////
2116                                    ("rb30_rate_zero" , TofAlert {
2117                                                        key          : "rb30_rate_zero", 
2118                                                        descr        : "RB30 is not triggering!", 
2119                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
2120                                                        config       : TofAlertConfig::new(),
2121                                                        variable     : Variable::TriggerRate,
2122                                                        outofbound   : OutOfBound::Zero,
2123                                                        component    : Component::RB(30),
2124                                                        n_paged      : 0,
2125                                                        triggered    : None}),
2126                                    ("rb30_temp" , TofAlert {
2127                                                        key          : "rb30_temp", 
2128                                                        descr        : "RB30 (FPGA) temperature is out of bounds!",
2129                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
2130                                                        config       : TofAlertConfig::new(),
2131                                                        variable     : Variable::FPGATemp,
2132                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
2133                                                        component    : Component::RB(30),
2134                                                        n_paged      : 0,
2135                                                        triggered    : None}),
2136                                    ("rb30_hk_too_old" , TofAlert {
2137                                                        key          : "rb30_hk_too_old", 
2138                                                        descr        : "RB30 RBMoniData is out-of-date!",
2139                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
2140                                                        config       : TofAlertConfig::new(),
2141                                                        variable     : Variable::MoniData,
2142                                                        outofbound   : OutOfBound::TooOld,
2143                                                        component    : Component::RB(30),
2144                                                        n_paged      : 0,
2145                                                        triggered    : None}),
2146                                    ("rb31_rate_zero" , TofAlert {
2147                                                        key          : "rb31_rate_zero", 
2148                                                        descr        : "RB31 is not triggering!", 
2149                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
2150                                                        config       : TofAlertConfig::new(),
2151                                                        variable     : Variable::TriggerRate,
2152                                                        outofbound   : OutOfBound::Zero,
2153                                                        component    : Component::RB(31),
2154                                                        n_paged      : 0,
2155                                                        triggered    : None}),
2156                                    ("rb31_temp" , TofAlert {
2157                                                        key          : "rb31_temp", 
2158                                                        descr        : "RB31 (FPGA) temperature is out of bounds!",
2159                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
2160                                                        config       : TofAlertConfig::new(),
2161                                                        variable     : Variable::FPGATemp,
2162                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
2163                                                        component    : Component::RB(31),
2164                                                        n_paged      : 0,
2165                                                        triggered    : None}),
2166                                    ("rb31_hk_too_old" , TofAlert {
2167                                                        key          : "rb31_hk_too_old", 
2168                                                        descr        : "RB31 RBMoniData is out-of-date!",
2169                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
2170                                                        config       : TofAlertConfig::new(),
2171                                                        variable     : Variable::MoniData,
2172                                                        outofbound   : OutOfBound::TooOld,
2173                                                        component    : Component::RB(31),
2174                                                        n_paged      : 0,
2175                                                        triggered    : None}),
2176                                    ("rb32_rate_zero" , TofAlert {
2177                                                        key          : "rb32_rate_zero", 
2178                                                        descr        : "RB32 is not triggering!", 
2179                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
2180                                                        config       : TofAlertConfig::new(),
2181                                                        variable     : Variable::TriggerRate,
2182                                                        outofbound   : OutOfBound::Zero,
2183                                                        component    : Component::RB(32),
2184                                                        n_paged      : 0,
2185                                                        triggered    : None}),
2186                                    ("rb32_temp" , TofAlert {
2187                                                        key          : "rb32_temp", 
2188                                                        descr        : "RB32 (FPGA) temperature is out of bounds!",
2189                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
2190                                                        config       : TofAlertConfig::new(),
2191                                                        variable     : Variable::FPGATemp,
2192                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
2193                                                        component    : Component::RB(32),
2194                                                        n_paged      : 0,
2195                                                        triggered    : None}),
2196                                    ("rb32_hk_too_old" , TofAlert {
2197                                                        key          : "rb32_hk_too_old", 
2198                                                        descr        : "RB30 RBMoniData is out-of-date!",
2199                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
2200                                                        config       : TofAlertConfig::new(),
2201                                                        variable     : Variable::MoniData,
2202                                                        outofbound   : OutOfBound::TooOld,
2203                                                        component    : Component::RB(32),
2204                                                        n_paged      : 0,
2205                                                        triggered    : None}),
2206                                    ("rb33_rate_zero" , TofAlert {
2207                                                        key          : "rb33_rate_zero", 
2208                                                        descr        : "RB33 is not triggering!", 
2209                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
2210                                                        config       : TofAlertConfig::new(),
2211                                                        variable     : Variable::TriggerRate,
2212                                                        outofbound   : OutOfBound::Zero,
2213                                                        component    : Component::RB(33),
2214                                                        n_paged      : 0,
2215                                                        triggered    : None}),
2216                                    ("rb33_temp" , TofAlert {
2217                                                        key          : "rb33_temp", 
2218                                                        descr        : "RB33 (FPGA) temperature is out of bounds!",
2219                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
2220                                                        config       : TofAlertConfig::new(),
2221                                                        variable     : Variable::FPGATemp,
2222                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
2223                                                        component    : Component::RB(33),
2224                                                        n_paged      : 0,
2225                                                        triggered    : None}),
2226                                    ("rb33_hk_too_old" , TofAlert {
2227                                                        key          : "rb33_hk_too_old", 
2228                                                        descr        : "RB33 RBMoniData is out-of-date!",
2229                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
2230                                                        config       : TofAlertConfig::new(),
2231                                                        variable     : Variable::MoniData,
2232                                                        outofbound   : OutOfBound::TooOld,
2233                                                        component    : Component::RB(33),
2234                                                        n_paged      : 0,
2235                                                        triggered    : None}),
2236                                    ("rb34_rate_zero" , TofAlert {
2237                                                        key          : "rb34_rate_zero", 
2238                                                        descr        : "RB34 is not triggering!", 
2239                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
2240                                                        config       : TofAlertConfig::new(),
2241                                                        variable     : Variable::TriggerRate,
2242                                                        outofbound   : OutOfBound::Zero,
2243                                                        component    : Component::RB(34),
2244                                                        n_paged      : 0,
2245                                                        triggered    : None}),
2246                                    ("rb34_temp" , TofAlert {
2247                                                        key          : "rb34_temp", 
2248                                                        descr        : "RB34 (FPGA) temperature is out of bounds!",
2249                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
2250                                                        config       : TofAlertConfig::new(),
2251                                                        variable     : Variable::FPGATemp,
2252                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
2253                                                        component    : Component::RB(34),
2254                                                        n_paged      : 0,
2255                                                        triggered    : None}),
2256                                    ("rb34_hk_too_old" , TofAlert {
2257                                                        key          : "rb34_hk_too_old", 
2258                                                        descr        : "RB34 RBMoniData is out-of-date!",
2259                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
2260                                                        config       : TofAlertConfig::new(),
2261                                                        variable     : Variable::MoniData,
2262                                                        outofbound   : OutOfBound::TooOld,
2263                                                        component    : Component::RB(34),
2264                                                        n_paged      : 0,
2265                                                        triggered    : None}),
2266                                    ("rb35_rate_zero" , TofAlert {
2267                                                        key          : "rb35_rate_zero", 
2268                                                        descr        : "RB35 is not triggering!", 
2269                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
2270                                                        config       : TofAlertConfig::new(),
2271                                                        variable     : Variable::TriggerRate,
2272                                                        outofbound   : OutOfBound::Zero,
2273                                                        component    : Component::RB(35),
2274                                                        n_paged      : 0,
2275                                                        triggered    : None}),
2276                                    ("rb35_temp" , TofAlert {
2277                                                        key          : "rb35_temp", 
2278                                                        descr        : "RB35 (FPGA) temperature is out of bounds!",
2279                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
2280                                                        config       : TofAlertConfig::new(),
2281                                                        variable     : Variable::FPGATemp,
2282                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
2283                                                        component    : Component::RB(35),
2284                                                        n_paged      : 0,
2285                                                        triggered    : None}),
2286                                    ("rb35_hk_too_old" , TofAlert {
2287                                                        key          : "rb35_hk_too_old", 
2288                                                        descr        : "RB35 RBMoniData is out-of-date!",
2289                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
2290                                                        config       : TofAlertConfig::new(),
2291                                                        variable     : Variable::MoniData,
2292                                                        outofbound   : OutOfBound::TooOld,
2293                                                        component    : Component::RB(35),
2294                                                        n_paged      : 0,
2295                                                        triggered    : None}),
2296                                    ("rb36_rate_zero" , TofAlert {
2297                                                        key          : "rb36_rate_zero", 
2298                                                        descr        : "RB36 is not triggering!", 
2299                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
2300                                                        config       : TofAlertConfig::new(),
2301                                                        variable     : Variable::TriggerRate,
2302                                                        outofbound   : OutOfBound::Zero,
2303                                                        component    : Component::RB(36),
2304                                                        n_paged      : 0,
2305                                                        triggered    : None}),
2306                                    ("rb36_temp" , TofAlert {
2307                                                        key          : "rb36_temp", 
2308                                                        descr        : "RB36 (FPGA) temperature is out of bounds!",
2309                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
2310                                                        config       : TofAlertConfig::new(),
2311                                                        variable     : Variable::FPGATemp,
2312                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
2313                                                        component    : Component::RB(36),
2314                                                        n_paged      : 0,
2315                                                        triggered    : None}),
2316                                    ("rb36_hk_too_old" , TofAlert {
2317                                                        key          : "rb36_hk_too_old", 
2318                                                        descr        : "RB36 RBMoniData is out-of-date!",
2319                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
2320                                                        config       : TofAlertConfig::new(),
2321                                                        variable     : Variable::MoniData,
2322                                                        outofbound   : OutOfBound::TooOld,
2323                                                        component    : Component::RB(36),
2324                                                        n_paged      : 0,
2325                                                        triggered    : None}),
2326                                    ("rb39_rate_zero" , TofAlert {
2327                                                        key          : "rb39_rate_zero", 
2328                                                        descr        : "RB39 is not triggering!", 
2329                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
2330                                                        config       : TofAlertConfig::new(),
2331                                                        variable     : Variable::TriggerRate,
2332                                                        outofbound   : OutOfBound::Zero,
2333                                                        component    : Component::RB(39),
2334                                                        n_paged      : 0,
2335                                                        triggered    : None}),
2336                                    ("rb39_temp" , TofAlert {
2337                                                        key          : "rb39_temp", 
2338                                                        descr        : "RB39 (FPGA) temperature is out of bounds!",
2339                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
2340                                                        config       : TofAlertConfig::new(),
2341                                                        variable     : Variable::FPGATemp,
2342                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
2343                                                        component    : Component::RB(39),
2344                                                        n_paged      : 0,
2345                                                        triggered    : None}),
2346                                    ("rb39_hk_too_old" , TofAlert {
2347                                                        key          : "rb39_hk_too_old", 
2348                                                        descr        : "RB39 RBMoniData is out-of-date!",
2349                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
2350                                                        config       : TofAlertConfig::new(),
2351                                                        variable     : Variable::MoniData,
2352                                                        outofbound   : OutOfBound::TooOld,
2353                                                        component    : Component::RB(39),
2354                                                        n_paged      : 0,
2355                                                        triggered    : None}),
2356                                ///////////////////////////////////////////////////////////
2357                                    ("rb40_rate_zero" , TofAlert {
2358                                                        key          : "rb40_rate_zero", 
2359                                                        descr        : "RB40 is not triggering!", 
2360                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
2361                                                        config       : TofAlertConfig::new(),
2362                                                        variable     : Variable::TriggerRate,
2363                                                        outofbound   : OutOfBound::Zero,
2364                                                        component    : Component::RB(40),
2365                                                        n_paged      : 0,
2366                                                        triggered    : None}),
2367                                    ("rb40_temp" , TofAlert {
2368                                                        key          : "rb40_temp", 
2369                                                        descr        : "RB40 (FPGA) temperature is out of bounds!",
2370                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
2371                                                        config       : TofAlertConfig::new(),
2372                                                        variable     : Variable::FPGATemp,
2373                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
2374                                                        component    : Component::RB(40),
2375                                                        n_paged      : 0,
2376                                                        triggered    : None}),
2377                                    ("rb40_hk_too_old" , TofAlert {
2378                                                        key          : "rb40_hk_too_old", 
2379                                                        descr        : "RB40 RBMoniData is out-of-date!",
2380                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
2381                                                        config       : TofAlertConfig::new(),
2382                                                        variable     : Variable::MoniData,
2383                                                        outofbound   : OutOfBound::TooOld,
2384                                                        component    : Component::RB(40),
2385                                                        n_paged      : 0,
2386                                                        triggered    : None}),
2387                                    ("rb41_rate_zero" , TofAlert {
2388                                                        key          : "rb41_rate_zero", 
2389                                                        descr        : "RB41 is not triggering!", 
2390                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
2391                                                        config       : TofAlertConfig::new(),
2392                                                        variable     : Variable::TriggerRate,
2393                                                        outofbound   : OutOfBound::Zero,
2394                                                        component    : Component::RB(41),
2395                                                        n_paged      : 0,
2396                                                        triggered    : None}),
2397                                    ("rb41_temp" , TofAlert {
2398                                                        key          : "rb41_temp", 
2399                                                        descr        : "RB41 (FPGA) temperature is out of bounds!",
2400                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
2401                                                        config       : TofAlertConfig::new(),
2402                                                        variable     : Variable::FPGATemp,
2403                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
2404                                                        component    : Component::RB(41),
2405                                                        n_paged      : 0,
2406                                                        triggered    : None}),
2407                                    ("rb41_hk_too_old" , TofAlert {
2408                                                        key          : "rb41_hk_too_old", 
2409                                                        descr        : "RB41 RBMoniData is out-of-date!",
2410                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
2411                                                        config       : TofAlertConfig::new(),
2412                                                        variable     : Variable::MoniData,
2413                                                        outofbound   : OutOfBound::TooOld,
2414                                                        component    : Component::RB(41),
2415                                                        n_paged      : 0,
2416                                                        triggered    : None}),
2417                                    ("rb42_rate_zero" , TofAlert {
2418                                                        key          : "rb42_rate_zero", 
2419                                                        descr        : "RB42 is not triggering!", 
2420                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
2421                                                        config       : TofAlertConfig::new(),
2422                                                        variable     : Variable::TriggerRate,
2423                                                        outofbound   : OutOfBound::Zero,
2424                                                        component    : Component::RB(42),
2425                                                        n_paged      : 0,
2426                                                        triggered    : None}),
2427                                    ("rb42_temp" , TofAlert {
2428                                                        key          : "rb42_temp", 
2429                                                        descr        : "RB42 (FPGA) temperature is out of bounds!",
2430                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
2431                                                        config       : TofAlertConfig::new(),
2432                                                        variable     : Variable::FPGATemp,
2433                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
2434                                                        component    : Component::RB(42),
2435                                                        n_paged      : 0,
2436                                                        triggered    : None}),
2437                                    ("rb42_hk_too_old" , TofAlert {
2438                                                        key          : "rb42_hk_too_old", 
2439                                                        descr        : "RB42 RBMoniData is out-of-date!",
2440                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
2441                                                        config       : TofAlertConfig::new(),
2442                                                        variable     : Variable::MoniData,
2443                                                        outofbound   : OutOfBound::TooOld,
2444                                                        component    : Component::RB(42),
2445                                                        n_paged      : 0,
2446                                                        triggered    : None}),
2447                                    ("rb44_rate_zero" , TofAlert {
2448                                                        key          : "rb44_rate_zero", 
2449                                                        descr        : "RB44 is not triggering!", 
2450                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
2451                                                        config       : TofAlertConfig::new(),
2452                                                        variable     : Variable::TriggerRate,
2453                                                        outofbound   : OutOfBound::Zero,
2454                                                        component    : Component::RB(44),
2455                                                        n_paged      : 0,
2456                                                        triggered    : None}),
2457                                    ("rb44_temp" , TofAlert {
2458                                                        key          : "rb44_temp", 
2459                                                        descr        : "RB44 (FPGA) temperature is out of bounds!",
2460                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
2461                                                        config       : TofAlertConfig::new(),
2462                                                        variable     : Variable::FPGATemp,
2463                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
2464                                                        component    : Component::RB(44),
2465                                                        n_paged      : 0,
2466                                                        triggered    : None}),
2467                                    ("rb44_hk_too_old" , TofAlert {
2468                                                        key          : "rb44_hk_too_old", 
2469                                                        descr        : "RB44 RBMoniData is out-of-date!",
2470                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
2471                                                        config       : TofAlertConfig::new(),
2472                                                        variable     : Variable::MoniData,
2473                                                        outofbound   : OutOfBound::TooOld,
2474                                                        component    : Component::RB(44),
2475                                                        n_paged      : 0,
2476                                                        triggered    : None}),
2477                                    ("rb46_rate_zero" , TofAlert {
2478                                                        key          : "rb46_rate_zero", 
2479                                                        descr        : "RB46 is not triggering!", 
2480                                                        whattodo     : vec!["If we are getting HK data, but the RAT is not triggering, this indicated an issue with either the RAT (the other RB might be down as well or the MTB", "If the other RB in the RAT is working, try soft rebooting the MTB", "Hard reboot the MTB"],
2481                                                        config       : TofAlertConfig::new(),
2482                                                        variable     : Variable::TriggerRate,
2483                                                        outofbound   : OutOfBound::Zero,
2484                                                        component    : Component::RB(46),
2485                                                        n_paged      : 0,
2486                                                        triggered    : None}),
2487                                    ("rb46_temp" , TofAlert {
2488                                                        key          : "rb46_temp", 
2489                                                        descr        : "RB46 (FPGA) temperature is out of bounds!",
2490                                                        whattodo     : vec!["This is critical!" ,  "Check with everybody and then likeliy switch off the RAT"],
2491                                                        config       : TofAlertConfig::new(),
2492                                                        variable     : Variable::FPGATemp,
2493                                                        outofbound   : OutOfBound::TooLowOrTooHigh,
2494                                                        component    : Component::RB(46),
2495                                                        n_paged      : 0,
2496                                                        triggered    : None}),
2497                                    ("rb46_hk_too_old" , TofAlert {
2498                                                        key          : "rb46_hk_too_old", 
2499                                                        descr        : "RB46 RBMoniData is out-of-date!",
2500                                                        whattodo     : vec!["Maybe the RB is down?" ,  "Telemetry issues?", "Have an eye on it and take notes"],
2501                                                        config       : TofAlertConfig::new(),
2502                                                        variable     : Variable::MoniData,
2503                                                        outofbound   : OutOfBound::TooOld,
2504                                                        component    : Component::RB(46),
2505                                                        n_paged      : 0,
2506                                                        triggered    : None}),
2507
2508  ]);
2509   // rb ids
2510   // 1,2,3,4,5,6,7,8,9,11,13,14,15,16,17,18,19,
2511   // 20,21,22,23,24,25,26,27,28,29,
2512   // 30,31,32,33,34,35,36,
2513   // 39,40,41,44,46
2514
2515//  TofAlert {
2516//    key          : "rb_rate_zero",
2517//    descr        : "RB1 is not triggering!",
2518//    whattodo     : vec!["1) Run restart", "2) Restart liftof-rb", "3) Soft reboot RB", "4) Soft reboot MTB", "5) Hard reboot RAT",  "6) Hard reboot MTB"],
2519//    config       : TofAlertConfig::new(),
2520//    variable     : Variable::TriggerRate,
2521//    outofbound   : OutOfBound::Zero,
2522//    component    : Component::RB(rb_id),
2523//    acknowledged : false,
2524//    created      : None,
2525//  }
2526  //let mut alerts = HashMap::<&str, TofAlert>::new();
2527  for k in manifest.keys() {
2528    println!("{}",k);
2529    alerts.get_mut(k).unwrap().config = manifest.get(k).unwrap().clone();
2530  }
2531  alerts
2532
2533}
2534
2535#[test]
2536fn write_alert_manifest() {
2537  let settings = TofAlertManifest::new();
2538  //println!("{}", settings);
2539  settings.to_toml(String::from("alert-manifest-test.toml"));
2540}
2541
2542#[test]
2543fn read_alert_manifest() {
2544  let _settings = TofAlertManifest::from_toml("alert-manifest-test.toml");
2545}