blackbox: Add black-box debug functionality

On any panic, we dump out the last ~30 seconds of IVSENSE data along
with the starting state and panic reason.

Also add a feature to panic if the gain reduces too much. This can be
used to try to catch badness.

Signed-off-by: Hector Martin <marcan@marcan.st>

Cargo.lock

@@ -24,6 +24,21 @@ dependencies = [
  "pkg-config",
 ]
 
+[[package]]
+name = "android-tzdata"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0"
+
+[[package]]
+name = "android_system_properties"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "atty"
 version = "0.2.14"
@@ -35,6 +50,12 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "autocfg"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
+
 [[package]]
 name = "bitflags"
 version = "1.3.2"
@@ -47,6 +68,12 @@ version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635"
 
+[[package]]
+name = "bumpalo"
+version = "3.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec"
+
 [[package]]
 name = "cc"
 version = "1.0.79"
@@ -59,6 +86,20 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
+[[package]]
+name = "chrono"
+version = "0.4.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38"
+dependencies = [
+ "android-tzdata",
+ "iana-time-zone",
+ "js-sys",
+ "num-traits",
+ "wasm-bindgen",
+ "windows-targets 0.48.5",
+]
+
 [[package]]
 name = "clap"
 version = "4.1.6"
@@ -94,7 +135,7 @@ dependencies = [
  "proc-macro-error",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.108",
 ]
 
 [[package]]
@@ -123,6 +164,12 @@ version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f7201ee416d124d589a820111ba755930df8b75855321a9a1b87312a0597ec8f"
 
+[[package]]
+name = "core-foundation-sys"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa"
+
 [[package]]
 name = "errno"
 version = "0.2.8"
@@ -165,6 +212,29 @@ version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286"
 
+[[package]]
+name = "iana-time-zone"
+version = "0.1.58"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20"
+dependencies = [
+ "android_system_properties",
+ "core-foundation-sys",
+ "iana-time-zone-haiku",
+ "js-sys",
+ "wasm-bindgen",
+ "windows-core",
+]
+
+[[package]]
+name = "iana-time-zone-haiku"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
+dependencies = [
+ "cc",
+]
+
 [[package]]
 name = "io-lifetimes"
 version = "1.0.5"
@@ -193,6 +263,21 @@ version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440"
 
+[[package]]
+name = "js-sys"
+version = "0.3.64"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a"
+dependencies = [
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "json"
+version = "0.12.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "078e285eafdfb6c4b434e0d31e8cfcb5115b651496faca5749b88fafd4f23bfd"
+
 [[package]]
 name = "lazy_static"
 version = "1.4.0"
@@ -231,6 +316,15 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "num-traits"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c"
+dependencies = [
+ "autocfg",
+]
+
 [[package]]
 name = "num_threads"
 version = "0.1.6"
@@ -267,7 +361,7 @@ dependencies = [
  "proc-macro-error-attr",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.108",
  "version_check",
 ]
 
@@ -284,18 +378,18 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.51"
+version = "1.0.69"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6"
+checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da"
 dependencies = [
  "unicode-ident",
 ]
 
 [[package]]
 name = "quote"
-version = "1.0.23"
+version = "1.0.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b"
+checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"
 dependencies = [
  "proc-macro2",
 ]
@@ -338,9 +432,11 @@ name = "speakersafetyd"
 version = "0.1.3"
 dependencies = [
  "alsa",
+ "chrono",
  "clap",
  "clap-verbosity-flag",
  "configparser",
+ "json",
  "log",
  "simple_logger",
 ]
@@ -362,6 +458,17 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "syn"
+version = "2.0.38"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
 [[package]]
 name = "termcolor"
 version = "1.2.0"
@@ -412,6 +519,60 @@ version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
 
+[[package]]
+name = "wasm-bindgen"
+version = "0.2.87"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342"
+dependencies = [
+ "cfg-if",
+ "wasm-bindgen-macro",
+]
+
+[[package]]
+name = "wasm-bindgen-backend"
+version = "0.2.87"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd"
+dependencies = [
+ "bumpalo",
+ "log",
+ "once_cell",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.38",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-macro"
+version = "0.2.87"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d"
+dependencies = [
+ "quote",
+ "wasm-bindgen-macro-support",
+]
+
+[[package]]
+name = "wasm-bindgen-macro-support"
+version = "0.2.87"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.38",
+ "wasm-bindgen-backend",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-shared"
+version = "0.2.87"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1"
+
 [[package]]
 name = "winapi"
 version = "0.3.9"
@@ -443,13 +604,22 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
+[[package]]
+name = "windows-core"
+version = "0.51.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64"
+dependencies = [
+ "windows-targets 0.48.5",
+]
+
 [[package]]
 name = "windows-sys"
 version = "0.45.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"
 dependencies = [
- "windows-targets",
+ "windows-targets 0.42.1",
 ]
 
 [[package]]
@@ -458,13 +628,28 @@ version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7"
 dependencies = [
- "windows_aarch64_gnullvm",
+ "windows_aarch64_gnullvm 0.42.1",
- "windows_aarch64_msvc",
+ "windows_aarch64_msvc 0.42.1",
- "windows_i686_gnu",
+ "windows_i686_gnu 0.42.1",
- "windows_i686_msvc",
+ "windows_i686_msvc 0.42.1",
- "windows_x86_64_gnu",
+ "windows_x86_64_gnu 0.42.1",
- "windows_x86_64_gnullvm",
+ "windows_x86_64_gnullvm 0.42.1",
- "windows_x86_64_msvc",
+ "windows_x86_64_msvc 0.42.1",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"
+dependencies = [
+ "windows_aarch64_gnullvm 0.48.5",
+ "windows_aarch64_msvc 0.48.5",
+ "windows_i686_gnu 0.48.5",
+ "windows_i686_msvc 0.48.5",
+ "windows_x86_64_gnu 0.48.5",
+ "windows_x86_64_gnullvm 0.48.5",
+ "windows_x86_64_msvc 0.48.5",
 ]
 
 [[package]]
@@ -473,38 +658,80 @@ version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608"
 
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
+
 [[package]]
 name = "windows_aarch64_msvc"
 version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7"
 
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
+
 [[package]]
 name = "windows_i686_gnu"
 version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640"
 
+[[package]]
+name = "windows_i686_gnu"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
+
 [[package]]
 name = "windows_i686_msvc"
 version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605"
 
+[[package]]
+name = "windows_i686_msvc"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
+
 [[package]]
 name = "windows_x86_64_gnu"
 version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45"
 
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
+
 [[package]]
 name = "windows_x86_64_gnullvm"
 version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463"
 
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
+
 [[package]]
 name = "windows_x86_64_msvc"
 version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"

Cargo.toml

@@ -13,3 +13,5 @@ clap = { version = "4.1.6", features=["derive"] }
 log = "0.4.17"
 clap-verbosity-flag = "2.0.0"
 simple_logger = "1.16.0"
+chrono = "0.4.31"
+json = "0.12.4"

src/blackbox.rs

@@ -0,0 +1,111 @@
+use crate::types::SpeakerState;
+use chrono;
+use log::warn;
+use std::fs::File;
+use std::io;
+use std::io::Write;
+use std::path::Path;
+use std::slice;
+
+use json::object;
+
+struct Block {
+    sample_rate: i32,
+    state: Vec<Vec<SpeakerState>>,
+    data: Vec<i16>,
+}
+
+pub struct Blackbox {
+    machine: String,
+    globals: crate::types::Globals,
+    path: Box<Path>,
+    blocks: Vec<Block>,
+}
+
+/// Maximum number of blocks in the ring buffer (around 30 seconds at 4096/48000)
+const MAX_BLOCKS: usize = 330;
+
+impl Blackbox {
+    pub fn new(machine: &str, path: &Path, globals: &crate::types::Globals) -> Blackbox {
+        Blackbox {
+            machine: machine.into(),
+            globals: globals.clone(),
+            path: path.into(),
+            blocks: Vec::new(),
+        }
+    }
+
+    pub fn reset(&mut self) {
+        self.blocks.clear();
+    }
+
+    pub fn push(&mut self, sample_rate: i32, data: Vec<i16>, state: Vec<Vec<SpeakerState>>) {
+        while self.blocks.len() >= MAX_BLOCKS {
+            self.blocks.remove(0);
+        }
+        self.blocks.push(Block {
+            sample_rate,
+            state,
+            data,
+        })
+    }
+
+    pub fn preserve(&mut self, reason: String) -> io::Result<()> {
+        if self.blocks.is_empty() {
+            warn!("Blackbox is empty, nothing to save");
+            return Ok(());
+        }
+
+        let now = chrono::Local::now().to_rfc3339();
+        let meta_name = self.path.join(now.clone() + ".meta");
+        let data_name = self.path.join(now.clone() + ".raw");
+
+        warn!("Preserving blackbox {}", now);
+
+        let mut metafd = File::create(meta_name)?;
+        let mut datafd = File::create(data_name)?;
+
+        for blk in self.blocks.iter() {
+            // meh unsafe
+            let slice_u8: &[u8] = unsafe {
+                slice::from_raw_parts(
+                    blk.data.as_ptr() as *const u8,
+                    blk.data.len() * std::mem::size_of::<u16>(),
+                )
+            };
+            datafd.write_all(slice_u8)?;
+        }
+
+        let mut meta = object! {
+            message: reason,
+            machine: self.machine.clone(),
+            sample_rate: self.blocks[0].sample_rate,
+            channels: self.globals.channels,
+            t_ambient: self.globals.t_ambient,
+            t_safe_max: self.globals.t_safe_max,
+            t_hysteresis: self.globals.t_hysteresis,
+            state: null
+        };
+
+        let mut state = json::JsonValue::new_array();
+
+        for group in self.blocks[0].state.iter() {
+            for speaker in group.iter() {
+                let _ = state.push(object! {
+                    t_coil: speaker.t_coil,
+                    t_magnet: speaker.t_magnet,
+                    t_coil_hyst: speaker.t_coil_hyst,
+                    t_magnet_hyst: speaker.t_magnet_hyst,
+                    min_gain: speaker.min_gain,
+                    gain: speaker.gain,
+                });
+            }
+        }
+
+        meta["state"] = state;
+
+        metafd.write_all(meta.dump().as_bytes())?;
+
+        Ok(())
+    }
+}

src/main.rs

@@ -9,17 +9,19 @@
 */
 use std::collections::BTreeMap;
 use std::fs;
+use std::panic::{catch_unwind, resume_unwind, AssertUnwindSafe};
 use std::path::{Path, PathBuf};
 use std::time::Instant;
 
+use alsa::nix::errno::Errno;
 use clap::Parser;
 use clap_verbosity_flag::{InfoLevel, Verbosity};
 use configparser::ini::Ini;
 use log;
 use log::{debug, info, warn};
 use simple_logger::SimpleLogger;
-use alsa::nix::errno::Errno;
 
+mod blackbox;
 mod helpers;
 mod types;
 
@@ -40,6 +42,14 @@ struct Options {
     /// Increase the log level
     #[command(flatten)]
     verbose: Verbosity<InfoLevel>,
+
+    /// Path to the blackbox dump directory
+    #[arg(short, long)]
+    blackbox_path: Option<PathBuf>,
+
+    /// Maximum gain reduction before panicing (for debugging)
+    #[arg(short, long)]
+    max_reduction: Option<f32>,
 }
 
 fn get_machine() -> String {
@@ -113,6 +123,13 @@ fn main() {
 
     let globals = types::Globals::parse(&cfg);
 
+    let mut blackbox = args.blackbox_path.map(|p| {
+        info!("Enabling blackbox, path: {:?}", p);
+        blackbox::Blackbox::new(&machine, &p, &globals)
+    });
+
+    let mut blackbox_ref = AssertUnwindSafe(&mut blackbox);
+    let result = catch_unwind(move || {
         let speaker_names = get_speakers(&cfg);
         let speaker_count = speaker_names.len();
         info!("Found {} speakers", speaker_count);
@@ -165,10 +182,8 @@ fn main() {
 
         let pcm_name = format!("{},{}", device, globals.visense_pcm);
         // Set up PCM to buffer in V/ISENSE
-    let pcm: alsa::pcm::PCM = helpers::open_pcm(&pcm_name, globals.channels.try_into().unwrap(), 0);
+        let pcm: alsa::pcm::PCM =
-    let mut buf = Vec::new();
+            helpers::open_pcm(&pcm_name, globals.channels.try_into().unwrap(), 0);
-    buf.resize(globals.period * globals.channels, 0i16);
-
         let io = pcm.io_i16().unwrap();
 
         let mut sample_rate_elem = types::Elem::new(
@@ -200,9 +215,15 @@ fn main() {
 
         let mut last_update = Instant::now();
 
+        let mut buf = Vec::new();
+        buf.resize(globals.period * globals.channels, 0i16);
+
+        let mut once_nominal = false;
+
         loop {
             // Block while we're reading into the buffer
-        io.readi(&mut buf).or_else(|e| {
+            io.readi(&mut buf)
+                .or_else(|e| {
                     if e.errno() == Errno::ESTRPIPE {
                         // Resume handling
                         loop {
@@ -211,12 +232,14 @@ fn main() {
                                 Err(e) if e.errno() == Errno::EAGAIN => continue,
                                 Err(e) => break Err(e),
                             }
-                }.unwrap();
+                        }
+                        .unwrap();
                         io.readi(&mut buf)
                     } else {
                         Err(e)
                     }
-        }).unwrap();
+                })
+                .unwrap();
 
             let cur_sample_rate = sample_rate_elem.read_int(&ctl);
 
@@ -224,6 +247,7 @@ fn main() {
                 if cur_sample_rate != sample_rate {
                     sample_rate = cur_sample_rate;
                     info!("Sample rate: {}", sample_rate);
+                    blackbox_ref.as_mut().map(|bb| bb.reset());
                 }
             }
 
@@ -243,10 +267,20 @@ fn main() {
                 for (_, group) in groups.iter_mut() {
                     group.speakers.iter_mut().for_each(|s| s.skip_model(skip));
                 }
+                blackbox_ref.as_mut().map(|bb| bb.reset());
             }
 
             last_update = now;
 
+            if let Some(bb) = blackbox_ref.as_mut() {
+                let gstates = groups
+                    .iter()
+                    .map(|g| g.1.speakers.iter().map(|s| s.s.clone()).collect())
+                    .collect();
+                bb.push(sample_rate, buf.clone(), gstates);
+            }
+
+            let mut all_nominal = true;
             for (idx, group) in groups.iter_mut() {
                 let gain = group
                     .speakers
@@ -263,8 +297,40 @@ fn main() {
                     group.speakers.iter_mut().for_each(|s| s.update(&ctl, gain));
                     group.gain = gain;
                 }
+                if gain != 0. {
+                    all_nominal = false;
+                }
+                if let Some(max_reduction) = args.max_reduction {
+                    if once_nominal && gain < -max_reduction {
+                        panic!("Gain reduction exceeded threshold");
+                    }
+                }
+            }
+
+            if all_nominal {
+                once_nominal = true;
             }
 
             unlock_elem.write_int(&ctl, UNLOCK_MAGIC);
         }
+    });
+    if let Err(e) = result {
+        warn!("Panic!");
+
+        let mut reason: String = "Unknown panic".into();
+
+        if let Some(s) = e.downcast_ref::<&'static str>() {
+            reason = (*s).into();
+        } else if let Some(s) = e.downcast_ref::<String>() {
+            reason = s.clone();
+        }
+
+        blackbox.as_mut().map(|bb| {
+            if bb.preserve(reason).is_err() {
+                warn!("Failed to write blackbox");
+            }
+        });
+
+        resume_unwind(e);
+    }
 }

src/types.rs

@@ -202,16 +202,16 @@ impl Globals {
 
     Borrows the handle to the control interface to do calculations.
 */
-#[derive(Debug, Default)]
+#[derive(Debug, Default, Copy, Clone)]
 pub struct SpeakerState {
-    t_coil: f64,
+    pub t_coil: f64,
-    t_magnet: f64,
+    pub t_magnet: f64,
 
-    t_coil_hyst: f32,
+    pub t_coil_hyst: f32,
-    t_magnet_hyst: f32,
+    pub t_magnet_hyst: f32,
 
-    min_gain: f32,
+    pub min_gain: f32,
-    gain: f32,
+    pub gain: f32,
 }
 
 pub struct Speaker {
@@ -231,7 +231,7 @@ pub struct Speaker {
     vs_chan: usize,
 
     g: Globals,
-    s: SpeakerState,
+    pub s: SpeakerState,
 }
 
 impl Speaker {