blackbox: Add black-box debug functionality

On any panic, we dump out the last ~30 seconds of IVSENSE data along
with the starting state and panic reason.

Also add a feature to panic if the gain reduces too much. This can be
used to try to catch badness.

Signed-off-by: Hector Martin <marcan@marcan.st>
This commit is contained in:
Hector Martin 2023-10-23 02:20:39 +09:00
parent a476f7daf1
commit 7fbb53df51
5 changed files with 571 additions and 165 deletions

255
Cargo.lock generated
View file

@ -24,6 +24,21 @@ dependencies = [
"pkg-config",
]
[[package]]
name = "android-tzdata"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0"
[[package]]
name = "android_system_properties"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
dependencies = [
"libc",
]
[[package]]
name = "atty"
version = "0.2.14"
@ -35,6 +50,12 @@ dependencies = [
"winapi",
]
[[package]]
name = "autocfg"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]]
name = "bitflags"
version = "1.3.2"
@ -47,6 +68,12 @@ version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635"
[[package]]
name = "bumpalo"
version = "3.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec"
[[package]]
name = "cc"
version = "1.0.79"
@ -59,6 +86,20 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "chrono"
version = "0.4.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38"
dependencies = [
"android-tzdata",
"iana-time-zone",
"js-sys",
"num-traits",
"wasm-bindgen",
"windows-targets 0.48.5",
]
[[package]]
name = "clap"
version = "4.1.6"
@ -94,7 +135,7 @@ dependencies = [
"proc-macro-error",
"proc-macro2",
"quote",
"syn",
"syn 1.0.108",
]
[[package]]
@ -123,6 +164,12 @@ version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7201ee416d124d589a820111ba755930df8b75855321a9a1b87312a0597ec8f"
[[package]]
name = "core-foundation-sys"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa"
[[package]]
name = "errno"
version = "0.2.8"
@ -165,6 +212,29 @@ version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286"
[[package]]
name = "iana-time-zone"
version = "0.1.58"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20"
dependencies = [
"android_system_properties",
"core-foundation-sys",
"iana-time-zone-haiku",
"js-sys",
"wasm-bindgen",
"windows-core",
]
[[package]]
name = "iana-time-zone-haiku"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
dependencies = [
"cc",
]
[[package]]
name = "io-lifetimes"
version = "1.0.5"
@ -193,6 +263,21 @@ version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440"
[[package]]
name = "js-sys"
version = "0.3.64"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a"
dependencies = [
"wasm-bindgen",
]
[[package]]
name = "json"
version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "078e285eafdfb6c4b434e0d31e8cfcb5115b651496faca5749b88fafd4f23bfd"
[[package]]
name = "lazy_static"
version = "1.4.0"
@ -231,6 +316,15 @@ dependencies = [
"libc",
]
[[package]]
name = "num-traits"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c"
dependencies = [
"autocfg",
]
[[package]]
name = "num_threads"
version = "0.1.6"
@ -267,7 +361,7 @@ dependencies = [
"proc-macro-error-attr",
"proc-macro2",
"quote",
"syn",
"syn 1.0.108",
"version_check",
]
@ -284,18 +378,18 @@ dependencies = [
[[package]]
name = "proc-macro2"
version = "1.0.51"
version = "1.0.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6"
checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da"
dependencies = [
"unicode-ident",
]
[[package]]
name = "quote"
version = "1.0.23"
version = "1.0.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b"
checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"
dependencies = [
"proc-macro2",
]
@ -338,9 +432,11 @@ name = "speakersafetyd"
version = "0.1.3"
dependencies = [
"alsa",
"chrono",
"clap",
"clap-verbosity-flag",
"configparser",
"json",
"log",
"simple_logger",
]
@ -362,6 +458,17 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "syn"
version = "2.0.38"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "termcolor"
version = "1.2.0"
@ -412,6 +519,60 @@ version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
[[package]]
name = "wasm-bindgen"
version = "0.2.87"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342"
dependencies = [
"cfg-if",
"wasm-bindgen-macro",
]
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.87"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd"
dependencies = [
"bumpalo",
"log",
"once_cell",
"proc-macro2",
"quote",
"syn 2.0.38",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.87"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
]
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.87"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.38",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.87"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1"
[[package]]
name = "winapi"
version = "0.3.9"
@ -443,13 +604,22 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows-core"
version = "0.51.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64"
dependencies = [
"windows-targets 0.48.5",
]
[[package]]
name = "windows-sys"
version = "0.45.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"
dependencies = [
"windows-targets",
"windows-targets 0.42.1",
]
[[package]]
@ -458,13 +628,28 @@ version = "0.42.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7"
dependencies = [
"windows_aarch64_gnullvm",
"windows_aarch64_msvc",
"windows_i686_gnu",
"windows_i686_msvc",
"windows_x86_64_gnu",
"windows_x86_64_gnullvm",
"windows_x86_64_msvc",
"windows_aarch64_gnullvm 0.42.1",
"windows_aarch64_msvc 0.42.1",
"windows_i686_gnu 0.42.1",
"windows_i686_msvc 0.42.1",
"windows_x86_64_gnu 0.42.1",
"windows_x86_64_gnullvm 0.42.1",
"windows_x86_64_msvc 0.42.1",
]
[[package]]
name = "windows-targets"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"
dependencies = [
"windows_aarch64_gnullvm 0.48.5",
"windows_aarch64_msvc 0.48.5",
"windows_i686_gnu 0.48.5",
"windows_i686_msvc 0.48.5",
"windows_x86_64_gnu 0.48.5",
"windows_x86_64_gnullvm 0.48.5",
"windows_x86_64_msvc 0.48.5",
]
[[package]]
@ -473,38 +658,80 @@ version = "0.42.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608"
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
[[package]]
name = "windows_aarch64_msvc"
version = "0.42.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7"
[[package]]
name = "windows_aarch64_msvc"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
[[package]]
name = "windows_i686_gnu"
version = "0.42.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640"
[[package]]
name = "windows_i686_gnu"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
[[package]]
name = "windows_i686_msvc"
version = "0.42.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605"
[[package]]
name = "windows_i686_msvc"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
[[package]]
name = "windows_x86_64_gnu"
version = "0.42.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45"
[[package]]
name = "windows_x86_64_gnu"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.42.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
[[package]]
name = "windows_x86_64_msvc"
version = "0.42.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd"
[[package]]
name = "windows_x86_64_msvc"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"

View file

@ -13,3 +13,5 @@ clap = { version = "4.1.6", features=["derive"] }
log = "0.4.17"
clap-verbosity-flag = "2.0.0"
simple_logger = "1.16.0"
chrono = "0.4.31"
json = "0.12.4"

111
src/blackbox.rs Normal file
View file

@ -0,0 +1,111 @@
use crate::types::SpeakerState;
use chrono;
use log::warn;
use std::fs::File;
use std::io;
use std::io::Write;
use std::path::Path;
use std::slice;
use json::object;
struct Block {
sample_rate: i32,
state: Vec<Vec<SpeakerState>>,
data: Vec<i16>,
}
pub struct Blackbox {
machine: String,
globals: crate::types::Globals,
path: Box<Path>,
blocks: Vec<Block>,
}
/// Maximum number of blocks in the ring buffer (around 30 seconds at 4096/48000)
const MAX_BLOCKS: usize = 330;
impl Blackbox {
pub fn new(machine: &str, path: &Path, globals: &crate::types::Globals) -> Blackbox {
Blackbox {
machine: machine.into(),
globals: globals.clone(),
path: path.into(),
blocks: Vec::new(),
}
}
pub fn reset(&mut self) {
self.blocks.clear();
}
pub fn push(&mut self, sample_rate: i32, data: Vec<i16>, state: Vec<Vec<SpeakerState>>) {
while self.blocks.len() >= MAX_BLOCKS {
self.blocks.remove(0);
}
self.blocks.push(Block {
sample_rate,
state,
data,
})
}
pub fn preserve(&mut self, reason: String) -> io::Result<()> {
if self.blocks.is_empty() {
warn!("Blackbox is empty, nothing to save");
return Ok(());
}
let now = chrono::Local::now().to_rfc3339();
let meta_name = self.path.join(now.clone() + ".meta");
let data_name = self.path.join(now.clone() + ".raw");
warn!("Preserving blackbox {}", now);
let mut metafd = File::create(meta_name)?;
let mut datafd = File::create(data_name)?;
for blk in self.blocks.iter() {
// meh unsafe
let slice_u8: &[u8] = unsafe {
slice::from_raw_parts(
blk.data.as_ptr() as *const u8,
blk.data.len() * std::mem::size_of::<u16>(),
)
};
datafd.write_all(slice_u8)?;
}
let mut meta = object! {
message: reason,
machine: self.machine.clone(),
sample_rate: self.blocks[0].sample_rate,
channels: self.globals.channels,
t_ambient: self.globals.t_ambient,
t_safe_max: self.globals.t_safe_max,
t_hysteresis: self.globals.t_hysteresis,
state: null
};
let mut state = json::JsonValue::new_array();
for group in self.blocks[0].state.iter() {
for speaker in group.iter() {
let _ = state.push(object! {
t_coil: speaker.t_coil,
t_magnet: speaker.t_magnet,
t_coil_hyst: speaker.t_coil_hyst,
t_magnet_hyst: speaker.t_magnet_hyst,
min_gain: speaker.min_gain,
gain: speaker.gain,
});
}
}
meta["state"] = state;
metafd.write_all(meta.dump().as_bytes())?;
Ok(())
}
}

View file

@ -9,17 +9,19 @@
*/
use std::collections::BTreeMap;
use std::fs;
use std::panic::{catch_unwind, resume_unwind, AssertUnwindSafe};
use std::path::{Path, PathBuf};
use std::time::Instant;
use alsa::nix::errno::Errno;
use clap::Parser;
use clap_verbosity_flag::{InfoLevel, Verbosity};
use configparser::ini::Ini;
use log;
use log::{debug, info, warn};
use simple_logger::SimpleLogger;
use alsa::nix::errno::Errno;
mod blackbox;
mod helpers;
mod types;
@ -40,6 +42,14 @@ struct Options {
/// Increase the log level
#[command(flatten)]
verbose: Verbosity<InfoLevel>,
/// Path to the blackbox dump directory
#[arg(short, long)]
blackbox_path: Option<PathBuf>,
/// Maximum gain reduction before panicing (for debugging)
#[arg(short, long)]
max_reduction: Option<f32>,
}
fn get_machine() -> String {
@ -113,6 +123,13 @@ fn main() {
let globals = types::Globals::parse(&cfg);
let mut blackbox = args.blackbox_path.map(|p| {
info!("Enabling blackbox, path: {:?}", p);
blackbox::Blackbox::new(&machine, &p, &globals)
});
let mut blackbox_ref = AssertUnwindSafe(&mut blackbox);
let result = catch_unwind(move || {
let speaker_names = get_speakers(&cfg);
let speaker_count = speaker_names.len();
info!("Found {} speakers", speaker_count);
@ -165,10 +182,8 @@ fn main() {
let pcm_name = format!("{},{}", device, globals.visense_pcm);
// Set up PCM to buffer in V/ISENSE
let pcm: alsa::pcm::PCM = helpers::open_pcm(&pcm_name, globals.channels.try_into().unwrap(), 0);
let mut buf = Vec::new();
buf.resize(globals.period * globals.channels, 0i16);
let pcm: alsa::pcm::PCM =
helpers::open_pcm(&pcm_name, globals.channels.try_into().unwrap(), 0);
let io = pcm.io_i16().unwrap();
let mut sample_rate_elem = types::Elem::new(
@ -200,9 +215,15 @@ fn main() {
let mut last_update = Instant::now();
let mut buf = Vec::new();
buf.resize(globals.period * globals.channels, 0i16);
let mut once_nominal = false;
loop {
// Block while we're reading into the buffer
io.readi(&mut buf).or_else(|e| {
io.readi(&mut buf)
.or_else(|e| {
if e.errno() == Errno::ESTRPIPE {
// Resume handling
loop {
@ -211,12 +232,14 @@ fn main() {
Err(e) if e.errno() == Errno::EAGAIN => continue,
Err(e) => break Err(e),
}
}.unwrap();
}
.unwrap();
io.readi(&mut buf)
} else {
Err(e)
}
}).unwrap();
})
.unwrap();
let cur_sample_rate = sample_rate_elem.read_int(&ctl);
@ -224,6 +247,7 @@ fn main() {
if cur_sample_rate != sample_rate {
sample_rate = cur_sample_rate;
info!("Sample rate: {}", sample_rate);
blackbox_ref.as_mut().map(|bb| bb.reset());
}
}
@ -243,10 +267,20 @@ fn main() {
for (_, group) in groups.iter_mut() {
group.speakers.iter_mut().for_each(|s| s.skip_model(skip));
}
blackbox_ref.as_mut().map(|bb| bb.reset());
}
last_update = now;
if let Some(bb) = blackbox_ref.as_mut() {
let gstates = groups
.iter()
.map(|g| g.1.speakers.iter().map(|s| s.s.clone()).collect())
.collect();
bb.push(sample_rate, buf.clone(), gstates);
}
let mut all_nominal = true;
for (idx, group) in groups.iter_mut() {
let gain = group
.speakers
@ -263,8 +297,40 @@ fn main() {
group.speakers.iter_mut().for_each(|s| s.update(&ctl, gain));
group.gain = gain;
}
if gain != 0. {
all_nominal = false;
}
if let Some(max_reduction) = args.max_reduction {
if once_nominal && gain < -max_reduction {
panic!("Gain reduction exceeded threshold");
}
}
}
if all_nominal {
once_nominal = true;
}
unlock_elem.write_int(&ctl, UNLOCK_MAGIC);
}
});
if let Err(e) = result {
warn!("Panic!");
let mut reason: String = "Unknown panic".into();
if let Some(s) = e.downcast_ref::<&'static str>() {
reason = (*s).into();
} else if let Some(s) = e.downcast_ref::<String>() {
reason = s.clone();
}
blackbox.as_mut().map(|bb| {
if bb.preserve(reason).is_err() {
warn!("Failed to write blackbox");
}
});
resume_unwind(e);
}
}

View file

@ -202,16 +202,16 @@ impl Globals {
Borrows the handle to the control interface to do calculations.
*/
#[derive(Debug, Default)]
#[derive(Debug, Default, Copy, Clone)]
pub struct SpeakerState {
t_coil: f64,
t_magnet: f64,
pub t_coil: f64,
pub t_magnet: f64,
t_coil_hyst: f32,
t_magnet_hyst: f32,
pub t_coil_hyst: f32,
pub t_magnet_hyst: f32,
min_gain: f32,
gain: f32,
pub min_gain: f32,
pub gain: f32,
}
pub struct Speaker {
@ -231,7 +231,7 @@ pub struct Speaker {
vs_chan: usize,
g: Globals,
s: SpeakerState,
pub s: SpeakerState,
}
impl Speaker {