From b4d14628bcdb7b705b6364377018106b629d2887 Mon Sep 17 00:00:00 2001 From: Dandaedre Date: Mon, 5 Oct 2026 01:53:10 +0200 Subject: [PATCH] test: add guided hardware validation --- .gitignore | 1 + Cargo.toml | 5 + README.md | 1 + docs/README.md | 2 + docs/hardware-validation.md | 74 +++++++ docs/testing.md | 3 + examples/validate-hardware.rs | 354 ++++++++++++++++++++++++++++++++++ scripts/test-hardware.sh | 14 ++ 8 files changed, 454 insertions(+) create mode 100644 docs/hardware-validation.md create mode 100644 examples/validate-hardware.rs create mode 100755 scripts/test-hardware.sh diff --git a/.gitignore b/.gitignore index d4f035b..d358677 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ config.toml /coverage/ /target-ci/ +/hardware-validation/ # rust_realease artifacts and build cache /dist/ diff --git a/Cargo.toml b/Cargo.toml index 9c2bcd1..0386a30 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,11 @@ name = "mock-tinysa-server" path = "examples/mock-tinysa-server.rs" required-features = ["mock"] +[[example]] +name = "validate-hardware" +path = "examples/validate-hardware.rs" +required-features = ["serial"] + [[test]] name = "mock_device" path = "tests/mock_device.rs" diff --git a/README.md b/README.md index 260a94d..698344e 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ before connecting an Ultra or Ultra+. - [Configuration](docs/configuration.md) — defaults, partial overrides, units, validation, and saved selections - [Release guide](docs/releases.md) — wrapper, target/artifact policy, token handling, and publication readiness - [Testing guide](docs/testing.md) — hardware-free regression tests and test boundaries +- [Hardware validation](docs/hardware-validation.md) — guided Basic and Ultra acceptance checks - [Project architecture](docs/architecture.md) — components, state, data flow, files, dependencies, and extension points - [tinySA field guide](docs/tinysa-field-guide.md) — models, RF architecture, diff --git a/docs/README.md b/docs/README.md index be6fd2e..0d67139 100644 --- a/docs/README.md +++ b/docs/README.md @@ -18,6 +18,8 @@ not need to reread the upstream wiki before changing the code. reproducible-build and supply-chain gates. 6. [Upstream source catalogue](sources.md) records the primary references and their scope. +7. [Hardware validation](hardware-validation.md) provides guided Basic and Ultra + checklists and the interactive validator command. ## Scope and authority diff --git a/docs/hardware-validation.md b/docs/hardware-validation.md new file mode 100644 index 0000000..69ee682 --- /dev/null +++ b/docs/hardware-validation.md @@ -0,0 +1,74 @@ +# tinySA hardware validation checklist + +This checklist qualifies one physical instrument and firmware build. A passing +mock or CI run is not hardware evidence. Run the validator once with a tinySA +Basic and once with a tinySA Ultra; keep their reports separate. + +## Before each run + +- [ ] Identify the rear-label model and record the firmware shown by the device. +- [ ] Disconnect every RF source. This test needs only the analyzer and a + data-capable USB cable. +- [ ] Close serial terminals and other tinySA software. Confirm the current user + can open the USB serial port without running the validator as root. +- [ ] If the front-panel configuration matters, save a preset. The validator + changes repeat, raw zero offset, input mode, sweep limits, and pause state. + +From the repository root, run exactly one command: + +```console +scripts/test-hardware.sh basic +``` + +For an Ultra, use: + +```console +scripts/test-hardware.sh ultra +``` + +The script asks when to unplug and plug the instrument so it can detect the new +serial port. If discovery is ambiguous, choose from the displayed ports. To skip +discovery, pass `--port /dev/ttyACM0` or `--port COM7`. It builds the validator, +runs two 101-point scans, validates the returned values, writes CSV files and an +identity report under ignored `hardware-validation/`, and asks whether the +front-panel sweep resumed. + +## Basic acceptance + +- [ ] The automatically selected port belongs to the Basic. +- [ ] Both `legacy` (240–300 MHz low input) and `direct` (350–400 MHz high input) + scans report `PASS` and create CSV files with 101 data rows plus a units header. +- [ ] The printed `version`/`info` text matches the connected Basic and its + recorded firmware. +- [ ] The live front-panel sweep resumes after the scans; answer `y` only after + checking it. +- [ ] The final `basic-report.txt` records both CSV paths and + `operator_confirmed=y` (or `yes`). + +## Ultra acceptance and limit + +- [ ] The automatically selected port belongs to the Ultra. +- [ ] Both `legacy` and `direct` scans report `PASS` at 240–300 MHz and create + CSV files with 101 data rows plus a units header. +- [ ] The printed `version`/`info` text identifies the exact connected Ultra + variant and firmware. +- [ ] The live front-panel sweep resumes after the scans; answer `y` only after + checking it. +- [ ] The final `ultra-report.txt` records both CSV paths and + `operator_confirmed=y` (or `yes`). + +A passing Ultra run validates only the shared low-input command path below +350 MHz, including `zero 128`, binary framing, CSV export, and best-effort +resume. It does not validate Ultra mode, harmonic mode, LNA behavior, ranges at +or above 350 MHz, level accuracy, or frequency accuracy. Those require +model-aware acquisition and a controlled reference source; do not infer them +from this communications test. + +## Failure record + +- [ ] Keep the generated report and any completed CSV; do not edit them. +- [ ] Record the command, host OS, selected port, exact model, firmware, and the + complete `FAIL:` line. +- [ ] If the sweep did not resume, disconnect any RF setup and use `resume`, + reload the saved preset, or power-cycle the instrument. +- [ ] Do not connect a transmitter merely to reproduce a communications failure. diff --git a/docs/testing.md b/docs/testing.md index 95e77c4..9862e08 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -43,6 +43,9 @@ Hardware validation still requires a known model/version, safe RF input, and a bounded capture of the real command/response sequence. In particular, verify frequency-axis policy, the legacy preparatory scan, zero-offset acceptance and resume behavior against the connected firmware before changing their defaults. +Use the guided [Basic and Ultra hardware checklist](hardware-validation.md) to +produce CSV and identity evidence without connecting an RF source. Its Ultra +case intentionally stays below 350 MHz and does not certify extended ranges. ## Library feature matrix and mock integration diff --git a/examples/validate-hardware.rs b/examples/validate-hardware.rs new file mode 100644 index 0000000..8c6f0c1 --- /dev/null +++ b/examples/validate-hardware.rs @@ -0,0 +1,354 @@ +//! Guided, conservative hardware acceptance for tinySA Basic and Ultra. + +use std::{ + collections::HashSet, + env, + error::Error, + fs, + io::{self, Write}, + path::PathBuf, + thread, + time::Duration, +}; +use tiny_wb::{ + acquisition::{acquire, ScanEvent}, + config::{Config, FrequencyAxis, OutputConfig}, + connection::open_connection, + export::save_csv, + protocol::{Sample, ScanRequest}, + transport::Shell, +}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Model { + Basic, + Ultra, +} + +impl Model { + fn parse(value: &str) -> Option { + match value.to_ascii_lowercase().as_str() { + "basic" => Some(Self::Basic), + "ultra" => Some(Self::Ultra), + _ => None, + } + } + + fn name(self) -> &'static str { + match self { + Self::Basic => "Basic", + Self::Ultra => "Ultra", + } + } + + fn scans(self) -> [(&'static str, ScanRequest, bool); 2] { + let low = ScanRequest { + start_hz: 240_000_000, + stop_hz: 300_000_000, + points: 101, + repeat: 1, + }; + let second = match self { + Self::Basic => ScanRequest { + start_hz: 350_000_000, + stop_hz: 400_000_000, + ..low + }, + // Ultra support is experimental. Stay on the shared low-input + // command path until model-aware extended ranges are implemented. + Self::Ultra => low, + }; + [("legacy", low, true), ("direct", second, false)] + } +} + +struct Arguments { + model: Model, + port: Option, + output: PathBuf, +} + +fn usage() -> &'static str { + "usage: validate-hardware [--port DEVICE] [--output DIRECTORY]" +} + +fn parse_arguments(arguments: impl IntoIterator) -> Result { + let mut arguments = arguments.into_iter(); + let model = arguments + .next() + .as_deref() + .and_then(Model::parse) + .ok_or_else(|| usage().to_owned())?; + let mut port = None; + let mut output = PathBuf::from("hardware-validation"); + while let Some(option) = arguments.next() { + let value = arguments + .next() + .ok_or_else(|| format!("missing value for {option}\n{}", usage()))?; + match option.as_str() { + "--port" if port.is_none() && !value.is_empty() => port = Some(value), + "--output" if !value.is_empty() => output = PathBuf::from(value), + _ => return Err(format!("invalid option: {option}\n{}", usage())), + } + } + Ok(Arguments { + model, + port, + output, + }) +} + +fn prompt(message: &str) -> io::Result { + print!("{message}"); + io::stdout().flush()?; + let mut answer = String::new(); + io::stdin().read_line(&mut answer)?; + Ok(answer.trim().to_owned()) +} + +fn ports() -> Result, serialport::Error> { + Ok(serialport::available_ports()? + .into_iter() + .map(|port| port.port_name) + .collect()) +} + +fn select_port(explicit: Option) -> Result> { + if let Some(port) = explicit { + prompt("Disconnect every RF source, plug in and power on the tinySA, then press Enter: ")?; + return Ok(port); + } + + prompt("Disconnect the tinySA USB cable and every RF source, then press Enter: ")?; + let before: HashSet<_> = ports()?.into_iter().collect(); + prompt("Plug in and power on the tinySA, then press Enter: ")?; + thread::sleep(Duration::from_millis(500)); + let available = ports()?; + let added: Vec<_> = available + .iter() + .filter(|port| !before.contains(*port)) + .cloned() + .collect(); + if added.len() == 1 { + println!("Detected {}", added[0]); + return Ok(added[0].clone()); + } + if available.len() == 1 { + println!("Using the only available serial port: {}", available[0]); + return Ok(available[0].clone()); + } + if available.is_empty() { + return Err("no serial port appeared; check the USB data cable and permissions".into()); + } + println!("Select the tinySA serial port:"); + for (index, port) in available.iter().enumerate() { + println!(" {}. {port}", index + 1); + } + let selection: usize = prompt("Port number: ")?.parse()?; + available + .get( + selection + .checked_sub(1) + .ok_or("port number must be at least 1")?, + ) + .cloned() + .ok_or_else(|| "port number is outside the displayed list".into()) +} + +fn validate_samples(samples: &[Sample], request: &ScanRequest) -> Result<(), Box> { + if samples.len() != request.points { + return Err(format!( + "expected {} samples, received {}", + request.points, + samples.len() + ) + .into()); + } + if samples.iter().any(|sample| { + !sample.frequency_mhz.is_finite() + || !sample.level_dbm.is_finite() + || sample.frequency_mhz < request.start_hz as f64 / 1_000_000.0 + || sample.frequency_mhz > request.stop_hz as f64 / 1_000_000.0 + }) { + return Err("CSV data contains a non-finite or out-of-range value".into()); + } + Ok(()) +} + +fn safe_identity(identity: &str) -> String { + identity + .chars() + .filter(|character| *character == '\n' || *character == '\r' || !character.is_control()) + .collect() +} + +fn run() -> Result<(), Box> { + let arguments = parse_arguments(env::args().skip(1)).map_err(io::Error::other)?; + println!( + "tinySA {} hardware validation\nNo RF source should be connected during this communications test.", + arguments.model.name() + ); + let port = select_port(arguments.port)?; + let session = arguments.output.join(format!( + "{}-{}", + arguments.model.name().to_ascii_lowercase(), + chrono::Local::now().format("%Y%m%d-%H%M%S-%6f") + )); + fs::create_dir_all(&session)?; + + let mut report = format!( + "requested_model={}\nport={}\nbaudrate=115200\n", + arguments.model.name(), + port + ); + let mut identity = String::new(); + for (name, request, legacy_pre_scan) in arguments.model.scans() { + println!( + "Running {name} scan: {}–{} MHz, {} points...", + request.start_hz / 1_000_000, + request.stop_hz / 1_000_000, + request.points + ); + let mut config = Config::default(); + config.port.clone_from(&port); + config.start_frequency = (request.start_hz / 1_000_000) as usize; + config.stop_frequency = (request.stop_hz / 1_000_000) as usize; + config.points = request.points; + config.repeat = request.repeat; + config.scan.frequency_axis = FrequencyAxis::Firmware; + config.scan.legacy_pre_scan = legacy_pre_scan; + config.validate()?; + let connection = open_connection(&config)?; + let mut shell = Shell::new(connection, config.serial.max_response_bytes); + let samples = acquire( + &mut shell, + &request, + &config.scan, + &config.serial, + |event| { + if let ScanEvent::InstrumentInfo(value) = event { + identity = safe_identity(&value); + } + }, + )?; + validate_samples(&samples, &request)?; + let output = OutputConfig { + directory: session.clone(), + filename_prefix: format!("{}-{name}", arguments.model.name().to_ascii_lowercase()), + include_header: true, + }; + let path = save_csv(&samples, &output)?; + println!( + "PASS: saved {} validated samples to {}", + samples.len(), + path.display() + ); + report.push_str(&format!( + "scan={name},start_hz={},stop_hz={},points={},legacy_pre_scan={},csv={}\n", + request.start_hz, + request.stop_hz, + request.points, + legacy_pre_scan, + path.display() + )); + } + + println!("\nInstrument identity returned by the device:\n{identity}"); + let confirmed = prompt(&format!( + "Does this match a tinySA {} and is its live sweep running again? [y/N] ", + arguments.model.name() + ))?; + report.push_str(&format!( + "identity=\n{identity}\noperator_confirmed={}\n", + confirmed + )); + let report_path = session.join(format!( + "{}-report.txt", + arguments.model.name().to_ascii_lowercase() + )); + fs::write(&report_path, report)?; + if !matches!(confirmed.as_str(), "y" | "Y" | "yes" | "YES" | "Yes") { + return Err(format!( + "operator confirmation failed; partial report: {}", + report_path.display() + ) + .into()); + } + println!( + "PASS: hardware validation report saved to {}", + report_path.display() + ); + if arguments.model == Model::Ultra { + println!("Ultra result covers only the shared 240–300 MHz low-input path; extended ranges remain unsupported."); + } + Ok(()) +} + +fn main() { + if let Err(error) = run() { + eprintln!("FAIL: {error}"); + std::process::exit(1); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn plans_keep_ultra_on_the_shared_low_input_path() { + let basic = Model::Basic.scans(); + assert_eq!(basic[0].1.input_mode().unwrap().as_str(), "low"); + assert_eq!(basic[1].1.input_mode().unwrap().as_str(), "high"); + let ultra = Model::Ultra.scans(); + assert!(ultra.iter().all(|(_, request, _)| { + request.stop_hz < 350_000_000 && request.input_mode().unwrap().as_str() == "low" + })); + assert!(ultra[0].2); + assert!(!ultra[1].2); + } + + #[test] + fn sample_validation_rejects_bad_count_and_values() { + let request = ScanRequest { + start_hz: 240_000_000, + stop_hz: 300_000_000, + points: 2, + repeat: 1, + }; + let valid = [ + Sample { + frequency_mhz: 240.0, + level_dbm: -90.0, + }, + Sample { + frequency_mhz: 270.0, + level_dbm: -80.0, + }, + ]; + assert!(validate_samples(&valid, &request).is_ok()); + assert!(validate_samples(&valid[..1], &request).is_err()); + let invalid = [ + valid[0], + Sample { + frequency_mhz: 270.0, + level_dbm: f64::NAN, + }, + ]; + assert!(validate_samples(&invalid, &request).is_err()); + } + + #[test] + fn arguments_require_a_known_model_and_option_values() { + assert_eq!( + parse_arguments(["ultra".into(), "--port".into(), "COM7".into()]) + .unwrap() + .port + .as_deref(), + Some("COM7") + ); + assert!(parse_arguments(["unknown".into()]).is_err()); + assert!(parse_arguments(["basic".into(), "--port".into()]).is_err()); + assert!(parse_arguments(["basic".into(), "--bad".into(), "x".into()]).is_err()); + } +} diff --git a/scripts/test-hardware.sh b/scripts/test-hardware.sh new file mode 100755 index 0000000..36a76bd --- /dev/null +++ b/scripts/test-hardware.sh @@ -0,0 +1,14 @@ +#!/bin/sh +set -eu + +case "${1:-}" in + basic|ultra) ;; + *) + echo "usage: $0 [--port DEVICE] [--output DIRECTORY]" >&2 + exit 2 + ;; +esac + +repository=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$repository" +exec cargo run --locked --release --example validate-hardware -- "$@"