diff --git a/crates/rundown-converter/src/gui/review.rs b/crates/rundown-converter/src/gui/review.rs index 5789d6e..02e88bf 100644 --- a/crates/rundown-converter/src/gui/review.rs +++ b/crates/rundown-converter/src/gui/review.rs @@ -1,6 +1,6 @@ use super::*; use crate::model::{Diagnostic, Mapping}; -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; #[derive(Default, PartialEq)] pub(super) enum ReviewView { @@ -1190,11 +1190,35 @@ impl App { }); } Err(errors) => { + let grouped = errors.iter().fold( + BTreeMap::<(&str, &str), Vec<&Diagnostic>>::new(), + |mut groups, error| { + groups + .entry((&error.code, &error.message)) + .or_default() + .push(error); + groups + }, + ); ui.colored_label( PREVIEW_AMBER, - format!("! Column mapping needs attention · {} issues", errors.len()), + if grouped.len() == errors.len() { + format!("! Column mapping needs attention · {} issues", errors.len()) + } else { + let issue_type = if grouped.len() == 1 { + "issue type" + } else { + "issue types" + }; + format!( + "! Column mapping needs attention · {} affected cells · {} {issue_type}", + errors.len(), + grouped.len() + ) + }, ); - for error in errors { + for ((code, message), occurrences) in grouped { + let error = occurrences[0]; if let Some((role, candidates)) = ambiguous_columns(&error.message) { ambiguities.push((role.clone(), candidates.clone())); let letters = candidates @@ -1227,12 +1251,32 @@ impl App { ); }); } else { - ui.add( - egui::Label::new( - RichText::new(&error.message).color(PREVIEW_AMBER), + let summary = if code == "FORMULA_WITHOUT_VALUE" { + format!( + "{} formulas have no saved result. Simple arithmetic is calculated automatically; recalculate the remaining formulas in Excel or LibreOffice.", + occurrences.len() ) - .wrap(), + } else if occurrences.len() > 1 { + format!("{message} · {} occurrences", occurrences.len()) + } else { + message.to_string() + }; + ui.add( + egui::Label::new(RichText::new(summary).color(PREVIEW_AMBER)) + .wrap(), ); + if occurrences.len() > 1 { + egui::CollapsingHeader::new(RichText::new("Affected cells").weak()) + .show(ui, |ui| { + ui.weak( + occurrences + .iter() + .map(|error| error.location.as_str()) + .collect::>() + .join(", "), + ); + }); + } } } } @@ -2338,6 +2382,66 @@ mod tests { ); } + #[test] + fn repeated_formula_errors_are_one_actionable_column_issue() { + let cells = |values: &[&str]| { + values + .iter() + .map(|text| crate::model::Cell { + text: (*text).into(), + ..Default::default() + }) + .collect::>() + }; + let mut table = Table { + name: "Formulas".into(), + rows: vec![cells(&["Start", "Duration", "Title"])], + merges: vec![], + }; + for row in 2..=42 { + let mut cells = cells(&["09:00", "", &format!("Cue {row}")]); + cells[1].formula = Some(format!("A{row}-A1")); + table.rows.push(cells); + } + let profile = Profile { + header_row: Some(1), + columns: BTreeMap::from([ + ("start".into(), "A".into()), + ("duration".into(), "B".into()), + ("title".into(), "C".into()), + ]), + duration_unit: Some("excel".into()), + ..Profile::default() + }; + let analysis = pipeline::analyze(&table, &profile, &AtomicBool::new(false), |_, _| {}); + let mut app = App { + sheet: table.name.clone(), + tables: vec![table], + analysis: Some(analysis), + profile: serde_json::to_string_pretty(&profile).unwrap(), + tab: Tab::Review, + review_view: ReviewView::Columns, + ..Default::default() + }; + let ctx = egui::Context::default(); + configure_style(&ctx); + render(&mut app, &ctx, [900.0, 900.0], vec![]); + let labels = render(&mut app, &ctx, [900.0, 900.0], vec![]); + assert!(labels.iter().any(|(text, _)| { + text == "! Column mapping needs attention · 41 affected cells · 1 issue type" + })); + assert_eq!( + labels + .iter() + .filter(|(text, _)| text.starts_with("41 formulas have no saved result.")) + .count(), + 1 + ); + assert!(!labels.iter().any(|(text, _)| { + text == "Recalculate and save the workbook; this formula has no cached value" + })); + } + #[test] fn stored_value_headers_have_unique_ids() { let mut app = sample(); diff --git a/crates/rundown-converter/src/reader.rs b/crates/rundown-converter/src/reader.rs index 4441efb..0d3aec5 100644 --- a/crates/rundown-converter/src/reader.rs +++ b/crates/rundown-converter/src/reader.rs @@ -44,7 +44,8 @@ fn coord(s: &str) -> Result<(usize, usize)> { ); Ok((row - 1, col)) } -/// Read XLSX or delimited UTF-8 CSV. No macros or formula evaluation is performed. +/// Read XLSX or delimited UTF-8 CSV. Macros are never executed. Simple arithmetic +/// formulas that only reference cells on the same sheet are calculated locally. pub fn read(path: &Path) -> Result> { ensure!( path.metadata()?.len() <= LIMIT, @@ -238,8 +239,256 @@ fn xlsx(path: &Path) -> Result> { last_col, }); } + calculate_simple_formulas(&mut table); tables.push(table); } ensure!(!tables.is_empty(), "Workbook has no worksheets"); Ok(tables) } + +fn calculate_simple_formulas(table: &mut Table) { + let formula_count = table + .rows + .iter() + .flatten() + .filter(|cell| cell.formula.is_some() && cell.text.is_empty()) + .count(); + for _ in 0..formula_count { + let mut calculated = Vec::new(); + for (row, cells) in table.rows.iter().enumerate() { + for (col, cell) in cells.iter().enumerate() { + let Some(formula) = cell.formula.as_deref().filter(|_| cell.text.is_empty()) else { + continue; + }; + if let Some(value) = FormulaParser::new(formula, table).parse() { + calculated.push((row, col, value)); + } + } + } + if calculated.is_empty() { + break; + } + for (row, col, value) in calculated { + let cell = &mut table.rows[row][col]; + cell.text = value.to_string(); + cell.numeric = true; + } + } +} + +struct FormulaParser<'a> { + input: &'a [u8], + position: usize, + table: &'a Table, +} + +impl<'a> FormulaParser<'a> { + fn new(formula: &'a str, table: &'a Table) -> Self { + Self { + input: formula.strip_prefix('=').unwrap_or(formula).as_bytes(), + position: 0, + table, + } + } + + fn parse(mut self) -> Option { + let value = self.expression()?; + self.space(); + (self.position == self.input.len() && value.is_finite()).then_some(value) + } + + fn expression(&mut self) -> Option { + let mut value = self.term()?; + loop { + self.space(); + match self.peek() { + Some(b'+') => { + self.position += 1; + value += self.term()?; + } + Some(b'-') => { + self.position += 1; + value -= self.term()?; + } + _ => break, + } + } + Some(value) + } + + fn term(&mut self) -> Option { + let mut value = self.factor()?; + loop { + self.space(); + match self.peek() { + Some(b'*') => { + self.position += 1; + value *= self.factor()?; + } + Some(b'/') => { + self.position += 1; + let divisor = self.factor()?; + if divisor == 0.0 { + return None; + } + value /= divisor; + } + _ => break, + } + } + Some(value) + } + + fn factor(&mut self) -> Option { + self.space(); + if self.consume(b'+') { + return self.factor(); + } + if self.consume(b'-') { + return Some(-self.factor()?); + } + if self.consume(b'(') { + let value = self.expression()?; + self.space(); + return self.consume(b')').then_some(value); + } + self.reference().or_else(|| self.number()) + } + + fn reference(&mut self) -> Option { + let start = self.position; + let value = (|| { + self.consume(b'$'); + let letters = self.position; + while self.peek().is_some_and(|b| b.is_ascii_alphabetic()) { + self.position += 1; + } + (self.position > letters).then_some(())?; + let column = std::str::from_utf8(&self.input[letters..self.position]).ok()?; + self.consume(b'$'); + let digits = self.position; + while self.peek().is_some_and(|b| b.is_ascii_digit()) { + self.position += 1; + } + let row = std::str::from_utf8(&self.input[digits..self.position]) + .ok()? + .parse::() + .ok()? + .checked_sub(1)?; + let column = column_index(column)?; + let cell = self.table.rows.get(row)?.get(column)?; + (cell.numeric && !cell.text.is_empty()).then_some(())?; + cell.text.parse().ok() + })(); + if value.is_none() { + self.position = start; + } + value + } + + fn number(&mut self) -> Option { + let start = self.position; + while self.peek().is_some_and(|b| b.is_ascii_digit()) { + self.position += 1; + } + if self.consume(b'.') { + while self.peek().is_some_and(|b| b.is_ascii_digit()) { + self.position += 1; + } + } + if self.peek().is_some_and(|b| matches!(b, b'e' | b'E')) { + self.position += 1; + if self.peek().is_some_and(|b| matches!(b, b'+' | b'-')) { + self.position += 1; + } + let exponent = self.position; + while self.peek().is_some_and(|b| b.is_ascii_digit()) { + self.position += 1; + } + if exponent == self.position { + self.position = start; + return None; + } + } + if start == self.position || &self.input[start..self.position] == b"." { + return None; + } + let value = std::str::from_utf8(&self.input[start..self.position]) + .ok()? + .parse() + .ok(); + if value.is_none() { + self.position = start; + } + value + } + + fn space(&mut self) { + while self.peek().is_some_and(|byte| byte.is_ascii_whitespace()) { + self.position += 1; + } + } + + fn consume(&mut self, expected: u8) -> bool { + if self.peek() == Some(expected) { + self.position += 1; + true + } else { + false + } + } + + fn peek(&self) -> Option { + self.input.get(self.position).copied() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn numeric(text: &str) -> Cell { + Cell { + text: text.into(), + numeric: true, + ..Cell::default() + } + } + + fn formula(text: &str) -> Cell { + Cell { + formula: Some(text.into()), + ..Cell::default() + } + } + + #[test] + fn calculates_bounded_arithmetic_and_formula_dependencies() { + let mut table = Table { + name: "Formula".into(), + rows: vec![vec![ + numeric("0.5"), + numeric("0.75"), + formula("=$B$1-A1"), + formula("C1 * 2"), + ]], + merges: vec![], + }; + calculate_simple_formulas(&mut table); + assert_eq!(table.rows[0][2].text, "0.25"); + assert_eq!(table.rows[0][3].text, "0.5"); + assert!(table.rows[0][2].numeric); + } + + #[test] + fn leaves_functions_and_cycles_for_a_spreadsheet_application() { + let mut table = Table { + name: "Formula".into(), + rows: vec![vec![formula("SUM(B1:C1)"), formula("A1"), numeric("1")]], + merges: vec![], + }; + calculate_simple_formulas(&mut table); + assert!(table.rows[0][0].text.is_empty()); + assert!(table.rows[0][1].text.is_empty()); + } +} diff --git a/docs/rundown-converter-plan.md b/docs/rundown-converter-plan.md index 56fdc36..28e295b 100644 --- a/docs/rundown-converter-plan.md +++ b/docs/rundown-converter-plan.md @@ -37,8 +37,9 @@ ranges and source coordinates when available. A table model separates extraction from interpretation so format-specific quirks do not leak into the Ontime adapter. PDF extraction must retain page/position information and expose unreliable reading order; image-only documents must produce an actionable unsupported/extraction -error until a tested reader exists. Never execute document macros or formulas. -Missing or stale formula results require explicit handling. +error until a tested reader exists. Never execute document macros or spreadsheet +functions. A bounded arithmetic parser may calculate same-sheet references when +cached results are missing; unsupported or stale results require explicit handling. Planned core modules cover readers, detection, neutral model, timing, diagnostics, export and orchestration. GUI and CLI are thin clients of the same core; neither diff --git a/docs/rundown-converter.md b/docs/rundown-converter.md index 3032e20..f7ec9f5 100644 --- a/docs/rundown-converter.md +++ b/docs/rundown-converter.md @@ -178,10 +178,13 @@ GUI follow this same sequence; export cannot bypass unresolved errors. The CLI and GUI use the same detection, validation and export code. CLI failures identify the source location and return a nonzero exit status. Inputs stay unchanged; existing output requires explicit overwrite permission. -Processing is local. XLSX preserves merged ranges and cached formula values; -formulas and macros are never executed. Recalculate and save formula-based -workbooks before importing. The converter cannot determine whether cached values -are stale. CSV/TSV uses UTF-8, quoted fields and comma/semicolon/tab detection. +Processing is local. XLSX preserves merged ranges and cached formula values. +When a cached result is missing, the converter safely calculates arithmetic made +of numbers, same-sheet cell references, parentheses and `+`, `-`, `*` or `/`. +It does not run spreadsheet functions or macros. Remaining formulas are grouped +into one review issue with the affected cells; recalculate and save those files +in Excel or LibreOffice. The converter cannot determine whether cached values are +stale. CSV/TSV uses UTF-8, quoted fields and comma/semicolon/tab detection. Legacy XLS, ODS, Word, PDF and scans are not yet supported. A [PDF feasibility trial](rundown-corpus.md#pdf-feasibility-trial) recovered table text from the supplied PDFs but could not establish a complete timer schedule;