From a83d8d47c0a80b43d4b1da424c90d6a0486a85b8 Mon Sep 17 00:00:00 2001 From: Dandaedre Date: Tue, 3 Nov 2026 14:21:07 +0100 Subject: [PATCH] Add metadata and ignored column choices to rundown conversion --- crates/rundown-converter/src/detect.rs | 35 ++++- crates/rundown-converter/src/export.rs | 7 +- crates/rundown-converter/src/gui.rs | 9 +- crates/rundown-converter/src/gui/review.rs | 77 ++++++++++- crates/rundown-converter/src/model.rs | 12 ++ crates/rundown-converter/src/pipeline.rs | 8 +- crates/rundown-converter/tests/conversion.rs | 128 +++++++++++++++++++ docs/rundown-converter.md | 16 ++- 8 files changed, 276 insertions(+), 16 deletions(-) diff --git a/crates/rundown-converter/src/detect.rs b/crates/rundown-converter/src/detect.rs index 391495e..7c6bfed 100644 --- a/crates/rundown-converter/src/detect.rs +++ b/crates/rundown-converter/src/detect.rs @@ -73,6 +73,13 @@ pub fn structural_row(table: &Table, row: usize, header: usize) -> bool { && m.last_col + 1 >= width }) } +/// Ignored columns cannot make an otherwise empty row require clarification. +pub fn has_content(table: &Table, row: usize, profile: &Profile) -> bool { + table.rows[row] + .iter() + .enumerate() + .any(|(col, cell)| profile.usage(col) != Some("ignore") && !placeholder(&cell.text)) +} pub fn active_rows(table: &Table, header: usize, profile: &Profile) -> Vec { (header + 1..table.rows.len()) .filter(|r| { @@ -86,7 +93,6 @@ pub fn active_rows(table: &Table, header: usize, profile: &Profile) -> Vec Vec { + column_usage.insert(column_name(c), usage.clone()); + evidence.push(format!("Explicit {usage} column {}", column_name(c))); + } + _ => profile_errors.push(err("INVALID_COLUMN_USAGE", table, header, 0, format!("Invalid or duplicate column usage {letter}: {usage}; use an existing column with metadata or ignore"))), + } + } + if !profile_errors.is_empty() { + return Err(profile_errors); + } let rows = active_rows(table, header, profile); let mut diagnostics = vec![]; @@ -206,6 +226,13 @@ fn infer_inner( continue; } match column_index(col).filter(|c| *c < width) { + Some(c) if excluded.contains(&c) => diagnostics.push(err( + "INVALID_MAPPING", + table, + header, + c, + format!("Column {col} cannot be both {r} and metadata/ignore; choose one meaning"), + )), Some(c) => { selected.insert(r.as_str(), c); evidence.push(format!("Explicit {r} column {col}")); @@ -224,6 +251,7 @@ fn infer_inner( continue; } let candidates: Vec<_> = (0..width) + .filter(|c| !excluded.contains(c)) .filter(|c| role(&table.cell(header, *c).text) == Some(r)) .collect(); if candidates.len() == 1 { @@ -242,6 +270,7 @@ fn infer_inner( continue; } let candidates: Vec<_> = (0..width) + .filter(|c| !excluded.contains(c)) .filter(|c| !selected.values().any(|v| v == c)) .filter(|c| { let cells: Vec<_> = rows @@ -278,6 +307,7 @@ fn infer_inner( // Cue is a title only as a fallback; it is also commonly a cue number. if !selected.contains_key("title") { let cols: Vec<_> = (0..width) + .filter(|c| !excluded.contains(c)) .filter(|c| normalize(&table.cell(header, *c).text) == "cue") .collect(); if cols.len() == 1 { @@ -426,5 +456,6 @@ fn infer_inner( title: selected["title"], duration_unit: unit, evidence: evidence.clone(), + column_usage, }) } diff --git a/crates/rundown-converter/src/export.rs b/crates/rundown-converter/src/export.rs index 881c389..3131435 100644 --- a/crates/rundown-converter/src/export.rs +++ b/crates/rundown-converter/src/export.rs @@ -21,7 +21,12 @@ pub fn ontime(r: &Rundown) -> Result { let mut note = e.decisions.join("\n"); for row in &e.source_rows { for (col, cell) in &row.cells { - if !cell.text.is_empty() { + if !cell.text.is_empty() + && r.mapping + .column_usage + .get(col) + .is_none_or(|usage| usage != "ignore") + { note.push_str(&format!("\n{col}{}: {}", row.row, cell.text)); } } diff --git a/crates/rundown-converter/src/gui.rs b/crates/rundown-converter/src/gui.rs index bc41765..967f702 100644 --- a/crates/rundown-converter/src/gui.rs +++ b/crates/rundown-converter/src/gui.rs @@ -286,6 +286,7 @@ impl App { for role in ["start", "end", "duration", "title"] { ui.label(role); let mut col = profile.columns.get(role).cloned().unwrap_or_default(); + let previous = col.clone(); egui::ComboBox::from_id_salt(role) .selected_text(if col.is_empty() { "Infer automatically" @@ -310,6 +311,12 @@ impl App { if col.is_empty() { profile.columns.remove(role); } else { + if col != previous { + profile.column_usage.retain(|letter, _| { + crate::model::column_index(letter) + != crate::model::column_index(&col) + }); + } profile.columns.insert(role.into(), col); } } @@ -337,7 +344,7 @@ impl App { self.decisions_changed(); } } - ui.label("Optional JSON profile: columns (start/end/duration/title), duration_unit, row_kinds, durations, untimed_as_notes. Column letters and source rows are shown in diagnostics."); + ui.label("Optional JSON profile: columns (start/end/duration/title), column_usage (metadata/ignore), duration_unit, row_kinds, durations, untimed_as_notes. Column letters and source rows are shown in diagnostics."); if ui .add( egui::TextEdit::multiline(&mut self.profile) diff --git a/crates/rundown-converter/src/gui/review.rs b/crates/rundown-converter/src/gui/review.rs index 05b613d..4e18c02 100644 --- a/crates/rundown-converter/src/gui/review.rs +++ b/crates/rundown-converter/src/gui/review.rs @@ -489,6 +489,7 @@ impl App { fn columns_view(&mut self, ui: &mut egui::Ui) { ui.heading("Column meanings"); ui.label("Choose the intended role when inference is ambiguous. Every timing cell is still validated."); + ui.label("Metadata goes into Ontime notes. Ignore excludes the column from inference and Ontime export. Auto includes other columns as metadata."); self.mapping_controls(ui); if let (Some(table), Ok(mut profile)) = ( self.tables.iter().find(|t| t.name == self.sheet), @@ -522,26 +523,37 @@ impl App { let mut chosen = profile .columns .iter() - .find(|(_, v)| **v == col) + .find(|(_, v)| crate::model::column_index(v) == Some(c)) .map(|(k, _)| k.clone()) + .or_else(|| profile.usage(c).map(str::to_string)) .unwrap_or_default(); let old = chosen.clone(); egui::ComboBox::from_id_salt(("column", c)) .selected_text(if chosen.is_empty() { "Auto" } else { &chosen }) .show_ui(ui, |ui| { ui.selectable_value(&mut chosen, String::new(), "Auto"); - for role in ["start", "end", "duration", "title"] { + for role in ["start", "end", "duration", "title", "metadata", "ignore"] + { ui.selectable_value(&mut chosen, role.into(), role); } }); if chosen != old { - profile.columns.retain(|_, v| *v != col); - if !chosen.is_empty() { + profile + .columns + .retain(|_, v| crate::model::column_index(v) != Some(c)); + profile + .column_usage + .retain(|letter, _| crate::model::column_index(letter) != Some(c)); + if matches!(chosen.as_str(), "metadata" | "ignore") { + profile.column_usage.insert(col, chosen); + } else if !chosen.is_empty() { profile.columns.insert(chosen, col); } } if let Ok(m) = &mapping { - let role = if m.start == c { + let role = if profile.usage(c) == Some("ignore") { + "ignore" + } else if m.start == c { "start" } else if m.title == c { "title" @@ -707,6 +719,15 @@ mod tests { } fn click(app: &mut App, ctx: &egui::Context, pos: egui::Pos2) -> Vec<(String, egui::Rect)> { + click_size(app, ctx, pos, [560.0, 400.0]) + } + + fn click_size( + app: &mut App, + ctx: &egui::Context, + pos: egui::Pos2, + size: [f32; 2], + ) -> Vec<(String, egui::Rect)> { let events = vec![ egui::Event::PointerMoved(pos), egui::Event::PointerButton { @@ -722,7 +743,7 @@ mod tests { modifiers: Default::default(), }, ]; - render(app, ctx, [560.0, 400.0], events) + render(app, ctx, size, events) } #[test] @@ -800,4 +821,48 @@ mod tests { assert!(labels.iter().any(|(s, _)| s == "Opening")); assert!(labels.iter().any(|(s, _)| s == "Review columns")); } + #[test] + fn column_menu_selects_metadata_ignore_and_restores_auto() { + let mut app = sample(); + app.review_view = ReviewView::Columns; + app.profile = r#"{"columns":{"start":"A"}}"#.into(); + let ctx = egui::Context::default(); + configure_style(&ctx); + let size = [860.0, 900.0]; + for (current, choice) in [ + ("start", "metadata"), + ("metadata", "ignore"), + ("ignore", "Auto"), + ] { + render(&mut app, &ctx, size, vec![]); + let labels = render(&mut app, &ctx, size, vec![]); + let button = labels + .iter() + .find(|(s, _)| s == current) + .unwrap() + .1 + .center(); + click_size(&mut app, &ctx, button, size); + let menu = render(&mut app, &ctx, size, vec![]); + let option = menu + .iter() + .rev() + .find(|(s, _)| s == choice) + .unwrap() + .1 + .center(); + click_size(&mut app, &ctx, option, size); + let profile: Profile = serde_json::from_str(&app.profile).unwrap(); + assert!( + !profile.columns.contains_key("start"), + "{current} -> {choice}: {}", + app.profile + ); + assert_eq!( + profile.usage(0), + if choice == "Auto" { None } else { Some(choice) } + ); + assert!(app.dirty); + } + } } diff --git a/crates/rundown-converter/src/model.rs b/crates/rundown-converter/src/model.rs index 314d8c1..74f4732 100644 --- a/crates/rundown-converter/src/model.rs +++ b/crates/rundown-converter/src/model.rs @@ -64,6 +64,8 @@ pub struct Profile { /// One-based header row. Column letters are Excel-style, e.g. A or AB. pub header_row: Option, pub columns: BTreeMap, + /// Column letter -> metadata (notes only) or ignore (excluded from export). + pub column_usage: BTreeMap, pub expected_headers: BTreeMap, /// minutes, seconds, hh:mm, mm:ss, or excel. pub duration_unit: Option, @@ -74,6 +76,14 @@ pub struct Profile { /// Explicit duration overrides, expressed with units (e.g. 2 min). pub durations: BTreeMap, } +impl Profile { + pub fn usage(&self, col: usize) -> Option<&str> { + self.column_usage + .iter() + .find(|(letter, _)| column_index(letter) == Some(col)) + .map(|(_, usage)| usage.as_str()) + } +} #[derive(Clone, Debug, Serialize, Deserialize)] pub struct Mapping { pub header_row: usize, @@ -83,6 +93,8 @@ pub struct Mapping { pub title: usize, pub duration_unit: Option, pub evidence: Vec, + #[serde(default)] + pub column_usage: BTreeMap, } #[derive(Clone, Debug, Serialize, Deserialize)] pub struct SourceRow { diff --git a/crates/rundown-converter/src/pipeline.rs b/crates/rundown-converter/src/pipeline.rs index f243484..f9d2b3d 100644 --- a/crates/rundown-converter/src/pipeline.rs +++ b/crates/rundown-converter/src/pipeline.rs @@ -62,7 +62,8 @@ fn times(table: &Table, row: usize, map: &Mapping) -> (Option, Option) pub fn identify_gaps(table: &Table, map: &Mapping, profile: &Profile) -> Vec { let mut rows = detect::active_rows(table, map.header_row - 1, profile); for r in map.header_row..table.rows.len() { - if detect::structural_row(table, r, map.header_row - 1) + if detect::has_content(table, r, profile) + && detect::structural_row(table, r, map.header_row - 1) && table.rows[r].iter().any(|c| is_gap(&c.text)) && !profile .row_kinds @@ -191,10 +192,7 @@ pub fn construct( while row < table.rows.len() { let r = row; row += 1; - if table.rows[r] - .iter() - .all(|c| matches!(c.text.as_str(), "" | "-" | "—")) - { + if !detect::has_content(table, r, profile) { continue; } if detect::structural_row(table, r, map.header_row - 1) diff --git a/crates/rundown-converter/tests/conversion.rs b/crates/rundown-converter/tests/conversion.rs index 6fd41d7..9e211b4 100644 --- a/crates/rundown-converter/tests/conversion.rs +++ b/crates/rundown-converter/tests/conversion.rs @@ -581,3 +581,131 @@ fn explicit_duration_can_replace_a_missing_formula_result() { }; assert!(analyze(&t, &p).diagnostics.is_empty()); } + +#[test] +fn metadata_and_ignored_columns_control_inference_and_ontime_notes() { + let t = table(&[ + &[ + "Start", "Timer", "Timer", "Title", "Speaker", "Private", "Notes", + ], + &[ + "09:00", + "5 min", + "Operator reference", + "Opening", + "Alice", + "SECRET ONE", + "Microphone 1", + ], + &[ + "", + "", + "", + "Stage direction", + "Bob", + "SECRET TWO", + "Move lectern", + ], + &["", "", "", "", "", "SECRET ONLY ROW", ""], + &[ + "09:05", + "5 min", + "Backup reference", + "Closing", + "Carol", + "SECRET THREE", + "Microphone 2", + ], + ]); + let p = Profile { + column_usage: BTreeMap::from([ + ("C".into(), "metadata".into()), + ("E".into(), "metadata".into()), + ("f".into(), "ignore".into()), + ("G".into(), "metadata".into()), + ]), + row_kinds: BTreeMap::from([(3, "note".into())]), + ..Profile::default() + }; + let a = analyze(&t, &p); + assert!(a.diagnostics.is_empty(), "{:?}", a.diagnostics); + let r = a.rundown.unwrap(); + assert_eq!(r.mapping.duration, Some(1)); + assert_eq!(r.mapping.column_usage["F"], "ignore"); + assert_eq!(r.entries.len(), 2); + assert_eq!(r.entries[0].source_rows[0].cells["F"].text, "SECRET ONE"); + assert_eq!(r.entries[0].source_rows[1].cells["F"].text, "SECRET TWO"); + let json = export::ontime(&r).unwrap(); + let note = json["rundowns"]["default"]["entries"][&r.entries[0].id]["note"] + .as_str() + .unwrap(); + for text in [ + "Operator reference", + "Alice", + "Bob", + "Microphone 1", + "Move lectern", + ] { + assert!(note.contains(text), "Missing metadata {text}"); + } + assert!(!json.to_string().contains("SECRET")); + let restored: Rundown = serde_json::from_str(&serde_json::to_string(&r).unwrap()).unwrap(); + assert!( + !export::ontime(&restored) + .unwrap() + .to_string() + .contains("SECRET") + ); + let restored_profile: Profile = + serde_json::from_str(&serde_json::to_string(&p).unwrap()).unwrap(); + assert_eq!(restored_profile.usage(5), Some("ignore")); +} + +#[test] +fn column_usage_excludes_data_candidates_and_rejects_conflicting_profiles() { + let t = table(&[ + &["Start", "End", "Reference", "Title"], + &["09:00", "09:05", "1 min", "Opening"], + &["09:05", "09:10", "2 min", "Closing"], + ]); + for usage in ["metadata", "ignore"] { + let p = Profile { + column_usage: BTreeMap::from([("C".into(), usage.into())]), + ..Default::default() + }; + let a = analyze(&t, &p); + assert!(a.diagnostics.is_empty(), "{:?}", a.diagnostics); + assert_eq!(a.mapping.unwrap().duration, None); + assert_eq!(a.rundown.unwrap().entries[0].duration_ms, 300_000); + } + for usages in [ + vec![("Z", "ignore")], + vec![("C", "unknown")], + vec![("C", "ignore"), ("c", "metadata")], + ] { + let p = Profile { + column_usage: usages + .into_iter() + .map(|(k, v)| (k.into(), v.into())) + .collect(), + ..Default::default() + }; + assert!( + analyze(&t, &p) + .diagnostics + .iter() + .any(|d| d.code == "INVALID_COLUMN_USAGE") + ); + } + let p = Profile { + columns: BTreeMap::from([("start".into(), "A".into())]), + column_usage: BTreeMap::from([("A".into(), "ignore".into())]), + ..Default::default() + }; + assert!( + analyze(&t, &p) + .diagnostics + .iter() + .any(|d| d.code == "INVALID_MAPPING") + ); +} diff --git a/docs/rundown-converter.md b/docs/rundown-converter.md index 880f244..e276609 100644 --- a/docs/rundown-converter.md +++ b/docs/rundown-converter.md @@ -45,7 +45,10 @@ click **Analyze rundown**. Review has three focused views: and out-of-date schedules are labeled; they cannot be exported. - **Columns** shows inference evidence, per-column role dropdowns, and sample values. Selecting one duration column explicitly resolves duplicate TIMER - headers. **Auto** restores inference. Expand **Remap columns, header and units / + headers. **Metadata** keeps a column as Ontime notes without treating its values + as timing or titles; **Ignore** excludes it from inference and Ontime output. + Several columns can be metadata or ignored. **Auto** restores inference; other + columns remain included as metadata by default. Expand **Remap columns, header and units / advanced profile** for header row, duration units and JSON settings (also in Setup). Click **Apply & recheck** after corrections. Pending edits retain the issue list @@ -163,6 +166,7 @@ Pass `--profile mapping.json`, or load/edit the same JSON in the desktop app: { "header_row": 1, "columns": {"start": "A", "duration": "B", "title": "G"}, + "column_usage": {"C": "metadata", "F": "ignore"}, "expected_headers": {"A": "Heure", "B": "Durée"}, "untimed_as_notes": true, "row_kinds": {"20": "marker"}, @@ -178,6 +182,16 @@ units where available; otherwise units must be explicit or consistently supporte by independent start/end pairs. Unknown profile keys and changed expected headers are errors. Explicit mappings still undergo whole-column validation. +`column_usage` maps column letters to `metadata` or `ignore`. Both choices prevent +that column from being inferred as a start, end, duration or title, even when its +header suggests one of those roles. A column cannot also have an explicit timing +or title mapping. Metadata appears in Ontime event notes, including metadata from +rows attached as notes or merged continuations. Ignore removes those cell values +from Ontime notes; rows containing only ignored values need no clarification. +Original cells remain available in the review and neutral IR for retained rows. +These choices are saved in profiles and remembered resolutions; use **Apply & +recheck** before export. The default inclusion of other source columns is unchanged. + `row_kinds` accepts `event`, `gap`, `marker`, `note` and `ignore`. Notes attach to the preceding event. Untimed markers remain in the IR but are not exported as timers. An ignored row is recorded as an explicit exclusion. Duration overrides