diff --git a/.gitignore b/.gitignore index 6b5d048..501a002 100644 --- a/.gitignore +++ b/.gitignore @@ -161,5 +161,12 @@ cython_debug/ # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ -# Private source rundowns supplied for local evaluation -/rundowns/ +# Local rundowns and generated exports; only the regression corpus is tracked. +/rundowns/* +!/rundowns/README.md +!/rundowns/sources.json +!/rundowns/estivales-2025-plenieres.xlsx +!/rundowns/moebius-apres-midi-sante-mentale.xlsx +!/rundowns/moebius-matin-pleniere-sante-environnement.xlsx +!/rundowns/gifav-2026-auditorium.xlsx +!/rundowns/pleniere-2.xlsx diff --git a/AGENTS.md b/AGENTS.md index 60d9392..5af1408 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -20,8 +20,10 @@ with quick-start examples and links to detailed documentation in `docs/`. construction, consistency validation and Ontime export separate. A timing column must have a consistent meaning across all relevant rows. Ambiguities block export; explicit starts and untimed gaps must survive conversion. - Private examples in `rundowns/` and public downloads stay untracked; tests use - synthetic fixtures. PDF, XLS/ODS readers and buffer correction remain future work. + The five user-approved XLSX fixtures listed in `rundowns/sources.json` are tracked + and exercised by `tests/supplied_corpus.rs`; other local examples, generated exports + and public downloads stay untracked. Keep synthetic tests for isolated cases. + PDF, XLS/ODS readers and buffer correction remain future work. - `tools/progress.py` provides shared transient terminal progress and plain logs for redirected output; `tests/test_progress.py` checks display cleanup. diff --git a/crates/rundown-converter/tests/supplied_corpus.rs b/crates/rundown-converter/tests/supplied_corpus.rs new file mode 100644 index 0000000..7da7f21 --- /dev/null +++ b/crates/rundown-converter/tests/supplied_corpus.rs @@ -0,0 +1,164 @@ +//! User-supplied workbooks: exercise the actual XLSX reader and conversion pipeline. +use rundown_converter::{export, model::*, pipeline, reader}; +use std::{path::PathBuf, sync::atomic::AtomicBool}; + +fn workbook(name: &str) -> Vec { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../rundowns") + .join(name); + reader::read(&path).unwrap() +} + +fn analyze(table: &Table, profile: &Profile) -> Analysis { + pipeline::analyze(table, profile, &AtomicBool::new(false), |_, _| {}) +} + +fn codes(analysis: &Analysis) -> Vec<&str> { + analysis + .diagnostics + .iter() + .map(|d| d.code.as_str()) + .collect() +} + +#[test] +fn moebius_french_headers_and_excel_times_reach_schedule_validation() { + for (file, count, last_row, expected_codes) in [ + ( + "moebius-matin-pleniere-sante-environnement.xlsx", + 26, + 35, + vec!["INVALID_EVENT"], + ), + ( + "moebius-apres-midi-sante-mentale.xlsx", + 20, + 21, + vec!["INVALID_EVENT", "OVERLAP"], + ), + ] { + let tables = workbook(file); + assert_eq!(tables.len(), 1); + let analysis = analyze(&tables[0], &Profile::default()); + let mapping = analysis.mapping.as_ref().unwrap(); + assert_eq!( + ( + mapping.header_row, + mapping.start, + mapping.title, + mapping.duration + ), + (1, 0, 1, Some(2)), + "{file}" + ); + assert_eq!(codes(&analysis), expected_codes, "{file}"); + assert!( + analysis + .diagnostics + .iter() + .all(|d| d.location == format!("Feuil1!row {last_row}")) + ); + let rundown = analysis.rundown.unwrap(); + assert_eq!(rundown.entries.len(), count); + let first = &rundown.entries[0]; + assert_eq!(first.duration_ms, 15 * 60_000); + assert_eq!(first.end_ms - first.start_ms, first.duration_ms); + assert!(first.explicit_start); + assert!(!first.inferred_duration); + } +} + +#[test] +fn competing_pleniere_timers_require_a_column_choice() { + let tables = workbook("pleniere-2.xlsx"); + let analysis = analyze(&tables[0], &Profile::default()); + assert!(analysis.mapping.is_none()); + assert!(analysis.rundown.is_none()); + assert_eq!(codes(&analysis), ["AMBIGUOUS_COLUMN"]); + assert!(analysis.diagnostics[0].message.contains("C, D")); +} + +#[test] +fn estivales_merged_blocks_preserve_notes_and_missing_final_duration() { + let tables = workbook("estivales-2025-plenieres.xlsx"); + assert_eq!(tables.len(), 2); + let profile = Profile { + untimed_as_notes: true, + ..Default::default() + }; + let opening = tables.iter().find(|t| t.name == "OUVERTURE").unwrap(); + assert_eq!(opening.merges.len(), 9); + let analysis = analyze(opening, &profile); + assert_eq!(codes(&analysis), ["MISSING_DURATION"]); + let closing = tables.iter().find(|t| t.name == "CLOTURE").unwrap(); + assert_eq!(closing.merges.len(), 2); + let analysis = analyze(closing, &profile); + assert!( + analysis.diagnostics.is_empty(), + "{:?}", + analysis.diagnostics + ); + let rundown = analysis.rundown.unwrap(); + assert_eq!(rundown.entries.len(), 5); + let output = export::ontime(&rundown).unwrap(); + assert_eq!( + output["rundowns"]["default"]["order"] + .as_array() + .unwrap() + .len(), + 5 + ); + let rows: Vec<_> = rundown + .entries + .iter() + .flat_map(|entry| entry.source_rows.iter().map(|source| source.row)) + .collect(); + assert_eq!(rows, (2..=14).collect::>()); +} + +#[test] +fn gifav_parent_ranges_preserve_gaps_and_friday_overlap() { + let tables = workbook("gifav-2026-auditorium.xlsx"); + assert_eq!(tables.len(), 2); + let profile = Profile { + columns: [ + ("start_end".into(), "A".into()), + ("title".into(), "D".into()), + ] + .into(), + column_usage: [("B".into(), "ignore".into()), ("E".into(), "notes".into())].into(), + ..Default::default() + }; + for table in &tables { + let unmapped = analyze(table, &Profile::default()); + assert_eq!(codes(&unmapped), ["MISSING_COLUMN"]); + let analysis = analyze(table, &profile); + assert!(analysis.mapping.as_ref().unwrap().start_end); + let thursday = table.name == "Jeudi 5nov26"; + if thursday { + assert!(analysis.diagnostics.is_empty()); + } else { + assert_eq!(table.name, "Vendredi 6nov26_Auditorium"); + assert_eq!(codes(&analysis), ["OVERLAP"]); + assert_eq!( + analysis.diagnostics[0].location, + "Vendredi 6nov26_Auditorium!row 16" + ); + } + let rundown = analysis.rundown.unwrap(); + assert_eq!(rundown.entries.len(), if thursday { 11 } else { 10 }); + assert_eq!(rundown.gaps.len(), 3); + assert!(rundown.gaps.iter().all(|gap| !gap.display_timer)); + if thursday { + let output = export::ontime(&rundown).unwrap(); + let entries = output["rundowns"]["default"]["entries"] + .as_object() + .unwrap(); + assert_eq!(entries.len(), 11); + for event in &rundown.entries { + assert_eq!(entries[&event.id]["timeStart"], event.start_ms); + assert_eq!(entries[&event.id]["duration"], event.duration_ms); + } + } + } +} diff --git a/docs/rundown-converter.md b/docs/rundown-converter.md index 3ce9ef8..baa081d 100644 --- a/docs/rundown-converter.md +++ b/docs/rundown-converter.md @@ -546,7 +546,9 @@ Every adjusted schedule must pass validation again before export. See the The [sample findings](rundown-corpus.md) list the supplied documents, public workbooks and unresolved interpretation questions. Further examples should keep table structure, cell formats and representative time values when redacted. -Private originals stay out of Git; regression fixtures are synthetic. +Five user-approved source workbooks are tracked in [rundowns](../rundowns/README.md) +and exercised by offline regression tests alongside the synthetic fixtures. Other +local originals, PDFs and generated exports remain untracked. See [all tools](../README.md) and [repository development](development.md). diff --git a/docs/rundown-corpus.md b/docs/rundown-corpus.md index ba97264..15def65 100644 --- a/docs/rundown-corpus.md +++ b/docs/rundown-corpus.md @@ -1,10 +1,12 @@ # Rundown evaluation corpus -The implementation was evaluated on three supplied XLSX workbooks (five sheets), -two supplied PDFs, seven downloaded public XLSX workbooks and three -public CSV files. Source originals remain in ignored `rundowns/` and -`downloads/rundown-corpus/`; they are -not redistributed in Git. Automated regression tests use synthetic data. +The supplied corpus now includes five XLSX workbooks (seven sheets) and two PDFs. +The five workbooks are tracked at the user's request, with normalized filenames +and unchanged contents; see the [fixture inventory](../rundowns/README.md) and +[original-name/checksum manifest](../rundowns/sources.json). Four offline integration +tests exercise all seven sheets through the real reader and pipeline. PDFs and +generated exports remain ignored. Seven public XLSX downloads and three public +CSV files remain local evaluation inputs under `downloads/rundown-corpus/`. The [public source manifest](../crates/rundown-converter/tests/public-corpus.json) records download URLs, source pages, SHA-256 hashes and retrieval date. Downloads @@ -17,14 +19,16 @@ behaviors and limitations, not a general success percentage. | Example | Evidence and result | | --- | --- | | ESTIVALES, OUVERTURE | Correctly identifies Heure, Durée and Contenu / Détail. Shared merged durations remain one timer; merged windows with individual durations become consecutive cues with leftover slack. Untimed production directions need a note classification. Final 11:45 cue lacks duration and blocks export pending clarification. | -| ESTIVALES, CLOTURE | Five timing blocks. With the explicit `untimed_as_notes` policy, converts to five timers; all original continuation/direction rows remain in the IR and exported notes. This policy is a test scenario, not a confirmed interpretation from the owner. | +| ESTIVALES, CLOTURE | Five timing blocks. With the explicit `untimed_as_notes` policy, converts to five timers; all original continuation/direction rows remain in the IR; exported notes depend on the selected Notes columns. This policy is a test scenario, not a confirmed interpretation from the owner. | | Plénière 2, Feuil1 | Both C and D are named TIMER and contain plausible durations. Returns `AMBIGUOUS_COLUMN`; parent-segment versus child-cue timing needs clarification. No automatic conversion is claimed. | | GIFAV 2026 Auditorium | `Horaires` contains consistent parent-session start/end ranges and is detected as a combined Start–end column. `Durée` mixes child ranges with speaking/QA text, so it cannot be flattened into one duration role. Mapping Start–end to A and Title to D, retaining presentations as notes, validates Thursday with 11 entries and three gaps. Friday reaches schedule validation with 10 entries and three gaps, then correctly reports the source overlap between the 13:45–15:45 poster session and the 14:45–15:45 following session. | | Montage PDF | Text is extractable, but includes overlapping logistics tasks, missing durations and different locations. Not a single established timer sequence. | | Exploitation/démontage PDF | Text is extractable; page-spanning directions, simultaneous activities and untimed arrivals require track and row classification. PDF import remains a stretch goal. | -Private originals contain production details and contacts. Tests and documentation -do not reproduce those details. An unchanged local file can be inspected with +The supplied originals contain production details and contacts. The five tracked +workbooks were explicitly approved for Git inclusion by the user; this does not +assert a blanket redistribution license for third-party material. Test assertions +and documentation avoid reproducing their production text. An unchanged local file can be inspected with `rundown-converter inspect INPUT --json`; this output contains the original cells, so treat it like the source document. @@ -146,7 +150,11 @@ Synthetic tests cover whole-column contradictions, duplicate TIMER headers, ambiguous duration units resolved by other rows, Excel time values and merges, notes/markers, gaps, conflicting triples, backward times, negative durations, output collisions, source preservation, cancellation, CLI exit codes and desktop -worker state. Neither private originals nor public binaries are CI fixtures. +worker state. `tests/supplied_corpus.rs` additionally tests French header detection +and typed Excel durations in both Moebius files, duplicate timers in Plénière 2, +merged blocks and source-row retention in ESTIVALES, and parent ranges, gaps and +the real Friday overlap in GIFAV. These tests are offline and never write exports +beside the fixtures. Public downloads remain outside CI. ## Validation of the first implementation diff --git a/rundowns/README.md b/rundowns/README.md new file mode 100644 index 0000000..edfa539 --- /dev/null +++ b/rundowns/README.md @@ -0,0 +1,31 @@ +# Supplied rundown fixtures + +These five XLSX files are committed at the repository owner's request for offline +regression testing. Only filenames changed: workbook bytes, sheet names, source +cells, formulas and formatting are intact. `sources.json` records the original +filenames and SHA-256 hashes. The files contain original production information; +inclusion does not assert a blanket third-party redistribution license. + +| Workbook | Regression coverage | +| --- | --- | +| `estivales-2025-plenieres.xlsx` | Two sheets; merged timing blocks, missing final duration, retaining every closing-sheet source row after classifying rows without timing as notes. | +| `pleniere-2.xlsx` | Competing TIMER columns C and D must block automatic mapping. | +| `gifav-2026-auditorium.xlsx` | Two sheets; missing title mapping, explicit parent ranges, three gaps per sheet, successful Thursday export and the Friday overlap. | +| `moebius-matin-pleniere-sante-environnement.xlsx` | French Heure/Action/Temps headers, Excel durations, merged cells and final-row validation error. | +| `moebius-apres-midi-sante-mentale.xlsx` | Same French headers and typed times, final-row validation error and overlap. | + +Run `cargo test --locked -p rundown-converter --test supplied_corpus`. +The full workspace tests also run these checks. Tests do not modify the sources +or require a network connection. Expected errors are useful regression cases; +these fixtures are not all ready-to-run schedules. + +Filenames use lowercase ASCII words separated by hyphens. Dates include a year +only when known. Local exports keep `.ontime.json` / `.ontime.xlsx` suffixes; +sheet-specific GIFAV exports include `2026-11-05` or `2026-11-06`. Existing export +contents were not regenerated during renaming. + +The local PDFs were renamed to `beffroi-montage-04-octobre.pdf` and +`beffroi-exploitation-demontage-05-octobre.pdf`. They remain untracked: the reader +rejects PDFs before inspecting their content, so including those binaries would +not add meaningful regression coverage. Other local inputs and generated exports +are ignored; `.gitignore` explicitly allows only the five tested workbooks. diff --git a/rundowns/estivales-2025-plenieres.xlsx b/rundowns/estivales-2025-plenieres.xlsx new file mode 100644 index 0000000..9e396b3 Binary files /dev/null and b/rundowns/estivales-2025-plenieres.xlsx differ diff --git a/rundowns/gifav-2026-auditorium.xlsx b/rundowns/gifav-2026-auditorium.xlsx new file mode 100644 index 0000000..25d53e0 Binary files /dev/null and b/rundowns/gifav-2026-auditorium.xlsx differ diff --git a/rundowns/moebius-apres-midi-sante-mentale.xlsx b/rundowns/moebius-apres-midi-sante-mentale.xlsx new file mode 100644 index 0000000..91c1f2e Binary files /dev/null and b/rundowns/moebius-apres-midi-sante-mentale.xlsx differ diff --git a/rundowns/moebius-matin-pleniere-sante-environnement.xlsx b/rundowns/moebius-matin-pleniere-sante-environnement.xlsx new file mode 100644 index 0000000..03a7a71 Binary files /dev/null and b/rundowns/moebius-matin-pleniere-sante-environnement.xlsx differ diff --git a/rundowns/pleniere-2.xlsx b/rundowns/pleniere-2.xlsx new file mode 100644 index 0000000..03136b5 Binary files /dev/null and b/rundowns/pleniere-2.xlsx differ diff --git a/rundowns/sources.json b/rundowns/sources.json new file mode 100644 index 0000000..874a457 --- /dev/null +++ b/rundowns/sources.json @@ -0,0 +1,30 @@ +{ + "description": "User-supplied regression workbooks; filenames normalized, original bytes unchanged. Inclusion requested by the repository owner. No blanket third-party redistribution license is asserted.", + "files": [ + { + "original": "CONDUCTEURS PLENIERES - ESTIVALES 2025 VF.xlsx", + "file": "estivales-2025-plenieres.xlsx", + "sha256": "0ec4bbf0dc3c35653b337e70a3f60e0b3a0eb4ef9cac88b0f07fe7c54ab74ebd" + }, + { + "original": "Conduite_Moebius_aprem_santémentale.xlsx", + "file": "moebius-apres-midi-sante-mentale.xlsx", + "sha256": "04665327735ceed70347a16b76c9ff46e299ce3a9ec64e4bc2abf249de1561cc" + }, + { + "original": "Conduite_Moebius_matin_pleniere+santé_environnement.xlsx", + "file": "moebius-matin-pleniere-sante-environnement.xlsx", + "sha256": "d28e4979268042034459e50dafb13cc9c5bc2c4ce03b4281972de08361510ee5" + }, + { + "original": "GIFAV2026_Programme Auditorium.xlsx", + "file": "gifav-2026-auditorium.xlsx", + "sha256": "b179d73dd4ac727d5c09b6f01969030335baa9b933138c45942e7db995302796" + }, + { + "original": "conduite pléniere 2_.xlsx", + "file": "pleniere-2.xlsx", + "sha256": "1835dcb69efc3b12ca5d8fcb96138954aaa495e0a1efa23bbc840a595744bca0" + } + ] +}