diff --git a/tools/xlsxread/src/error.rs b/tools/xlsxread/src/error.rs new file mode 100644 index 0000000..e15edc9 --- /dev/null +++ b/tools/xlsxread/src/error.rs @@ -0,0 +1,34 @@ +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum BuildError { + #[error("I/O error for {path}: {source}")] + Io { + path: String, + #[source] + source: std::io::Error, + }, + + #[error("Calamine error for {path}: {source}")] + Calamine { + path: String, + #[source] + source: calamine::Error, + }, + + #[error("SQLite error: {0}")] + Sqlite(#[from] rusqlite::Error), + + #[error("Config parse error: {0}")] + Config(#[from] toml::de::Error), + + #[error("Regex compile error for pattern '{pattern}': {source}")] + Regex { + pattern: String, + #[source] + source: regex::Error, + }, + + #[error("Schema has no sheets in file: {0}")] + NoSheets(String), +} diff --git a/tools/xlsxread/src/reader.rs b/tools/xlsxread/src/reader.rs new file mode 100644 index 0000000..ac71da1 --- /dev/null +++ b/tools/xlsxread/src/reader.rs @@ -0,0 +1,197 @@ +/// Spreadsheet reader: wraps calamine to iterate rows across sheets. +/// +/// Sheet selection mirrors the JS scripts: +/// - sheet_mode = "all" → iterate every sheet (handles HCM/HN 65k overflow in data/) +/// - sheet_mode = "first" → sheet 0 only (data-old/) +/// +/// Header detection mirrors build-lib.js isHeaderRow: +/// row[0].toUpperCase() in {"HO_TEN", "HỌ TÊN", "STT"} +use std::path::Path; + +use calamine::{open_workbook_auto, Data, Reader, Sheets}; + +use crate::config::{DatasetConfig, HeaderCfg, SheetMode}; +use crate::error::BuildError; + +// --------------------------------------------------------------------------- +// Public row representation from calamine +// --------------------------------------------------------------------------- + +pub type RawRow = Vec; + +// --------------------------------------------------------------------------- +// Header detection — mirrors build-lib.js isHeaderRow +// --------------------------------------------------------------------------- + +/// Returns true when the first cell (uppercased) matches one of the configured +/// header tokens. Used to skip the header row on the first row of each sheet. +pub fn is_header_row(row: &[Data], header_cfg: &HeaderCfg) -> bool { + if row.len() < 3 { + return false; + } + let first = row[0].to_string().trim().to_uppercase(); + header_cfg.tokens.iter().any(|t| t.to_uppercase() == first) +} + +// --------------------------------------------------------------------------- +// All-blank row check (data-old2: strip_blank_rows) +// --------------------------------------------------------------------------- + +pub fn is_all_blank(row: &[Data]) -> bool { + row.iter() + .all(|c| matches!(c, Data::Empty) || c.to_string().trim().is_empty()) +} + +// --------------------------------------------------------------------------- +// File processor — yields all data rows from the file +// --------------------------------------------------------------------------- + +/// Process one spreadsheet file, calling `on_row` for each data row. +/// +/// `on_row` receives `(sheet_index, row_index_in_sheet, raw_row)` where +/// `row_index_in_sheet` is 0-based AFTER the header has been consumed. +/// Returns `(sheets_seen, total_rows_yielded)`. +pub fn process_file( + path: &Path, + cfg: &DatasetConfig, + mut on_row: F, +) -> Result<(usize, usize), BuildError> +where + F: FnMut(usize, &RawRow), +{ + let path_str = path.display().to_string(); + + // calamine::open_workbook_auto dispatches on file extension + let mut workbook: Sheets<_> = open_workbook_auto(path).map_err(|e| BuildError::Calamine { + path: path_str.clone(), + source: e, + })?; + + let sheet_names: Vec = workbook.sheet_names().to_vec(); + if sheet_names.is_empty() { + return Err(BuildError::NoSheets(path_str.clone())); + } + + // Sheet selection per config + let sheets_to_read: Vec = match cfg.reader.sheet_mode { + SheetMode::All => sheet_names.clone(), + SheetMode::First => vec![sheet_names[0].clone()], + }; + + let mut total_rows = 0usize; + + for (sheet_idx, sheet_name) in sheets_to_read.iter().enumerate() { + let range = workbook + .worksheet_range(sheet_name) + .map_err(|e| BuildError::Calamine { + path: path_str.clone(), + source: e, + })?; + + let mut first_row = true; + + for raw in range.rows() { + let row: RawRow = raw.to_vec(); + + // Skip header row on first row of each sheet (matches JS: `if (i === 0 && isHeaderRow(...))`) + if first_row { + first_row = false; + if is_header_row(&row, &cfg.header) { + continue; + } + } + + on_row(sheet_idx, &row); + total_rows += 1; + } + } + + Ok((sheets_to_read.len(), total_rows)) +} + +// --------------------------------------------------------------------------- +// Unit tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::HeaderCfg; + + fn hdr(tokens: &[&str]) -> HeaderCfg { + HeaderCfg { + tokens: tokens.iter().map(|s| s.to_string()).collect(), + } + } + + #[test] + fn header_detects_ho_ten() { + let row = vec![ + Data::String("HO_TEN".into()), + Data::String("NGAY_SINH".into()), + Data::String("SBD".into()), + ]; + let cfg = hdr(&["HO_TEN", "HỌ TÊN", "STT"]); + assert!(is_header_row(&row, &cfg)); + } + + #[test] + fn header_detects_stt() { + let row = vec![ + Data::String("STT".into()), + Data::String("B".into()), + Data::String("C".into()), + ]; + let cfg = hdr(&["HO_TEN", "HỌ TÊN", "STT"]); + assert!(is_header_row(&row, &cfg)); + } + + #[test] + fn header_detects_ho_ten_unicode() { + let row = vec![ + Data::String("HỌ TÊN".into()), + Data::String("B".into()), + Data::String("C".into()), + ]; + let cfg = hdr(&["HO_TEN", "HỌ TÊN", "STT"]); + assert!(is_header_row(&row, &cfg)); + } + + #[test] + fn header_rejects_data_row() { + let row = vec![ + Data::String("Nguyen Van A".into()), + Data::String("01/01/2000".into()), + Data::String("12345678".into()), + ]; + let cfg = hdr(&["HO_TEN", "HỌ TÊN", "STT"]); + assert!(!is_header_row(&row, &cfg)); + } + + #[test] + fn header_rejects_short_row() { + let row = vec![Data::String("HO_TEN".into()), Data::Empty]; + let cfg = hdr(&["HO_TEN", "HỌ TÊN", "STT"]); + assert!(!is_header_row(&row, &cfg)); + } + + #[test] + fn header_case_insensitive() { + let row = vec![ + Data::String("ho_ten".into()), + Data::String("B".into()), + Data::String("C".into()), + ]; + let cfg = hdr(&["HO_TEN", "HỌ TÊN", "STT"]); + assert!(is_header_row(&row, &cfg)); + } + + #[test] + fn blank_row_detection() { + let row = vec![Data::Empty, Data::Empty, Data::String("".into())]; + assert!(is_all_blank(&row)); + + let row2 = vec![Data::String("Nguyen".into()), Data::Empty, Data::Empty]; + assert!(!is_all_blank(&row2)); + } +}