thptqg/parser/internal/transform/transform_test.go

package transform

import (
	"testing"

	"github.com/tiennm99/thptqg/parser/internal/config"
	"github.com/tiennm99/thptqg/parser/internal/reader"
)

// --- ToAscii ---

func TestToAscii(t *testing.T) {
	cases := []struct{ name, in, want string }{
		{"plain_latin", "Nguyen Van A", "nguyen van a"},
		{"nguyen_thi_hoa", "Nguyễn Thị Hoa", "nguyen thi hoa"},
		{"tran_van_duc", "Trần Văn Đức", "tran van duc"},
		{"le_thi_my_duyen", "Lê Thị Mỹ Duyên", "le thi my duyen"},
		{"pham_thi_lan", "Phạm Thị Lan", "pham thi lan"},
		{"bui_thi_thu", "Bùi Thị Thu", "bui thi thu"},
		{"hoang_van_truong", "Hoàng Văn Trường", "hoang van truong"},
		{"do_thi_ngan", "Đỗ Thị Ngân", "do thi ngan"},
		{"nguyen_van_khanh", "Nguyễn Văn Khánh", "nguyen van khanh"},
		{"trinh_thi_bich_ngoc", "Trịnh Thị Bích Ngọc", "trinh thi bich ngoc"},
		{"vu_thi_dieu", "Vũ Thị Diệu", "vu thi dieu"},
		{"nguyen_thi_tuong_vi", "Nguyễn Thị Tường Vi", "nguyen thi tuong vi"},
		{"lowercase_d_stroke", "đặng thị hằng", "dang thi hang"},
		{"uppercase_d_stroke", "ĐẶNG THỊ HẰNG", "dang thi hang"},
		{"mixed_case", "NGUYỄN VĂN AN", "nguyen van an"},
		{"tran_thi_kim_anh", "Trần Thị Kim Anh", "tran thi kim anh"},
		{"nguyen_thi_phuong_thao", "Nguyễn Thị Phương Thảo", "nguyen thi phuong thao"},
		{"le_van_long", "Lê Văn Long", "le van long"},
		{"vo_thi_xuan_mai", "Võ Thị Xuân Mai", "vo thi xuan mai"},
		{"empty_string", "", ""},
	}
	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			if got := ToAscii(c.in); got != c.want {
				t.Errorf("ToAscii(%q) = %q, want %q", c.in, got, c.want)
			}
		})
	}
}

// TestToAsciiUsesLiteralRangeNotUnicodeMn guards the highest-value trap in this
// package: ToAscii drops the literal range U+0300..U+036F, not Unicode category
// Mn, which is strictly broader. U+0654 (ARABIC HAMZA ABOVE) is in Mn but outside
// the range, so it must survive.
func TestToAsciiUsesLiteralRangeNotUnicodeMn(t *testing.T) {
	const in = "aٔb"
	if got := ToAscii(in); got != in {
		t.Errorf("ToAscii(%q) = %q — a combining mark outside U+0300..U+036F must survive; "+
			"stripping it means unicode.Mn was used instead of the literal range", in, got)
	}
	// And a mark inside the range must be stripped.
	if got := ToAscii("áb"); got != "ab" {
		t.Errorf("ToAscii(\"a\\u0301b\") = %q, want \"ab\"", got)
	}
}

// TestToAsciiDStrokeIndependentOfNFD proves the đ/Đ replacement is a separate
// step: NFD does not decompose them, so relying on the mark filter alone loses
// the letter entirely.
func TestToAsciiDStrokeIndependentOfNFD(t *testing.T) {
	for _, c := range []struct{ in, want string }{
		{"đ", "d"}, {"Đ", "d"}, {"đĐ", "dd"},
	} {
		if got := ToAscii(c.in); got != c.want {
			t.Errorf("ToAscii(%q) = %q, want %q", c.in, got, c.want)
		}
	}
}

// --- ParseScores ---

func TestParseScoresSingle(t *testing.T) {
	s := ParseScores("Toán: 8.5")
	if v, ok := s["toan"]; !ok || v != 8.5 {
		t.Errorf("toan = %v (present=%v), want 8.5", v, ok)
	}
	if _, ok := s["ngu_van"]; ok {
		t.Error("ngu_van should be absent")
	}
}

func TestParseScoresMultiple(t *testing.T) {
	s := ParseScores("Toán: 7.25  Ngữ văn: 6.0  Vật lí: 9")
	for field, want := range map[string]float64{"toan": 7.25, "ngu_van": 6.0, "vat_ly": 9.0} {
		if v, ok := s[field]; !ok || v != want {
			t.Errorf("%s = %v (present=%v), want %v", field, v, ok, want)
		}
	}
}

func TestParseScoresEmptyCell(t *testing.T) {
	if s := ParseScores(""); len(s) != 0 {
		t.Errorf("ParseScores(\"\") = %v, want empty", s)
	}
}

// TestParseScoresRealCellShape uses the wide space runs seen in the corpus.
func TestParseScoresRealCellShape(t *testing.T) {
	const cell = "Toán:   4.60   Ngữ văn:   5.50   Lịch sử:   4.50   "
	s := ParseScores(cell)
	if len(s) != 3 {
		t.Fatalf("matched %d subjects, want 3: %v", len(s), s)
	}
	if s["toan"] != 4.60 || s["ngu_van"] != 5.50 || s["lich_su"] != 4.50 {
		t.Errorf("got %v", s)
	}
}

// --- ValidateRow ---

func defaultValidation() *config.ValidationCfg {
	return &config.ValidationCfg{
		RequireNumericSbd:   false,
		RequireNonemptyName: true,
		RequireNonemptySbd:  true,
	}
}

func TestValidateOK(t *testing.T) {
	if r := ValidateRow("Nguyen Van A", "12345678", defaultValidation(), false, false); r != SkipNone {
		t.Errorf("got %v, want SkipNone", r)
	}
}

func TestValidateEmptySbd(t *testing.T) {
	if r := ValidateRow("Nguyen Van A", "", defaultValidation(), false, false); r != SkipEmptyField {
		t.Errorf("got %v, want SkipEmptyField", r)
	}
}

func TestValidateEmptyName(t *testing.T) {
	if r := ValidateRow("", "12345678", defaultValidation(), false, false); r != SkipEmptyField {
		t.Errorf("got %v, want SkipEmptyField", r)
	}
}

func TestValidateNonNumericSbdRejected(t *testing.T) {
	v := defaultValidation()
	v.RequireNumericSbd = true
	if r := ValidateRow("Nguyen Van A", "12AB5678", v, false, false); r != SkipNonNumericSbd {
		t.Errorf("got %v, want SkipNonNumericSbd", r)
	}
}

func TestValidateNumericSbdAccepted(t *testing.T) {
	v := defaultValidation()
	v.RequireNumericSbd = true
	if r := ValidateRow("Nguyen Van A", "12345678", v, false, false); r != SkipNone {
		t.Errorf("got %v, want SkipNone", r)
	}
}

func TestValidateBlankRowSkipped(t *testing.T) {
	if r := ValidateRow("", "", defaultValidation(), true, true); r != SkipBlankRow {
		t.Errorf("got %v, want SkipBlankRow", r)
	}
}

// TestValidateNumericSbdIsDigitScanNotAtoi: the check is a digit scan, not
// strconv.Atoi — Atoi accepts a leading sign and would wrongly admit "+123".
func TestValidateNumericSbdIsDigitScanNotAtoi(t *testing.T) {
	v := defaultValidation()
	v.RequireNumericSbd = true
	for _, sbd := range []string{"+123", "-123", "12 3", "1.0", "ABC123", "１２３"} {
		if r := ValidateRow("Nguyen Van A", sbd, v, false, false); r != SkipNonNumericSbd {
			t.Errorf("ValidateRow(sbd=%q) = %v, want SkipNonNumericSbd", sbd, r)
		}
	}
}

// TestValidateBlankRowOnlyWhenStripEnabled: with strip_blank_rows false, an
// all-blank row falls through to the empty-field checks instead.
func TestValidateBlankRowOnlyWhenStripEnabled(t *testing.T) {
	if r := ValidateRow("", "", defaultValidation(), false, true); r != SkipEmptyField {
		t.Errorf("got %v, want SkipEmptyField when strip_blank_rows is off", r)
	}
}

// --- TransformRow ---

func fixedColumnCfg() *config.DatasetConfig {
	return &config.DatasetConfig{
		Columns:    &config.ColumnMap{HoTen: 0, NgaySinh: 1, SoBaoDanh: 2, DiemThi: 3},
		Validation: *defaultValidation(),
	}
}

func cells(vals ...string) []reader.Cell {
	out := make([]reader.Cell, len(vals))
	for i, v := range vals {
		out[i] = reader.Cell{Str: v, IsEmpty: v == ""}
	}
	return out
}

func TestTransformRow(t *testing.T) {
	row := cells("Nguyễn Văn Đức", "04/04/1999", "51002167", "Toán: 8.5  Ngữ văn: 7")
	got, err := TransformRow(row, fixedColumnCfg())
	if err != nil {
		t.Fatalf("TransformRow: %v", err)
	}
	if got.HoTen != "Nguyễn Văn Đức" || got.HoTenAscii != "nguyen van duc" {
		t.Errorf("ho_ten=%q ascii=%q", got.HoTen, got.HoTenAscii)
	}
	if got.SoBaoDanh != "51002167" {
		t.Errorf("so_bao_danh = %q", got.SoBaoDanh)
	}
	if got.NgaySinh == nil || *got.NgaySinh != "04/04/1999" {
		t.Errorf("ngay_sinh = %v", got.NgaySinh)
	}
	// 2016-only columns are never populated on the fixed-column path.
	if got.TenCumThi != nil || got.GioiTinh != nil {
		t.Error("ten_cum_thi and gioi_tinh must stay nil on the 2017 path")
	}
	if got.Scores["toan"] != 8.5 || got.Scores["ngu_van"] != 7 {
		t.Errorf("scores = %v", got.Scores)
	}
}

func TestTransformRowEmptyNgaySinhBecomesNil(t *testing.T) {
	got, err := TransformRow(cells("A", "", "1", ""), fixedColumnCfg())
	if err != nil {
		t.Fatalf("TransformRow: %v", err)
	}
	if got.NgaySinh != nil {
		t.Errorf("empty ngay_sinh should be nil, got %q", *got.NgaySinh)
	}
}

// TestTransformRowShortRowYieldsEmptyFields: a row shorter than the configured
// indices yields empty strings rather than an error.
func TestTransformRowShortRowYieldsEmptyFields(t *testing.T) {
	got, err := TransformRow(cells("OnlyName"), fixedColumnCfg())
	if err != nil {
		t.Fatalf("TransformRow: %v", err)
	}
	if got.HoTen != "OnlyName" || got.SoBaoDanh != "" || got.NgaySinh != nil {
		t.Errorf("got ho_ten=%q sbd=%q ngay_sinh=%v", got.HoTen, got.SoBaoDanh, got.NgaySinh)
	}
}

// TestTransformRowTrimsEveryField: every column is read trimmed, diem_thi
// included. Source cells routinely carry padding — 850k of the 861k 2017 rows
// have whitespace around their score cell.
func TestTransformRowTrimsEveryField(t *testing.T) {
	row := cells("  A  ", "  01/01/2000  ", "  123  ", "   Toán: 5   ")
	got, err := TransformRow(row, fixedColumnCfg())
	if err != nil {
		t.Fatalf("TransformRow: %v", err)
	}
	if got.HoTen != "A" || got.SoBaoDanh != "123" {
		t.Errorf("trimmed fields wrong: ho_ten=%q sbd=%q", got.HoTen, got.SoBaoDanh)
	}
	if got.NgaySinh == nil || *got.NgaySinh != "01/01/2000" {
		t.Errorf("ngay_sinh = %v, want trimmed", got.NgaySinh)
	}
	if got.Scores["toan"] != 5 {
		t.Errorf("scores = %v", got.Scores)
	}
}

// TestTransformRowRequiresColumns: the fixed-column path is only reachable when
// the config has a columns: mapping, and must error rather than guess otherwise.
func TestTransformRowRequiresColumns(t *testing.T) {
	cfg := &config.DatasetConfig{Validation: *defaultValidation()}
	if _, err := TransformRow(cells("A", "B", "C", "D"), cfg); err == nil {
		t.Fatal("TransformRow without a columns: mapping must return an error")
	}
}