Address CSV export and field alignment review

main.go

@@ -6,6 +6,7 @@ import (
 	"context"
 	"crypto/sha256"
 	"database/sql"
+	"encoding/csv"
 	"encoding/hex"
 	"encoding/xml"
 	"errors"
@@ -57,6 +58,15 @@ func main() {
 }
 
 func run(args []string) error {
+	if len(args) > 0 {
+		switch args[0] {
+		case "export-fields":
+			return runExportFields(args[1:])
+		case "export-cells":
+			return runExportCells(args[1:])
+		}
+	}
+
 	fs := flag.NewFlagSet("wordtable2sqlite", flag.ContinueOnError)
 	dbPath := fs.String("db", "word_tables.sqlite", "SQLite database path")
 	reset := fs.Bool("reset", false, "remove the target database before importing")
@@ -114,6 +124,100 @@ func run(args []string) error {
 	return nil
 }
 
+func runExportFields(args []string) error {
+	fs := flag.NewFlagSet("wordtable2sqlite export-fields", flag.ContinueOnError)
+	dbPath := fs.String("db", "word_tables.sqlite", "SQLite database path")
+	outPath := fs.String("out", "", "CSV output path")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	if *outPath == "" {
+		return errors.New("usage: wordtable2sqlite export-fields -db report.sqlite -out report-fields.csv")
+	}
+
+	db, err := sql.Open("sqlite", *dbPath)
+	if err != nil {
+		return fmt.Errorf("open sqlite database: %w", err)
+	}
+	defer db.Close()
+
+	rows, err := db.Query(`SELECT d.filename, f.key, f.value
+		FROM fields f
+		JOIN documents d ON d.id = f.document_id
+		ORDER BY d.id, f.id`)
+	if err != nil {
+		return fmt.Errorf("query fields: %w", err)
+	}
+	defer rows.Close()
+
+	return writeCSV(*outPath, []string{"filename", "key", "value"}, rows, 3)
+}
+
+func runExportCells(args []string) error {
+	fs := flag.NewFlagSet("wordtable2sqlite export-cells", flag.ContinueOnError)
+	dbPath := fs.String("db", "word_tables.sqlite", "SQLite database path")
+	outPath := fs.String("out", "", "CSV output path")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	if *outPath == "" {
+		return errors.New("usage: wordtable2sqlite export-cells -db report.sqlite -out report-cells.csv")
+	}
+
+	db, err := sql.Open("sqlite", *dbPath)
+	if err != nil {
+		return fmt.Errorf("open sqlite database: %w", err)
+	}
+	defer db.Close()
+
+	rows, err := db.Query(`SELECT d.filename, c.table_index, c.row_index, c.cell_index, c.col_index, c.col_span, c.v_merge, c.text
+		FROM cells c
+		JOIN documents d ON d.id = c.document_id
+		ORDER BY d.id, c.table_index, c.row_index, c.cell_index`)
+	if err != nil {
+		return fmt.Errorf("query cells: %w", err)
+	}
+	defer rows.Close()
+
+	return writeCSV(*outPath, []string{"filename", "table_index", "row_index", "cell_index", "col_index", "col_span", "v_merge", "text"}, rows, 8)
+}
+
+func writeCSV(outPath string, header []string, rows *sql.Rows, columns int) error {
+	if dir := filepath.Dir(outPath); dir != "." {
+		if err := os.MkdirAll(dir, 0o755); err != nil {
+			return fmt.Errorf("create csv directory: %w", err)
+		}
+	}
+	f, err := os.Create(outPath)
+	if err != nil {
+		return fmt.Errorf("create csv: %w", err)
+	}
+	defer f.Close()
+
+	w := csv.NewWriter(f)
+	if err := w.Write(header); err != nil {
+		return err
+	}
+	for rows.Next() {
+		values := make([]string, columns)
+		dest := make([]any, columns)
+		for i := range values {
+			dest[i] = &values[i]
+		}
+		if err := rows.Scan(dest...); err != nil {
+			return fmt.Errorf("scan csv row: %w", err)
+		}
+		if err := w.Write(values); err != nil {
+			return err
+		}
+	}
+	if err := rows.Err(); err != nil {
+		return err
+	}
+	w.Flush()
+	return w.Error()
+}
+
 type stats struct {
 	Tables int
 	Cells  int
@@ -518,7 +622,7 @@ func nonEmptyCellCount(row Row) int {
 }
 
 func inferHeaderFields(table Table, headerIndex int) []Field {
-	headers := table.Rows[headerIndex].Cells
+	headers := cellsWithLogicalColumns(table.Rows[headerIndex])
 	var fields []Field
 	recordIndex := 0
 	for rowIndex := headerIndex + 1; rowIndex < len(table.Rows); rowIndex++ {
@@ -526,19 +630,17 @@ func inferHeaderFields(table Table, headerIndex int) []Field {
 		if nonEmptyCellCount(row) == 0 {
 			continue
 		}
-		for i, header := range headers {
-			if i >= len(row.Cells) {
-				continue
-			}
+		rowCells := cellsWithLogicalColumns(row)
+		for _, header := range headers {
 			key := cleanFieldKey(header.Text)
 			if key == "" {
 				continue
 			}
-			value := strings.TrimSpace(row.Cells[i].Text)
+			value := cellsTextForRange(rowCells, header.ColIndex, header.ColIndex+span(header))
 			fields = append(fields, Field{
 				RecordIndex: recordIndex,
 				RowIndex:    rowIndex,
-				ColIndex:    row.Cells[i].ColIndex,
+				ColIndex:    header.ColIndex,
 				Key:         key,
 				Value:       value,
 				SourceKind:  "header_row",
@@ -549,6 +651,61 @@ func inferHeaderFields(table Table, headerIndex int) []Field {
 	return fields
 }
 
+func cellsWithLogicalColumns(row Row) []Cell {
+	out := make([]Cell, len(row.Cells))
+	copy(out, row.Cells)
+
+	needsReindex := false
+	nextCol := 0
+	for i := range out {
+		if out[i].ColSpan < 1 {
+			out[i].ColSpan = 1
+		}
+		if i == 0 {
+			needsReindex = out[i].ColIndex != 0
+		} else if out[i].ColIndex < nextCol {
+			needsReindex = true
+		}
+		nextCol = out[i].ColIndex + out[i].ColSpan
+	}
+	if !needsReindex {
+		return out
+	}
+
+	nextCol = 0
+	for i := range out {
+		out[i].ColIndex = nextCol
+		nextCol += out[i].ColSpan
+	}
+	return out
+}
+
+func cellsTextForRange(cells []Cell, start, end int) string {
+	var values []string
+	for _, cell := range cells {
+		cellStart := cell.ColIndex
+		cellEnd := cell.ColIndex + span(cell)
+		if cellStart >= end || cellEnd <= start {
+			continue
+		}
+		text := strings.TrimSpace(cell.Text)
+		if text == "" {
+			continue
+		}
+		if len(values) == 0 || values[len(values)-1] != text {
+			values = append(values, text)
+		}
+	}
+	return strings.Join(values, "\n")
+}
+
+func span(cell Cell) int {
+	if cell.ColSpan < 1 {
+		return 1
+	}
+	return cell.ColSpan
+}
+
 func inferKeyValueFields(table Table) []Field {
 	var fields []Field
 	recordIndex := 0
@@ -588,7 +745,7 @@ func nonEmptyCells(row Row) []Cell {
 
 func cleanFieldKey(s string) string {
 	s = strings.TrimSpace(s)
-	s = strings.TrimRight(s, "：: \t\r\n")
+	s = strings.Trim(s, "：: \t\r\n")
 	return strings.TrimSpace(s)
 }