Address CSV export and field alignment review

2026-06-24 16:31:35 +08:00
parent 9fab4c73ed
commit 915702f8f3
5 changed files with 304 additions and 16 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,12 @@
 .DS_Store
 bin/
 *.sqlite
+*.sqlite-wal
+*.sqlite-shm
 *.sqlite3
+*.sqlite3-wal
+*.sqlite3-shm
 *.db
+*.db-wal
+*.db-shm
 *.csv
--- a/HOWTO.md
+++ b/HOWTO.md
@@ -73,7 +73,9 @@ database: report.sqlite

 ## 5. 查看字段化内容

-在 macOS/Linux 终端中：
+普通用户优先导出 CSV 后用 WPS/Excel 打开，见下一节。
+
+如果机器上安装了 `sqlite3`，也可以直接查看数据库：

 ```bash
 sqlite3 report.sqlite
@@ -107,7 +109,7 @@ SELECT key, value FROM fields ORDER BY id;
 这是最常用的导出方式：

 ```bash
-sqlite3 -header -csv report.sqlite "SELECT key, value FROM fields ORDER BY id;" > report-fields.csv
+./bin/wordtable2sqlite export-fields -db report.sqlite -out report-fields.csv
 ```

 生成的 `report-fields.csv` 可以直接用 WPS/Excel 打开。
@@ -123,11 +125,12 @@ UTF-8
 如果要检查 Word 表格的原始行列结构，导出 `cells`：

 ```bash
-sqlite3 -header -csv report.sqlite "SELECT table_index, row_index, cell_index, col_index, col_span, text FROM cells ORDER BY table_index, row_index, cell_index;" > report-cells.csv
+./bin/wordtable2sqlite export-cells -db report.sqlite -out report-cells.csv
 ```

 字段含义：

+- `filename`：Word 文件名。
 - `table_index`：第几张表，从 0 开始。
 - `row_index`：第几行，从 0 开始。
 - `cell_index`：该行第几个单元格。
@@ -137,6 +140,8 @@ sqlite3 -header -csv report.sqlite "SELECT table_index, row_index, cell_index, c

 ## 8. 常用查询

+下面这些查询需要本机有 `sqlite3` 或其他 SQLite 查看工具。
+
 查所有字段名：

 ```sql
@@ -180,22 +185,42 @@ ORDER BY table_index, row_index, cell_index;
 2. 查看 `cells` 判断 Word 原始表格结构。
 3. 查看 `fields` 判断字段抽取是否符合预期。

-## 10. 本次样例命令
+## 10. 表格复杂度限制

-本次样例数据库：
+当前字段化抽取主要面向常见报备表：
+
+- 一行标题。
+- 一行表头。
+- 一行或多行数据。
+
+工具会保留所有原始单元格到 `cells`，并按逻辑列处理横向合并的表头。如果 Word 表格包含复杂多行表头、纵向合并单元格、嵌套表格，建议同时导出 `report-cells.csv` 检查原始行列结构。
+
+## 11. 样例命令模板
+
+导入：

 ```bash
-/Users/zhaoyilun/Documents/Codex/2026-06-24/wo-x/outputs/sample-baiyin.sqlite
+./bin/wordtable2sqlite -reset -db report.sqlite input.docx
 ```

 导出字段化 CSV：

 ```bash
-sqlite3 -header -csv /Users/zhaoyilun/Documents/Codex/2026-06-24/wo-x/outputs/sample-baiyin.sqlite "SELECT key, value FROM fields ORDER BY id;" > /Users/zhaoyilun/Documents/Codex/2026-06-24/wo-x/outputs/sample-baiyin-fields.csv
+./bin/wordtable2sqlite export-fields -db report.sqlite -out report-fields.csv
 ```

 导出原始单元格 CSV：

 ```bash
-sqlite3 -header -csv /Users/zhaoyilun/Documents/Codex/2026-06-24/wo-x/outputs/sample-baiyin.sqlite "SELECT table_index, row_index, cell_index, col_index, col_span, text FROM cells ORDER BY table_index, row_index, cell_index;" > /Users/zhaoyilun/Documents/Codex/2026-06-24/wo-x/outputs/sample-baiyin-cells.csv
+./bin/wordtable2sqlite export-cells -db report.sqlite -out report-cells.csv
+```
+
+## 12. 开发者验证
+
+如果需要从源码编译或修改代码，使用：
+
+```bash
+go test ./...
+go vet ./...
+go build -o bin/wordtable2sqlite .
 ```
--- a/README.md
+++ b/README.md
@@ -11,6 +11,7 @@
 - SQLite 使用纯 Go 驱动 `modernc.org/sqlite`，方便编译成单文件可执行程序。
 - 保底保存所有表格单元格到 `cells`。
 - 对常见报备表这种“标题行 + 表头行 + 数据行”的结构，自动生成 `fields`，便于按字段检索。
+- 内置导出 `fields` / `cells` CSV，不要求普通用户额外安装 `sqlite3`。

 ## 编译

@@ -32,12 +33,28 @@ go build -o bin/wordtable2sqlite .
 ./bin/wordtable2sqlite -db data/report.sqlite *.docx
 ```

+导出字段化内容 CSV：
+
+```bash
+./bin/wordtable2sqlite export-fields -db data/report.sqlite -out data/report-fields.csv
+```
+
+导出原始单元格 CSV：
+
+```bash
+./bin/wordtable2sqlite export-cells -db data/report.sqlite -out data/report-cells.csv
+```
+
 参数：

 - `-db`：输出 SQLite 文件路径，默认 `word_tables.sqlite`
 - `-reset`：导入前删除旧数据库
 - `-quiet`：不输出每个文件的摘要

+## 适用范围
+
+当前字段推断优先覆盖常见报备表：一行标题、一行表头、一行或多行数据。横向合并的表头会按逻辑列范围合并对应值。复杂多行表头、纵向合并、嵌套表格仍建议同时查看 `cells` 原始单元格结果。
+
 ## 数据表

 - `documents`：每个导入的 Word 文件。
@@ -58,3 +75,11 @@ SELECT table_index, row_index, cell_index, text
 FROM cells
 ORDER BY table_index, row_index, cell_index;
 ```
+
+## 验证
+
+```bash
+go test ./...
+go vet ./...
+go build -o bin/wordtable2sqlite .
+```
--- a/main.go
+++ b/main.go
@@ -6,6 +6,7 @@ import (
 	"context"
 	"crypto/sha256"
 	"database/sql"
+	"encoding/csv"
 	"encoding/hex"
 	"encoding/xml"
 	"errors"
@@ -57,6 +58,15 @@ func main() {
 }

 func run(args []string) error {
+	if len(args) > 0 {
+		switch args[0] {
+		case "export-fields":
+			return runExportFields(args[1:])
+		case "export-cells":
+			return runExportCells(args[1:])
+		}
+	}
+
 	fs := flag.NewFlagSet("wordtable2sqlite", flag.ContinueOnError)
 	dbPath := fs.String("db", "word_tables.sqlite", "SQLite database path")
 	reset := fs.Bool("reset", false, "remove the target database before importing")
@@ -114,6 +124,100 @@ func run(args []string) error {
 	return nil
 }

+func runExportFields(args []string) error {
+	fs := flag.NewFlagSet("wordtable2sqlite export-fields", flag.ContinueOnError)
+	dbPath := fs.String("db", "word_tables.sqlite", "SQLite database path")
+	outPath := fs.String("out", "", "CSV output path")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	if *outPath == "" {
+		return errors.New("usage: wordtable2sqlite export-fields -db report.sqlite -out report-fields.csv")
+	}
+
+	db, err := sql.Open("sqlite", *dbPath)
+	if err != nil {
+		return fmt.Errorf("open sqlite database: %w", err)
+	}
+	defer db.Close()
+
+	rows, err := db.Query(`SELECT d.filename, f.key, f.value
+		FROM fields f
+		JOIN documents d ON d.id = f.document_id
+		ORDER BY d.id, f.id`)
+	if err != nil {
+		return fmt.Errorf("query fields: %w", err)
+	}
+	defer rows.Close()
+
+	return writeCSV(*outPath, []string{"filename", "key", "value"}, rows, 3)
+}
+
+func runExportCells(args []string) error {
+	fs := flag.NewFlagSet("wordtable2sqlite export-cells", flag.ContinueOnError)
+	dbPath := fs.String("db", "word_tables.sqlite", "SQLite database path")
+	outPath := fs.String("out", "", "CSV output path")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	if *outPath == "" {
+		return errors.New("usage: wordtable2sqlite export-cells -db report.sqlite -out report-cells.csv")
+	}
+
+	db, err := sql.Open("sqlite", *dbPath)
+	if err != nil {
+		return fmt.Errorf("open sqlite database: %w", err)
+	}
+	defer db.Close()
+
+	rows, err := db.Query(`SELECT d.filename, c.table_index, c.row_index, c.cell_index, c.col_index, c.col_span, c.v_merge, c.text
+		FROM cells c
+		JOIN documents d ON d.id = c.document_id
+		ORDER BY d.id, c.table_index, c.row_index, c.cell_index`)
+	if err != nil {
+		return fmt.Errorf("query cells: %w", err)
+	}
+	defer rows.Close()
+
+	return writeCSV(*outPath, []string{"filename", "table_index", "row_index", "cell_index", "col_index", "col_span", "v_merge", "text"}, rows, 8)
+}
+
+func writeCSV(outPath string, header []string, rows *sql.Rows, columns int) error {
+	if dir := filepath.Dir(outPath); dir != "." {
+		if err := os.MkdirAll(dir, 0o755); err != nil {
+			return fmt.Errorf("create csv directory: %w", err)
+		}
+	}
+	f, err := os.Create(outPath)
+	if err != nil {
+		return fmt.Errorf("create csv: %w", err)
+	}
+	defer f.Close()
+
+	w := csv.NewWriter(f)
+	if err := w.Write(header); err != nil {
+		return err
+	}
+	for rows.Next() {
+		values := make([]string, columns)
+		dest := make([]any, columns)
+		for i := range values {
+			dest[i] = &values[i]
+		}
+		if err := rows.Scan(dest...); err != nil {
+			return fmt.Errorf("scan csv row: %w", err)
+		}
+		if err := w.Write(values); err != nil {
+			return err
+		}
+	}
+	if err := rows.Err(); err != nil {
+		return err
+	}
+	w.Flush()
+	return w.Error()
+}
+
 type stats struct {
 	Tables int
 	Cells  int
@@ -518,7 +622,7 @@ func nonEmptyCellCount(row Row) int {
 }

 func inferHeaderFields(table Table, headerIndex int) []Field {
-	headers := table.Rows[headerIndex].Cells
+	headers := cellsWithLogicalColumns(table.Rows[headerIndex])
 	var fields []Field
 	recordIndex := 0
 	for rowIndex := headerIndex + 1; rowIndex < len(table.Rows); rowIndex++ {
@@ -526,19 +630,17 @@ func inferHeaderFields(table Table, headerIndex int) []Field {
 		if nonEmptyCellCount(row) == 0 {
 			continue
 		}
-		for i, header := range headers {
-			if i >= len(row.Cells) {
-				continue
-			}
+		rowCells := cellsWithLogicalColumns(row)
+		for _, header := range headers {
 			key := cleanFieldKey(header.Text)
 			if key == "" {
 				continue
 			}
-			value := strings.TrimSpace(row.Cells[i].Text)
+			value := cellsTextForRange(rowCells, header.ColIndex, header.ColIndex+span(header))
 			fields = append(fields, Field{
 				RecordIndex: recordIndex,
 				RowIndex:    rowIndex,
-				ColIndex:    row.Cells[i].ColIndex,
+				ColIndex:    header.ColIndex,
 				Key:         key,
 				Value:       value,
 				SourceKind:  "header_row",
@@ -549,6 +651,61 @@ func inferHeaderFields(table Table, headerIndex int) []Field {
 	return fields
 }

+func cellsWithLogicalColumns(row Row) []Cell {
+	out := make([]Cell, len(row.Cells))
+	copy(out, row.Cells)
+
+	needsReindex := false
+	nextCol := 0
+	for i := range out {
+		if out[i].ColSpan < 1 {
+			out[i].ColSpan = 1
+		}
+		if i == 0 {
+			needsReindex = out[i].ColIndex != 0
+		} else if out[i].ColIndex < nextCol {
+			needsReindex = true
+		}
+		nextCol = out[i].ColIndex + out[i].ColSpan
+	}
+	if !needsReindex {
+		return out
+	}
+
+	nextCol = 0
+	for i := range out {
+		out[i].ColIndex = nextCol
+		nextCol += out[i].ColSpan
+	}
+	return out
+}
+
+func cellsTextForRange(cells []Cell, start, end int) string {
+	var values []string
+	for _, cell := range cells {
+		cellStart := cell.ColIndex
+		cellEnd := cell.ColIndex + span(cell)
+		if cellStart >= end || cellEnd <= start {
+			continue
+		}
+		text := strings.TrimSpace(cell.Text)
+		if text == "" {
+			continue
+		}
+		if len(values) == 0 || values[len(values)-1] != text {
+			values = append(values, text)
+		}
+	}
+	return strings.Join(values, "\n")
+}
+
+func span(cell Cell) int {
+	if cell.ColSpan < 1 {
+		return 1
+	}
+	return cell.ColSpan
+}
+
 func inferKeyValueFields(table Table) []Field {
 	var fields []Field
 	recordIndex := 0
@@ -588,7 +745,7 @@ func nonEmptyCells(row Row) []Cell {

 func cleanFieldKey(s string) string {
 	s = strings.TrimSpace(s)
-	s = strings.TrimRight(s, "：: \t\r\n")
+	s = strings.Trim(s, "：: \t\r\n")
 	return strings.TrimSpace(s)
 }

--- a/main_test.go
+++ b/main_test.go
@@ -4,6 +4,7 @@ import (
 	"archive/zip"
 	"context"
 	"database/sql"
+	"encoding/csv"
 	"os"
 	"path/filepath"
 	"testing"
@@ -70,6 +71,27 @@ func TestInferHeaderFieldsSkipsTitleRow(t *testing.T) {
 	}
 }

+func TestInferHeaderFieldsAlignsByLogicalColumnWithColSpan(t *testing.T) {
+	table := Table{
+		Rows: []Row{
+			{Cells: []Cell{{Text: "序号", ColIndex: 0, ColSpan: 1}, {Text: "客户信息", ColIndex: 1, ColSpan: 2}, {Text: "工单编号", ColIndex: 3, ColSpan: 1}}},
+			{Cells: []Cell{{Text: "1", ColIndex: 0, ColSpan: 1}, {Text: "高先生", ColIndex: 1, ColSpan: 1}, {Text: "靖远县乌兰镇", ColIndex: 2, ColSpan: 1}, {Text: "9726060110063361", ColIndex: 3, ColSpan: 1}}},
+		},
+	}
+
+	fields := inferFields(table)
+	got := fieldMap(fields)
+	if got["序号"] != "1" {
+		t.Fatalf("序号 = %q, want 1", got["序号"])
+	}
+	if got["客户信息"] != "高先生\n靖远县乌兰镇" {
+		t.Fatalf("客户信息 = %q, want combined logical-column value", got["客户信息"])
+	}
+	if got["工单编号"] != "9726060110063361" {
+		t.Fatalf("工单编号 = %q, want 9726060110063361", got["工单编号"])
+	}
+}
+
 func TestImportDocumentWritesSQLiteTables(t *testing.T) {
 	dbPath := filepath.Join(t.TempDir(), "import.sqlite")
 	db, err := sql.Open("sqlite", dbPath)
@@ -115,6 +137,59 @@ func TestImportDocumentWritesSQLiteTables(t *testing.T) {
 	}
 }

+func TestRunExportsFieldsCSV(t *testing.T) {
+	dir := t.TempDir()
+	dbPath := filepath.Join(dir, "import.sqlite")
+	outPath := filepath.Join(dir, "fields.csv")
+	xml := `<?xml version="1.0" encoding="UTF-8"?>
+<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main">
+  <w:body>
+    <w:tbl>
+      <w:tr><w:tc><w:p><w:r><w:t>字段</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:t>内容</w:t></w:r></w:p></w:tc></w:tr>
+      <w:tr><w:tc><w:p><w:r><w:t>工单编号</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:t>9726060110063361</w:t></w:r></w:p></w:tc></w:tr>
+    </w:tbl>
+  </w:body>
+</w:document>`
+	docxPath := filepath.Join(dir, "sample.docx")
+	if err := writeTestDocx(docxPath, []byte(xml)); err != nil {
+		t.Fatalf("write test docx: %v", err)
+	}
+
+	if err := run([]string{"-reset", "-quiet", "-db", dbPath, docxPath}); err != nil {
+		t.Fatalf("import run: %v", err)
+	}
+	if err := run([]string{"export-fields", "-db", dbPath, "-out", outPath}); err != nil {
+		t.Fatalf("export run: %v", err)
+	}
+
+	f, err := os.Open(outPath)
+	if err != nil {
+		t.Fatalf("open csv: %v", err)
+	}
+	defer f.Close()
+	records, err := csv.NewReader(f).ReadAll()
+	if err != nil {
+		t.Fatalf("read csv: %v", err)
+	}
+	if len(records) != 3 {
+		t.Fatalf("got %d csv records, want header + 2 fields: %#v", len(records), records)
+	}
+	if records[0][0] != "filename" || records[0][1] != "key" || records[0][2] != "value" {
+		t.Fatalf("unexpected header: %#v", records[0])
+	}
+	if records[2][1] != "内容" || records[2][2] != "9726060110063361" {
+		t.Fatalf("unexpected exported field: %#v", records[2])
+	}
+}
+
+func fieldMap(fields []Field) map[string]string {
+	out := map[string]string{}
+	for _, field := range fields {
+		out[field.Key] = field.Value
+	}
+	return out
+}
+
 func writeTestDocx(path string, documentXML []byte) error {
 	f, err := os.Create(path)
 	if err != nil {