diff --git a/.gitignore b/.gitignore index 7f7a20c..bbf6208 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,12 @@ .DS_Store bin/ *.sqlite +*.sqlite-wal +*.sqlite-shm *.sqlite3 +*.sqlite3-wal +*.sqlite3-shm *.db +*.db-wal +*.db-shm *.csv diff --git a/HOWTO.md b/HOWTO.md index cf4c906..9756bfa 100644 --- a/HOWTO.md +++ b/HOWTO.md @@ -73,7 +73,9 @@ database: report.sqlite ## 5. 查看字段化内容 -在 macOS/Linux 终端中： +普通用户优先导出 CSV 后用 WPS/Excel 打开，见下一节。 + +如果机器上安装了 `sqlite3`，也可以直接查看数据库： ```bash sqlite3 report.sqlite @@ -107,7 +109,7 @@ SELECT key, value FROM fields ORDER BY id; 这是最常用的导出方式： ```bash -sqlite3 -header -csv report.sqlite "SELECT key, value FROM fields ORDER BY id;" > report-fields.csv +./bin/wordtable2sqlite export-fields -db report.sqlite -out report-fields.csv ``` 生成的 `report-fields.csv` 可以直接用 WPS/Excel 打开。 @@ -123,11 +125,12 @@ UTF-8 如果要检查 Word 表格的原始行列结构，导出 `cells`： ```bash -sqlite3 -header -csv report.sqlite "SELECT table_index, row_index, cell_index, col_index, col_span, text FROM cells ORDER BY table_index, row_index, cell_index;" > report-cells.csv +./bin/wordtable2sqlite export-cells -db report.sqlite -out report-cells.csv ``` 字段含义： +- `filename`：Word 文件名。 - `table_index`：第几张表，从 0 开始。 - `row_index`：第几行，从 0 开始。 - `cell_index`：该行第几个单元格。 @@ -137,6 +140,8 @@ sqlite3 -header -csv report.sqlite "SELECT table_index, row_index, cell_index, c ## 8. 常用查询 +下面这些查询需要本机有 `sqlite3` 或其他 SQLite 查看工具。 + 查所有字段名： ```sql @@ -180,22 +185,42 @@ ORDER BY table_index, row_index, cell_index; 2. 查看 `cells` 判断 Word 原始表格结构。 3. 查看 `fields` 判断字段抽取是否符合预期。 -## 10. 本次样例命令 +## 10. 表格复杂度限制 -本次样例数据库： +当前字段化抽取主要面向常见报备表： + +- 一行标题。 +- 一行表头。 +- 一行或多行数据。 + +工具会保留所有原始单元格到 `cells`，并按逻辑列处理横向合并的表头。如果 Word 表格包含复杂多行表头、纵向合并单元格、嵌套表格，建议同时导出 `report-cells.csv` 检查原始行列结构。 + +## 11. 样例命令模板 + +导入： ```bash -/Users/zhaoyilun/Documents/Codex/2026-06-24/wo-x/outputs/sample-baiyin.sqlite +./bin/wordtable2sqlite -reset -db report.sqlite input.docx ``` 导出字段化 CSV： ```bash -sqlite3 -header -csv /Users/zhaoyilun/Documents/Codex/2026-06-24/wo-x/outputs/sample-baiyin.sqlite "SELECT key, value FROM fields ORDER BY id;" > /Users/zhaoyilun/Documents/Codex/2026-06-24/wo-x/outputs/sample-baiyin-fields.csv +./bin/wordtable2sqlite export-fields -db report.sqlite -out report-fields.csv ``` 导出原始单元格 CSV： ```bash -sqlite3 -header -csv /Users/zhaoyilun/Documents/Codex/2026-06-24/wo-x/outputs/sample-baiyin.sqlite "SELECT table_index, row_index, cell_index, col_index, col_span, text FROM cells ORDER BY table_index, row_index, cell_index;" > /Users/zhaoyilun/Documents/Codex/2026-06-24/wo-x/outputs/sample-baiyin-cells.csv +./bin/wordtable2sqlite export-cells -db report.sqlite -out report-cells.csv +``` + +## 12. 开发者验证 + +如果需要从源码编译或修改代码，使用： + +```bash +go test ./... +go vet ./... +go build -o bin/wordtable2sqlite . ``` diff --git a/README.md b/README.md index daed3ae..ce7afc2 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ - SQLite 使用纯 Go 驱动 `modernc.org/sqlite`，方便编译成单文件可执行程序。 - 保底保存所有表格单元格到 `cells`。 - 对常见报备表这种“标题行 + 表头行 + 数据行”的结构，自动生成 `fields`，便于按字段检索。 +- 内置导出 `fields` / `cells` CSV，不要求普通用户额外安装 `sqlite3`。 ## 编译 @@ -32,12 +33,28 @@ go build -o bin/wordtable2sqlite . ./bin/wordtable2sqlite -db data/report.sqlite *.docx ``` +导出字段化内容 CSV： + +```bash +./bin/wordtable2sqlite export-fields -db data/report.sqlite -out data/report-fields.csv +``` + +导出原始单元格 CSV： + +```bash +./bin/wordtable2sqlite export-cells -db data/report.sqlite -out data/report-cells.csv +``` + 参数： - `-db`：输出 SQLite 文件路径，默认 `word_tables.sqlite` - `-reset`：导入前删除旧数据库 - `-quiet`：不输出每个文件的摘要 +## 适用范围 + +当前字段推断优先覆盖常见报备表：一行标题、一行表头、一行或多行数据。横向合并的表头会按逻辑列范围合并对应值。复杂多行表头、纵向合并、嵌套表格仍建议同时查看 `cells` 原始单元格结果。 + ## 数据表 - `documents`：每个导入的 Word 文件。 @@ -58,3 +75,11 @@ SELECT table_index, row_index, cell_index, text FROM cells ORDER BY table_index, row_index, cell_index; ``` + +## 验证 + +```bash +go test ./... +go vet ./... +go build -o bin/wordtable2sqlite . +``` diff --git a/main.go b/main.go index dfb5965..d8e56bc 100644 --- a/main.go +++ b/main.go @@ -6,6 +6,7 @@ import ( "context" "crypto/sha256" "database/sql" + "encoding/csv" "encoding/hex" "encoding/xml" "errors" @@ -57,6 +58,15 @@ func main() { } func run(args []string) error { + if len(args) > 0 { + switch args[0] { + case "export-fields": + return runExportFields(args[1:]) + case "export-cells": + return runExportCells(args[1:]) + } + } + fs := flag.NewFlagSet("wordtable2sqlite", flag.ContinueOnError) dbPath := fs.String("db", "word_tables.sqlite", "SQLite database path") reset := fs.Bool("reset", false, "remove the target database before importing") @@ -114,6 +124,100 @@ func run(args []string) error { return nil } +func runExportFields(args []string) error { + fs := flag.NewFlagSet("wordtable2sqlite export-fields", flag.ContinueOnError) + dbPath := fs.String("db", "word_tables.sqlite", "SQLite database path") + outPath := fs.String("out", "", "CSV output path") + if err := fs.Parse(args); err != nil { + return err + } + if *outPath == "" { + return errors.New("usage: wordtable2sqlite export-fields -db report.sqlite -out report-fields.csv") + } + + db, err := sql.Open("sqlite", *dbPath) + if err != nil { + return fmt.Errorf("open sqlite database: %w", err) + } + defer db.Close() + + rows, err := db.Query(`SELECT d.filename, f.key, f.value + FROM fields f + JOIN documents d ON d.id = f.document_id + ORDER BY d.id, f.id`) + if err != nil { + return fmt.Errorf("query fields: %w", err) + } + defer rows.Close() + + return writeCSV(*outPath, []string{"filename", "key", "value"}, rows, 3) +} + +func runExportCells(args []string) error { + fs := flag.NewFlagSet("wordtable2sqlite export-cells", flag.ContinueOnError) + dbPath := fs.String("db", "word_tables.sqlite", "SQLite database path") + outPath := fs.String("out", "", "CSV output path") + if err := fs.Parse(args); err != nil { + return err + } + if *outPath == "" { + return errors.New("usage: wordtable2sqlite export-cells -db report.sqlite -out report-cells.csv") + } + + db, err := sql.Open("sqlite", *dbPath) + if err != nil { + return fmt.Errorf("open sqlite database: %w", err) + } + defer db.Close() + + rows, err := db.Query(`SELECT d.filename, c.table_index, c.row_index, c.cell_index, c.col_index, c.col_span, c.v_merge, c.text + FROM cells c + JOIN documents d ON d.id = c.document_id + ORDER BY d.id, c.table_index, c.row_index, c.cell_index`) + if err != nil { + return fmt.Errorf("query cells: %w", err) + } + defer rows.Close() + + return writeCSV(*outPath, []string{"filename", "table_index", "row_index", "cell_index", "col_index", "col_span", "v_merge", "text"}, rows, 8) +} + +func writeCSV(outPath string, header []string, rows *sql.Rows, columns int) error { + if dir := filepath.Dir(outPath); dir != "." { + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("create csv directory: %w", err) + } + } + f, err := os.Create(outPath) + if err != nil { + return fmt.Errorf("create csv: %w", err) + } + defer f.Close() + + w := csv.NewWriter(f) + if err := w.Write(header); err != nil { + return err + } + for rows.Next() { + values := make([]string, columns) + dest := make([]any, columns) + for i := range values { + dest[i] = &values[i] + } + if err := rows.Scan(dest...); err != nil { + return fmt.Errorf("scan csv row: %w", err) + } + if err := w.Write(values); err != nil { + return err + } + } + if err := rows.Err(); err != nil { + return err + } + w.Flush() + return w.Error() +} + type stats struct { Tables int Cells int @@ -518,7 +622,7 @@ func nonEmptyCellCount(row Row) int { } func inferHeaderFields(table Table, headerIndex int) []Field { - headers := table.Rows[headerIndex].Cells + headers := cellsWithLogicalColumns(table.Rows[headerIndex]) var fields []Field recordIndex := 0 for rowIndex := headerIndex + 1; rowIndex < len(table.Rows); rowIndex++ { @@ -526,19 +630,17 @@ func inferHeaderFields(table Table, headerIndex int) []Field { if nonEmptyCellCount(row) == 0 { continue } - for i, header := range headers { - if i >= len(row.Cells) { - continue - } + rowCells := cellsWithLogicalColumns(row) + for _, header := range headers { key := cleanFieldKey(header.Text) if key == "" { continue } - value := strings.TrimSpace(row.Cells[i].Text) + value := cellsTextForRange(rowCells, header.ColIndex, header.ColIndex+span(header)) fields = append(fields, Field{ RecordIndex: recordIndex, RowIndex: rowIndex, - ColIndex: row.Cells[i].ColIndex, + ColIndex: header.ColIndex, Key: key, Value: value, SourceKind: "header_row", @@ -549,6 +651,61 @@ func inferHeaderFields(table Table, headerIndex int) []Field { return fields } +func cellsWithLogicalColumns(row Row) []Cell { + out := make([]Cell, len(row.Cells)) + copy(out, row.Cells) + + needsReindex := false + nextCol := 0 + for i := range out { + if out[i].ColSpan < 1 { + out[i].ColSpan = 1 + } + if i == 0 { + needsReindex = out[i].ColIndex != 0 + } else if out[i].ColIndex < nextCol { + needsReindex = true + } + nextCol = out[i].ColIndex + out[i].ColSpan + } + if !needsReindex { + return out + } + + nextCol = 0 + for i := range out { + out[i].ColIndex = nextCol + nextCol += out[i].ColSpan + } + return out +} + +func cellsTextForRange(cells []Cell, start, end int) string { + var values []string + for _, cell := range cells { + cellStart := cell.ColIndex + cellEnd := cell.ColIndex + span(cell) + if cellStart >= end || cellEnd <= start { + continue + } + text := strings.TrimSpace(cell.Text) + if text == "" { + continue + } + if len(values) == 0 || values[len(values)-1] != text { + values = append(values, text) + } + } + return strings.Join(values, "\n") +} + +func span(cell Cell) int { + if cell.ColSpan < 1 { + return 1 + } + return cell.ColSpan +} + func inferKeyValueFields(table Table) []Field { var fields []Field recordIndex := 0 @@ -588,7 +745,7 @@ func nonEmptyCells(row Row) []Cell { func cleanFieldKey(s string) string { s = strings.TrimSpace(s) - s = strings.TrimRight(s, "：: \t\r\n") + s = strings.Trim(s, "：: \t\r\n") return strings.TrimSpace(s) } diff --git a/main_test.go b/main_test.go index 6f78480..e0af3b7 100644 --- a/main_test.go +++ b/main_test.go @@ -4,6 +4,7 @@ import ( "archive/zip" "context" "database/sql" + "encoding/csv" "os" "path/filepath" "testing" @@ -70,6 +71,27 @@ func TestInferHeaderFieldsSkipsTitleRow(t *testing.T) { } } +func TestInferHeaderFieldsAlignsByLogicalColumnWithColSpan(t *testing.T) { + table := Table{ + Rows: []Row{ + {Cells: []Cell{{Text: "序号", ColIndex: 0, ColSpan: 1}, {Text: "客户信息", ColIndex: 1, ColSpan: 2}, {Text: "工单编号", ColIndex: 3, ColSpan: 1}}}, + {Cells: []Cell{{Text: "1", ColIndex: 0, ColSpan: 1}, {Text: "高先生", ColIndex: 1, ColSpan: 1}, {Text: "靖远县乌兰镇", ColIndex: 2, ColSpan: 1}, {Text: "9726060110063361", ColIndex: 3, ColSpan: 1}}}, + }, + } + + fields := inferFields(table) + got := fieldMap(fields) + if got["序号"] != "1" { + t.Fatalf("序号 = %q, want 1", got["序号"]) + } + if got["客户信息"] != "高先生\n靖远县乌兰镇" { + t.Fatalf("客户信息 = %q, want combined logical-column value", got["客户信息"]) + } + if got["工单编号"] != "9726060110063361" { + t.Fatalf("工单编号 = %q, want 9726060110063361", got["工单编号"]) + } +} + func TestImportDocumentWritesSQLiteTables(t *testing.T) { dbPath := filepath.Join(t.TempDir(), "import.sqlite") db, err := sql.Open("sqlite", dbPath) @@ -115,6 +137,59 @@ func TestImportDocumentWritesSQLiteTables(t *testing.T) { } } +func TestRunExportsFieldsCSV(t *testing.T) { + dir := t.TempDir() + dbPath := filepath.Join(dir, "import.sqlite") + outPath := filepath.Join(dir, "fields.csv") + xml := ` + + + + 字段内容 + 工单编号9726060110063361 + + +` + docxPath := filepath.Join(dir, "sample.docx") + if err := writeTestDocx(docxPath, []byte(xml)); err != nil { + t.Fatalf("write test docx: %v", err) + } + + if err := run([]string{"-reset", "-quiet", "-db", dbPath, docxPath}); err != nil { + t.Fatalf("import run: %v", err) + } + if err := run([]string{"export-fields", "-db", dbPath, "-out", outPath}); err != nil { + t.Fatalf("export run: %v", err) + } + + f, err := os.Open(outPath) + if err != nil { + t.Fatalf("open csv: %v", err) + } + defer f.Close() + records, err := csv.NewReader(f).ReadAll() + if err != nil { + t.Fatalf("read csv: %v", err) + } + if len(records) != 3 { + t.Fatalf("got %d csv records, want header + 2 fields: %#v", len(records), records) + } + if records[0][0] != "filename" || records[0][1] != "key" || records[0][2] != "value" { + t.Fatalf("unexpected header: %#v", records[0]) + } + if records[2][1] != "内容" || records[2][2] != "9726060110063361" { + t.Fatalf("unexpected exported field: %#v", records[2]) + } +} + +func fieldMap(fields []Field) map[string]string { + out := map[string]string{} + for _, field := range fields { + out[field.Key] = field.Value + } + return out +} + func writeTestDocx(path string, documentXML []byte) error { f, err := os.Create(path) if err != nil {