From 9fab4c73ed82bee047cf4e469820a4c026da7759 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B5=B5=E4=B9=89=E4=BB=91?= Date: Wed, 24 Jun 2026 16:21:57 +0800 Subject: [PATCH] Initial word table SQLite importer --- .gitignore | 6 + HOWTO.md | 201 +++++++++++++++++ README.md | 60 +++++ go.mod | 18 ++ go.sum | 49 +++++ main.go | 606 +++++++++++++++++++++++++++++++++++++++++++++++++++ main_test.go | 134 ++++++++++++ 7 files changed, 1074 insertions(+) create mode 100644 .gitignore create mode 100644 HOWTO.md create mode 100644 README.md create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go create mode 100644 main_test.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7f7a20c --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +.DS_Store +bin/ +*.sqlite +*.sqlite3 +*.db +*.csv diff --git a/HOWTO.md b/HOWTO.md new file mode 100644 index 0000000..cf4c906 --- /dev/null +++ b/HOWTO.md @@ -0,0 +1,201 @@ +# HOWTO: Word 表格导入 SQLite 并导出字段化 CSV + +这份说明面向实际使用：把报备表 `.docx` 导入 SQLite，然后把自动识别出的字段内容导出成 CSV，方便用 WPS/Excel 查看。 + +## 1. 准备文件 + +需要两个文件： + +- `wordtable2sqlite`：macOS 可执行文件。 +- `要导入的 .docx`：例如重要服务事项报备表。 + +在 Windows 上使用时，对应可执行文件是： + +- `wordtable2sqlite-windows-amd64.exe` + +## 2. 导入单个 Word 文件 + +macOS 示例： + +```bash +./bin/wordtable2sqlite -reset -db report.sqlite input.docx +``` + +Windows 示例： + +```powershell +.\wordtable2sqlite-windows-amd64.exe -reset -db report.sqlite input.docx +``` + +参数含义： + +- `-db report.sqlite`：指定输出的 SQLite 数据库文件。 +- `-reset`：导入前删除旧数据库，适合重新生成结果。 +- `input.docx`：要导入的 Word 文件路径。 + +执行成功后会看到类似输出： + +```text +imported input.docx: 1 tables, 31 cells, 15 fields +database: report.sqlite +``` + +含义是： + +- `tables`：识别到几张 Word 表格。 +- `cells`：保存了多少个原始单元格。 +- `fields`：自动抽取了多少个字段化内容。 + +## 3. 批量导入多个 Word 文件 + +把多个 `.docx` 导入到同一个数据库： + +```bash +./bin/wordtable2sqlite -reset -db reports.sqlite *.docx +``` + +如果是后续追加导入，不想清空旧数据，就不要加 `-reset`： + +```bash +./bin/wordtable2sqlite -db reports.sqlite another.docx +``` + +## 4. 数据库里有什么表 + +导入后会生成 4 张主要表： + +- `documents`：每个导入的 Word 文件。 +- `tables`：每个 Word 文件里的表格。 +- `cells`：原始单元格内容，保留行号、列号、合并单元格信息。 +- `fields`：字段化内容，最适合业务人员查看和检索。 + +一般看业务内容，优先看 `fields`。 + +## 5. 查看字段化内容 + +在 macOS/Linux 终端中： + +```bash +sqlite3 report.sqlite +``` + +进入 SQLite 后执行： + +```sql +.headers on +.mode line +SELECT key, value FROM fields ORDER BY id; +``` + +`.mode line` 适合看长文本。每条记录会显示成： + +```text + key = 事件过程 +value = ... +``` + +如果想看表格模式： + +```sql +.headers on +.mode box +SELECT key, value FROM fields ORDER BY id; +``` + +## 6. 导出字段化内容 CSV + +这是最常用的导出方式： + +```bash +sqlite3 -header -csv report.sqlite "SELECT key, value FROM fields ORDER BY id;" > report-fields.csv +``` + +生成的 `report-fields.csv` 可以直接用 WPS/Excel 打开。 + +如果打开后中文乱码，使用 WPS/Excel 的“从文本/CSV 导入”，编码选择： + +```text +UTF-8 +``` + +## 7. 导出原始单元格 CSV + +如果要检查 Word 表格的原始行列结构，导出 `cells`： + +```bash +sqlite3 -header -csv report.sqlite "SELECT table_index, row_index, cell_index, col_index, col_span, text FROM cells ORDER BY table_index, row_index, cell_index;" > report-cells.csv +``` + +字段含义： + +- `table_index`：第几张表，从 0 开始。 +- `row_index`：第几行，从 0 开始。 +- `cell_index`：该行第几个单元格。 +- `col_index`：逻辑列号。 +- `col_span`：横向合并了几列。 +- `text`：单元格文字。 + +## 8. 常用查询 + +查所有字段名： + +```sql +SELECT key FROM fields ORDER BY id; +``` + +查某个字段： + +```sql +SELECT value FROM fields WHERE key = '工单编号'; +``` + +按文件查看字段： + +```sql +SELECT d.filename, f.key, f.value +FROM fields f +JOIN documents d ON d.id = f.document_id +ORDER BY d.id, f.id; +``` + +查原始单元格： + +```sql +SELECT row_index, cell_index, text +FROM cells +ORDER BY table_index, row_index, cell_index; +``` + +## 9. 推荐查看方式 + +给业务人员看： + +1. 先导出 `report-fields.csv`。 +2. 用 WPS/Excel 打开。 +3. 每一行就是一个字段：`key` 是字段名，`value` 是字段内容。 + +给开发或调试看： + +1. 打开 `report.sqlite`。 +2. 查看 `cells` 判断 Word 原始表格结构。 +3. 查看 `fields` 判断字段抽取是否符合预期。 + +## 10. 本次样例命令 + +本次样例数据库： + +```bash +/Users/zhaoyilun/Documents/Codex/2026-06-24/wo-x/outputs/sample-baiyin.sqlite +``` + +导出字段化 CSV： + +```bash +sqlite3 -header -csv /Users/zhaoyilun/Documents/Codex/2026-06-24/wo-x/outputs/sample-baiyin.sqlite "SELECT key, value FROM fields ORDER BY id;" > /Users/zhaoyilun/Documents/Codex/2026-06-24/wo-x/outputs/sample-baiyin-fields.csv +``` + +导出原始单元格 CSV： + +```bash +sqlite3 -header -csv /Users/zhaoyilun/Documents/Codex/2026-06-24/wo-x/outputs/sample-baiyin.sqlite "SELECT table_index, row_index, cell_index, col_index, col_span, text FROM cells ORDER BY table_index, row_index, cell_index;" > /Users/zhaoyilun/Documents/Codex/2026-06-24/wo-x/outputs/sample-baiyin-cells.csv +``` diff --git a/README.md b/README.md new file mode 100644 index 0000000..daed3ae --- /dev/null +++ b/README.md @@ -0,0 +1,60 @@ +# wordtable2sqlite + +把 `.docx` 里的 Word 表格内容导入 SQLite 的最小 Go 工具。 + +详细操作说明见：[HOWTO.md](HOWTO.md) + +## 特点 + +- 不依赖 Word、WPS、Python 或 LibreOffice。 +- 直接解析 `.docx` 内部的 `word/document.xml`。 +- SQLite 使用纯 Go 驱动 `modernc.org/sqlite`，方便编译成单文件可执行程序。 +- 保底保存所有表格单元格到 `cells`。 +- 对常见报备表这种“标题行 + 表头行 + 数据行”的结构，自动生成 `fields`，便于按字段检索。 + +## 编译 + +```bash +go build -o bin/wordtable2sqlite . +``` + +给非技术用户使用时，分发 `bin/wordtable2sqlite` 这个可执行文件即可。 + +## 使用 + +```bash +./bin/wordtable2sqlite -reset -db data/report.sqlite input.docx +``` + +批量导入多个文件： + +```bash +./bin/wordtable2sqlite -db data/report.sqlite *.docx +``` + +参数： + +- `-db`：输出 SQLite 文件路径，默认 `word_tables.sqlite` +- `-reset`：导入前删除旧数据库 +- `-quiet`：不输出每个文件的摘要 + +## 数据表 + +- `documents`：每个导入的 Word 文件。 +- `tables`：Word 文档中的表格。 +- `cells`：每个表格单元格，包含表格序号、行号、列号、跨列数和文本。 +- `fields`：自动推断出的字段，常用于报备表检索。 + +常用查询： + +```sql +SELECT key, value +FROM fields +ORDER BY id; +``` + +```sql +SELECT table_index, row_index, cell_index, text +FROM cells +ORDER BY table_index, row_index, cell_index; +``` diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..f78ef59 --- /dev/null +++ b/go.mod @@ -0,0 +1,18 @@ +module wordtable2sqlite + +go 1.26 + +require modernc.org/sqlite v1.40.1 + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect + golang.org/x/sys v0.36.0 // indirect + modernc.org/libc v1.66.10 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..86155cd --- /dev/null +++ b/go.sum @@ -0,0 +1,49 @@ +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o= +golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= +golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= +golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= +golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= +golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= +modernc.org/cc/v4 v4.26.5 h1:xM3bX7Mve6G8K8b+T11ReenJOT+BmVqQj0FY5T4+5Y4= +modernc.org/cc/v4 v4.26.5/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= +modernc.org/ccgo/v4 v4.28.1 h1:wPKYn5EC/mYTqBO373jKjvX2n+3+aK7+sICCv4Fjy1A= +modernc.org/ccgo/v4 v4.28.1/go.mod h1:uD+4RnfrVgE6ec9NGguUNdhqzNIeeomeXf6CL0GTE5Q= +modernc.org/fileutil v1.3.40 h1:ZGMswMNc9JOCrcrakF1HrvmergNLAmxOPjizirpfqBA= +modernc.org/fileutil v1.3.40/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.66.10 h1:yZkb3YeLx4oynyR+iUsXsybsX4Ubx7MQlSYEw4yj59A= +modernc.org/libc v1.66.10/go.mod h1:8vGSEwvoUoltr4dlywvHqjtAqHBaw0j1jI7iFBTAr2I= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= +modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.40.1 h1:VfuXcxcUWWKRBuP8+BR9L7VnmusMgBNNnBYGEe9w/iY= +modernc.org/sqlite v1.40.1/go.mod h1:9fjQZ0mB1LLP0GYrp39oOJXx/I2sxEnZtzCmEQIKvGE= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/main.go b/main.go new file mode 100644 index 0000000..dfb5965 --- /dev/null +++ b/main.go @@ -0,0 +1,606 @@ +package main + +import ( + "archive/zip" + "bytes" + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "encoding/xml" + "errors" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + _ "modernc.org/sqlite" +) + +type Table struct { + Index int + Rows []Row +} + +type Row struct { + Index int + Cells []Cell +} + +type Cell struct { + RowIndex int + CellIndex int + ColIndex int + ColSpan int + VMerge string + Text string +} + +type Field struct { + RecordIndex int + RowIndex int + ColIndex int + Key string + Value string + SourceKind string +} + +func main() { + if err := run(os.Args[1:]); err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(1) + } +} + +func run(args []string) error { + fs := flag.NewFlagSet("wordtable2sqlite", flag.ContinueOnError) + dbPath := fs.String("db", "word_tables.sqlite", "SQLite database path") + reset := fs.Bool("reset", false, "remove the target database before importing") + quiet := fs.Bool("quiet", false, "suppress per-file summary output") + if err := fs.Parse(args); err != nil { + return err + } + docxPaths := fs.Args() + if len(docxPaths) == 0 { + return errors.New("usage: wordtable2sqlite -db output.sqlite [-reset] file1.docx [file2.docx ...]") + } + + if *reset { + if err := os.Remove(*dbPath); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("reset database: %w", err) + } + } + if dir := filepath.Dir(*dbPath); dir != "." { + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("create database directory: %w", err) + } + } + + db, err := sql.Open("sqlite", *dbPath) + if err != nil { + return fmt.Errorf("open sqlite database: %w", err) + } + defer db.Close() + + ctx := context.Background() + for _, docxPath := range docxPaths { + before, err := dbStats(ctx, db) + if err != nil { + return err + } + if err := importDocument(ctx, db, docxPath); err != nil { + return err + } + after, err := dbStats(ctx, db) + if err != nil { + return err + } + if !*quiet { + fmt.Printf("imported %s: %d tables, %d cells, %d fields\n", + docxPath, + after.Tables-before.Tables, + after.Cells-before.Cells, + after.Fields-before.Fields, + ) + } + } + if !*quiet { + fmt.Printf("database: %s\n", *dbPath) + } + return nil +} + +type stats struct { + Tables int + Cells int + Fields int +} + +func dbStats(ctx context.Context, db *sql.DB) (stats, error) { + if err := ensureSchema(ctx, db); err != nil { + return stats{}, err + } + var s stats + for _, item := range []struct { + query string + dst *int + }{ + {`SELECT COUNT(*) FROM tables`, &s.Tables}, + {`SELECT COUNT(*) FROM cells`, &s.Cells}, + {`SELECT COUNT(*) FROM fields`, &s.Fields}, + } { + if err := db.QueryRowContext(ctx, item.query).Scan(item.dst); err != nil { + return stats{}, err + } + } + return s, nil +} + +func importDocument(ctx context.Context, db *sql.DB, docxPath string) error { + if err := ensureSchema(ctx, db); err != nil { + return err + } + tables, err := readDocxTables(docxPath) + if err != nil { + return err + } + sum, err := fileSHA256(docxPath) + if err != nil { + return err + } + + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + + res, err := tx.ExecContext(ctx, + `INSERT INTO documents(path, filename, sha256, imported_at) VALUES (?, ?, ?, ?)`, + docxPath, + filepath.Base(docxPath), + sum, + time.Now().Format(time.RFC3339), + ) + if err != nil { + return fmt.Errorf("insert document: %w", err) + } + documentID, err := res.LastInsertId() + if err != nil { + return err + } + + for tableIndex, table := range tables { + cellCount := 0 + for _, row := range table.Rows { + cellCount += len(row.Cells) + } + res, err := tx.ExecContext(ctx, + `INSERT INTO tables(document_id, table_index, row_count, cell_count) VALUES (?, ?, ?, ?)`, + documentID, + tableIndex, + len(table.Rows), + cellCount, + ) + if err != nil { + return fmt.Errorf("insert table %d: %w", tableIndex, err) + } + tableID, err := res.LastInsertId() + if err != nil { + return err + } + + for rowIndex, row := range table.Rows { + for cellIndex, cell := range row.Cells { + if _, err := tx.ExecContext(ctx, + `INSERT INTO cells(document_id, table_id, table_index, row_index, cell_index, col_index, col_span, v_merge, text) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + documentID, + tableID, + tableIndex, + rowIndex, + cellIndex, + cell.ColIndex, + cell.ColSpan, + cell.VMerge, + cell.Text, + ); err != nil { + return fmt.Errorf("insert cell table=%d row=%d cell=%d: %w", tableIndex, rowIndex, cellIndex, err) + } + } + } + + for _, field := range inferFields(table) { + if _, err := tx.ExecContext(ctx, + `INSERT INTO fields(document_id, table_id, table_index, record_index, row_index, col_index, key, value, source_kind) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + documentID, + tableID, + tableIndex, + field.RecordIndex, + field.RowIndex, + field.ColIndex, + field.Key, + field.Value, + field.SourceKind, + ); err != nil { + return fmt.Errorf("insert field %q: %w", field.Key, err) + } + } + } + + return tx.Commit() +} + +func ensureSchema(ctx context.Context, db execer) error { + statements := []string{ + `PRAGMA foreign_keys = ON`, + `CREATE TABLE IF NOT EXISTS documents ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + path TEXT NOT NULL, + filename TEXT NOT NULL, + sha256 TEXT NOT NULL, + imported_at TEXT NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS tables ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + document_id INTEGER NOT NULL REFERENCES documents(id) ON DELETE CASCADE, + table_index INTEGER NOT NULL, + row_count INTEGER NOT NULL, + cell_count INTEGER NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS cells ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + document_id INTEGER NOT NULL REFERENCES documents(id) ON DELETE CASCADE, + table_id INTEGER NOT NULL REFERENCES tables(id) ON DELETE CASCADE, + table_index INTEGER NOT NULL, + row_index INTEGER NOT NULL, + cell_index INTEGER NOT NULL, + col_index INTEGER NOT NULL, + col_span INTEGER NOT NULL, + v_merge TEXT NOT NULL DEFAULT '', + text TEXT NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS fields ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + document_id INTEGER NOT NULL REFERENCES documents(id) ON DELETE CASCADE, + table_id INTEGER NOT NULL REFERENCES tables(id) ON DELETE CASCADE, + table_index INTEGER NOT NULL, + record_index INTEGER NOT NULL, + row_index INTEGER NOT NULL, + col_index INTEGER NOT NULL, + key TEXT NOT NULL, + value TEXT NOT NULL, + source_kind TEXT NOT NULL + )`, + `CREATE INDEX IF NOT EXISTS idx_cells_doc_table ON cells(document_id, table_index, row_index, cell_index)`, + `CREATE INDEX IF NOT EXISTS idx_fields_doc_key ON fields(document_id, key)`, + } + for _, statement := range statements { + if _, err := db.ExecContext(ctx, statement); err != nil { + return fmt.Errorf("schema statement failed: %w", err) + } + } + return nil +} + +type execer interface { + ExecContext(context.Context, string, ...any) (sql.Result, error) +} + +func readDocxTables(path string) ([]Table, error) { + zr, err := zip.OpenReader(path) + if err != nil { + return nil, fmt.Errorf("open docx zip: %w", err) + } + defer zr.Close() + + for _, file := range zr.File { + if file.Name != "word/document.xml" { + continue + } + rc, err := file.Open() + if err != nil { + return nil, err + } + defer rc.Close() + data, err := io.ReadAll(rc) + if err != nil { + return nil, err + } + return parseDocumentXML(data) + } + return nil, errors.New("word/document.xml not found in docx") +} + +func parseDocumentXML(data []byte) ([]Table, error) { + decoder := xml.NewDecoder(bytes.NewReader(data)) + var tables []Table + for { + token, err := decoder.Token() + if errors.Is(err, io.EOF) { + return tables, nil + } + if err != nil { + return nil, err + } + start, ok := token.(xml.StartElement) + if !ok || start.Name.Local != "tbl" { + continue + } + table, err := parseTable(decoder) + if err != nil { + return nil, err + } + table.Index = len(tables) + tables = append(tables, table) + } +} + +func parseTable(decoder *xml.Decoder) (Table, error) { + var table Table + for { + token, err := decoder.Token() + if err != nil { + return table, err + } + switch t := token.(type) { + case xml.StartElement: + if t.Name.Local == "tr" { + row, err := parseRow(decoder, len(table.Rows)) + if err != nil { + return table, err + } + table.Rows = append(table.Rows, row) + } + case xml.EndElement: + if t.Name.Local == "tbl" { + return table, nil + } + } + } +} + +func parseRow(decoder *xml.Decoder, rowIndex int) (Row, error) { + row := Row{Index: rowIndex} + logicalCol := 0 + for { + token, err := decoder.Token() + if err != nil { + return row, err + } + switch t := token.(type) { + case xml.StartElement: + if t.Name.Local == "tc" { + cell, err := parseCell(decoder) + if err != nil { + return row, err + } + cell.RowIndex = rowIndex + cell.CellIndex = len(row.Cells) + cell.ColIndex = logicalCol + if cell.ColSpan < 1 { + cell.ColSpan = 1 + } + logicalCol += cell.ColSpan + row.Cells = append(row.Cells, cell) + } + case xml.EndElement: + if t.Name.Local == "tr" { + return row, nil + } + } + } +} + +func parseCell(decoder *xml.Decoder) (Cell, error) { + cell := Cell{ColSpan: 1} + var b strings.Builder + inText := false + tcDepth := 1 + for { + token, err := decoder.Token() + if err != nil { + return cell, err + } + switch t := token.(type) { + case xml.StartElement: + switch t.Name.Local { + case "tc": + tcDepth++ + case "gridSpan": + if tcDepth == 1 { + if span, err := strconv.Atoi(attr(t, "val")); err == nil && span > 0 { + cell.ColSpan = span + } + } + case "vMerge": + if tcDepth == 1 { + cell.VMerge = attr(t, "val") + if cell.VMerge == "" { + cell.VMerge = "continue" + } + } + case "t": + inText = true + case "tab": + b.WriteByte('\t') + case "br", "cr": + b.WriteByte('\n') + } + case xml.CharData: + if inText { + b.Write([]byte(t)) + } + case xml.EndElement: + switch t.Name.Local { + case "t": + inText = false + case "p": + if b.Len() > 0 && !strings.HasSuffix(b.String(), "\n") { + b.WriteByte('\n') + } + case "tc": + tcDepth-- + if tcDepth == 0 { + cell.Text = normalizeCellText(b.String()) + return cell, nil + } + } + } + } +} + +func attr(start xml.StartElement, local string) string { + for _, a := range start.Attr { + if a.Name.Local == local { + return a.Value + } + } + return "" +} + +func normalizeCellText(s string) string { + s = strings.ReplaceAll(s, "\r\n", "\n") + s = strings.ReplaceAll(s, "\r", "\n") + lines := strings.Split(s, "\n") + out := make([]string, 0, len(lines)) + for _, line := range lines { + line = strings.TrimSpace(strings.Join(strings.Fields(line), " ")) + if line != "" { + out = append(out, line) + } + } + return strings.Join(out, "\n") +} + +func inferFields(table Table) []Field { + headerIndex := detectHeaderRow(table) + if headerIndex >= 0 { + return inferHeaderFields(table, headerIndex) + } + return inferKeyValueFields(table) +} + +func detectHeaderRow(table Table) int { + for i := 0; i < len(table.Rows)-1; i++ { + row := table.Rows[i] + if isTitleRow(row) { + continue + } + if nonEmptyCellCount(row) >= 2 && nonEmptyCellCount(table.Rows[i+1]) >= 2 { + return i + } + } + return -1 +} + +func isTitleRow(row Row) bool { + nonEmpty := nonEmptyCellCount(row) + if nonEmpty != 1 || len(row.Cells) == 0 { + return false + } + return len(row.Cells) == 1 || row.Cells[0].ColSpan > 1 +} + +func nonEmptyCellCount(row Row) int { + count := 0 + for _, cell := range row.Cells { + if strings.TrimSpace(cell.Text) != "" { + count++ + } + } + return count +} + +func inferHeaderFields(table Table, headerIndex int) []Field { + headers := table.Rows[headerIndex].Cells + var fields []Field + recordIndex := 0 + for rowIndex := headerIndex + 1; rowIndex < len(table.Rows); rowIndex++ { + row := table.Rows[rowIndex] + if nonEmptyCellCount(row) == 0 { + continue + } + for i, header := range headers { + if i >= len(row.Cells) { + continue + } + key := cleanFieldKey(header.Text) + if key == "" { + continue + } + value := strings.TrimSpace(row.Cells[i].Text) + fields = append(fields, Field{ + RecordIndex: recordIndex, + RowIndex: rowIndex, + ColIndex: row.Cells[i].ColIndex, + Key: key, + Value: value, + SourceKind: "header_row", + }) + } + recordIndex++ + } + return fields +} + +func inferKeyValueFields(table Table) []Field { + var fields []Field + recordIndex := 0 + for rowIndex, row := range table.Rows { + cells := nonEmptyCells(row) + if len(cells) < 2 || len(cells)%2 != 0 { + continue + } + for i := 0; i < len(cells); i += 2 { + key := cleanFieldKey(cells[i].Text) + if key == "" { + continue + } + fields = append(fields, Field{ + RecordIndex: recordIndex, + RowIndex: rowIndex, + ColIndex: cells[i+1].ColIndex, + Key: key, + Value: cells[i+1].Text, + SourceKind: "key_value", + }) + } + recordIndex++ + } + return fields +} + +func nonEmptyCells(row Row) []Cell { + out := make([]Cell, 0, len(row.Cells)) + for _, cell := range row.Cells { + if strings.TrimSpace(cell.Text) != "" { + out = append(out, cell) + } + } + return out +} + +func cleanFieldKey(s string) string { + s = strings.TrimSpace(s) + s = strings.TrimRight(s, "：: \t\r\n") + return strings.TrimSpace(s) +} + +func fileSHA256(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} diff --git a/main_test.go b/main_test.go new file mode 100644 index 0000000..6f78480 --- /dev/null +++ b/main_test.go @@ -0,0 +1,134 @@ +package main + +import ( + "archive/zip" + "context" + "database/sql" + "os" + "path/filepath" + "testing" + + _ "modernc.org/sqlite" +) + +func TestParseDocumentXMLExtractsTableCells(t *testing.T) { + xml := ` + + + + + 报备表 + + + 序号 + 事件过程 + + + 1 + 起因处理经过 + + + +` + + tables, err := parseDocumentXML([]byte(xml)) + if err != nil { + t.Fatalf("parseDocumentXML returned error: %v", err) + } + if len(tables) != 1 { + t.Fatalf("got %d tables, want 1", len(tables)) + } + if got := tables[0].Rows[0].Cells[0].Text; got != "报备表" { + t.Fatalf("title cell text = %q, want 报备表", got) + } + if got := tables[0].Rows[0].Cells[0].ColSpan; got != 2 { + t.Fatalf("title cell colspan = %d, want 2", got) + } + if got := tables[0].Rows[2].Cells[1].Text; got != "起因\n处理经过" { + t.Fatalf("multiline text = %q, want 起因\\n处理经过", got) + } +} + +func TestInferHeaderFieldsSkipsTitleRow(t *testing.T) { + table := Table{ + Rows: []Row{ + {Cells: []Cell{{Text: "报备表", ColSpan: 2}}}, + {Cells: []Cell{{Text: "序号", ColSpan: 1}, {Text: "事件过程", ColSpan: 1}}}, + {Cells: []Cell{{Text: "1", ColSpan: 1}, {Text: "客户反映设备位置不合理", ColSpan: 1}}}, + }, + } + + fields := inferFields(table) + if len(fields) != 2 { + t.Fatalf("got %d fields, want 2", len(fields)) + } + if fields[0].Key != "序号" || fields[0].Value != "1" || fields[0].SourceKind != "header_row" { + t.Fatalf("unexpected first field: %#v", fields[0]) + } + if fields[1].Key != "事件过程" || fields[1].Value != "客户反映设备位置不合理" { + t.Fatalf("unexpected second field: %#v", fields[1]) + } +} + +func TestImportDocumentWritesSQLiteTables(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "import.sqlite") + db, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + defer db.Close() + + xml := ` + + + + 报备表 + 序号事件过程 + 1客户反映设备位置不合理 + + +` + + docxPath := filepath.Join(t.TempDir(), "sample.docx") + if err := writeTestDocx(docxPath, []byte(xml)); err != nil { + t.Fatalf("write test docx: %v", err) + } + + if err := importDocument(context.Background(), db, docxPath); err != nil { + t.Fatalf("importDocument: %v", err) + } + + var cells int + if err := db.QueryRow(`SELECT COUNT(*) FROM cells`).Scan(&cells); err != nil { + t.Fatalf("count cells: %v", err) + } + if cells != 5 { + t.Fatalf("got %d cells, want 5", cells) + } + + var value string + if err := db.QueryRow(`SELECT value FROM fields WHERE key = '事件过程'`).Scan(&value); err != nil { + t.Fatalf("query field: %v", err) + } + if value != "客户反映设备位置不合理" { + t.Fatalf("field value = %q", value) + } +} + +func writeTestDocx(path string, documentXML []byte) error { + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + + zw := zip.NewWriter(f) + defer zw.Close() + + w, err := zw.Create("word/document.xml") + if err != nil { + return err + } + _, err = w.Write(documentXML) + return err +}