607 lines
13 KiB
Go
607 lines
13 KiB
Go
package main
|
||
|
||
import (
|
||
"archive/zip"
|
||
"bytes"
|
||
"context"
|
||
"crypto/sha256"
|
||
"database/sql"
|
||
"encoding/hex"
|
||
"encoding/xml"
|
||
"errors"
|
||
"flag"
|
||
"fmt"
|
||
"io"
|
||
"os"
|
||
"path/filepath"
|
||
"strconv"
|
||
"strings"
|
||
"time"
|
||
|
||
_ "modernc.org/sqlite"
|
||
)
|
||
|
||
type Table struct {
|
||
Index int
|
||
Rows []Row
|
||
}
|
||
|
||
type Row struct {
|
||
Index int
|
||
Cells []Cell
|
||
}
|
||
|
||
type Cell struct {
|
||
RowIndex int
|
||
CellIndex int
|
||
ColIndex int
|
||
ColSpan int
|
||
VMerge string
|
||
Text string
|
||
}
|
||
|
||
type Field struct {
|
||
RecordIndex int
|
||
RowIndex int
|
||
ColIndex int
|
||
Key string
|
||
Value string
|
||
SourceKind string
|
||
}
|
||
|
||
func main() {
|
||
if err := run(os.Args[1:]); err != nil {
|
||
fmt.Fprintln(os.Stderr, "error:", err)
|
||
os.Exit(1)
|
||
}
|
||
}
|
||
|
||
func run(args []string) error {
|
||
fs := flag.NewFlagSet("wordtable2sqlite", flag.ContinueOnError)
|
||
dbPath := fs.String("db", "word_tables.sqlite", "SQLite database path")
|
||
reset := fs.Bool("reset", false, "remove the target database before importing")
|
||
quiet := fs.Bool("quiet", false, "suppress per-file summary output")
|
||
if err := fs.Parse(args); err != nil {
|
||
return err
|
||
}
|
||
docxPaths := fs.Args()
|
||
if len(docxPaths) == 0 {
|
||
return errors.New("usage: wordtable2sqlite -db output.sqlite [-reset] file1.docx [file2.docx ...]")
|
||
}
|
||
|
||
if *reset {
|
||
if err := os.Remove(*dbPath); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||
return fmt.Errorf("reset database: %w", err)
|
||
}
|
||
}
|
||
if dir := filepath.Dir(*dbPath); dir != "." {
|
||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||
return fmt.Errorf("create database directory: %w", err)
|
||
}
|
||
}
|
||
|
||
db, err := sql.Open("sqlite", *dbPath)
|
||
if err != nil {
|
||
return fmt.Errorf("open sqlite database: %w", err)
|
||
}
|
||
defer db.Close()
|
||
|
||
ctx := context.Background()
|
||
for _, docxPath := range docxPaths {
|
||
before, err := dbStats(ctx, db)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if err := importDocument(ctx, db, docxPath); err != nil {
|
||
return err
|
||
}
|
||
after, err := dbStats(ctx, db)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if !*quiet {
|
||
fmt.Printf("imported %s: %d tables, %d cells, %d fields\n",
|
||
docxPath,
|
||
after.Tables-before.Tables,
|
||
after.Cells-before.Cells,
|
||
after.Fields-before.Fields,
|
||
)
|
||
}
|
||
}
|
||
if !*quiet {
|
||
fmt.Printf("database: %s\n", *dbPath)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
type stats struct {
|
||
Tables int
|
||
Cells int
|
||
Fields int
|
||
}
|
||
|
||
func dbStats(ctx context.Context, db *sql.DB) (stats, error) {
|
||
if err := ensureSchema(ctx, db); err != nil {
|
||
return stats{}, err
|
||
}
|
||
var s stats
|
||
for _, item := range []struct {
|
||
query string
|
||
dst *int
|
||
}{
|
||
{`SELECT COUNT(*) FROM tables`, &s.Tables},
|
||
{`SELECT COUNT(*) FROM cells`, &s.Cells},
|
||
{`SELECT COUNT(*) FROM fields`, &s.Fields},
|
||
} {
|
||
if err := db.QueryRowContext(ctx, item.query).Scan(item.dst); err != nil {
|
||
return stats{}, err
|
||
}
|
||
}
|
||
return s, nil
|
||
}
|
||
|
||
func importDocument(ctx context.Context, db *sql.DB, docxPath string) error {
|
||
if err := ensureSchema(ctx, db); err != nil {
|
||
return err
|
||
}
|
||
tables, err := readDocxTables(docxPath)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
sum, err := fileSHA256(docxPath)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
|
||
tx, err := db.BeginTx(ctx, nil)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer tx.Rollback()
|
||
|
||
res, err := tx.ExecContext(ctx,
|
||
`INSERT INTO documents(path, filename, sha256, imported_at) VALUES (?, ?, ?, ?)`,
|
||
docxPath,
|
||
filepath.Base(docxPath),
|
||
sum,
|
||
time.Now().Format(time.RFC3339),
|
||
)
|
||
if err != nil {
|
||
return fmt.Errorf("insert document: %w", err)
|
||
}
|
||
documentID, err := res.LastInsertId()
|
||
if err != nil {
|
||
return err
|
||
}
|
||
|
||
for tableIndex, table := range tables {
|
||
cellCount := 0
|
||
for _, row := range table.Rows {
|
||
cellCount += len(row.Cells)
|
||
}
|
||
res, err := tx.ExecContext(ctx,
|
||
`INSERT INTO tables(document_id, table_index, row_count, cell_count) VALUES (?, ?, ?, ?)`,
|
||
documentID,
|
||
tableIndex,
|
||
len(table.Rows),
|
||
cellCount,
|
||
)
|
||
if err != nil {
|
||
return fmt.Errorf("insert table %d: %w", tableIndex, err)
|
||
}
|
||
tableID, err := res.LastInsertId()
|
||
if err != nil {
|
||
return err
|
||
}
|
||
|
||
for rowIndex, row := range table.Rows {
|
||
for cellIndex, cell := range row.Cells {
|
||
if _, err := tx.ExecContext(ctx,
|
||
`INSERT INTO cells(document_id, table_id, table_index, row_index, cell_index, col_index, col_span, v_merge, text)
|
||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||
documentID,
|
||
tableID,
|
||
tableIndex,
|
||
rowIndex,
|
||
cellIndex,
|
||
cell.ColIndex,
|
||
cell.ColSpan,
|
||
cell.VMerge,
|
||
cell.Text,
|
||
); err != nil {
|
||
return fmt.Errorf("insert cell table=%d row=%d cell=%d: %w", tableIndex, rowIndex, cellIndex, err)
|
||
}
|
||
}
|
||
}
|
||
|
||
for _, field := range inferFields(table) {
|
||
if _, err := tx.ExecContext(ctx,
|
||
`INSERT INTO fields(document_id, table_id, table_index, record_index, row_index, col_index, key, value, source_kind)
|
||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||
documentID,
|
||
tableID,
|
||
tableIndex,
|
||
field.RecordIndex,
|
||
field.RowIndex,
|
||
field.ColIndex,
|
||
field.Key,
|
||
field.Value,
|
||
field.SourceKind,
|
||
); err != nil {
|
||
return fmt.Errorf("insert field %q: %w", field.Key, err)
|
||
}
|
||
}
|
||
}
|
||
|
||
return tx.Commit()
|
||
}
|
||
|
||
func ensureSchema(ctx context.Context, db execer) error {
|
||
statements := []string{
|
||
`PRAGMA foreign_keys = ON`,
|
||
`CREATE TABLE IF NOT EXISTS documents (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
path TEXT NOT NULL,
|
||
filename TEXT NOT NULL,
|
||
sha256 TEXT NOT NULL,
|
||
imported_at TEXT NOT NULL
|
||
)`,
|
||
`CREATE TABLE IF NOT EXISTS tables (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
document_id INTEGER NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
|
||
table_index INTEGER NOT NULL,
|
||
row_count INTEGER NOT NULL,
|
||
cell_count INTEGER NOT NULL
|
||
)`,
|
||
`CREATE TABLE IF NOT EXISTS cells (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
document_id INTEGER NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
|
||
table_id INTEGER NOT NULL REFERENCES tables(id) ON DELETE CASCADE,
|
||
table_index INTEGER NOT NULL,
|
||
row_index INTEGER NOT NULL,
|
||
cell_index INTEGER NOT NULL,
|
||
col_index INTEGER NOT NULL,
|
||
col_span INTEGER NOT NULL,
|
||
v_merge TEXT NOT NULL DEFAULT '',
|
||
text TEXT NOT NULL
|
||
)`,
|
||
`CREATE TABLE IF NOT EXISTS fields (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
document_id INTEGER NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
|
||
table_id INTEGER NOT NULL REFERENCES tables(id) ON DELETE CASCADE,
|
||
table_index INTEGER NOT NULL,
|
||
record_index INTEGER NOT NULL,
|
||
row_index INTEGER NOT NULL,
|
||
col_index INTEGER NOT NULL,
|
||
key TEXT NOT NULL,
|
||
value TEXT NOT NULL,
|
||
source_kind TEXT NOT NULL
|
||
)`,
|
||
`CREATE INDEX IF NOT EXISTS idx_cells_doc_table ON cells(document_id, table_index, row_index, cell_index)`,
|
||
`CREATE INDEX IF NOT EXISTS idx_fields_doc_key ON fields(document_id, key)`,
|
||
}
|
||
for _, statement := range statements {
|
||
if _, err := db.ExecContext(ctx, statement); err != nil {
|
||
return fmt.Errorf("schema statement failed: %w", err)
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
type execer interface {
|
||
ExecContext(context.Context, string, ...any) (sql.Result, error)
|
||
}
|
||
|
||
func readDocxTables(path string) ([]Table, error) {
|
||
zr, err := zip.OpenReader(path)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("open docx zip: %w", err)
|
||
}
|
||
defer zr.Close()
|
||
|
||
for _, file := range zr.File {
|
||
if file.Name != "word/document.xml" {
|
||
continue
|
||
}
|
||
rc, err := file.Open()
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rc.Close()
|
||
data, err := io.ReadAll(rc)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
return parseDocumentXML(data)
|
||
}
|
||
return nil, errors.New("word/document.xml not found in docx")
|
||
}
|
||
|
||
func parseDocumentXML(data []byte) ([]Table, error) {
|
||
decoder := xml.NewDecoder(bytes.NewReader(data))
|
||
var tables []Table
|
||
for {
|
||
token, err := decoder.Token()
|
||
if errors.Is(err, io.EOF) {
|
||
return tables, nil
|
||
}
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
start, ok := token.(xml.StartElement)
|
||
if !ok || start.Name.Local != "tbl" {
|
||
continue
|
||
}
|
||
table, err := parseTable(decoder)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
table.Index = len(tables)
|
||
tables = append(tables, table)
|
||
}
|
||
}
|
||
|
||
func parseTable(decoder *xml.Decoder) (Table, error) {
|
||
var table Table
|
||
for {
|
||
token, err := decoder.Token()
|
||
if err != nil {
|
||
return table, err
|
||
}
|
||
switch t := token.(type) {
|
||
case xml.StartElement:
|
||
if t.Name.Local == "tr" {
|
||
row, err := parseRow(decoder, len(table.Rows))
|
||
if err != nil {
|
||
return table, err
|
||
}
|
||
table.Rows = append(table.Rows, row)
|
||
}
|
||
case xml.EndElement:
|
||
if t.Name.Local == "tbl" {
|
||
return table, nil
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
func parseRow(decoder *xml.Decoder, rowIndex int) (Row, error) {
|
||
row := Row{Index: rowIndex}
|
||
logicalCol := 0
|
||
for {
|
||
token, err := decoder.Token()
|
||
if err != nil {
|
||
return row, err
|
||
}
|
||
switch t := token.(type) {
|
||
case xml.StartElement:
|
||
if t.Name.Local == "tc" {
|
||
cell, err := parseCell(decoder)
|
||
if err != nil {
|
||
return row, err
|
||
}
|
||
cell.RowIndex = rowIndex
|
||
cell.CellIndex = len(row.Cells)
|
||
cell.ColIndex = logicalCol
|
||
if cell.ColSpan < 1 {
|
||
cell.ColSpan = 1
|
||
}
|
||
logicalCol += cell.ColSpan
|
||
row.Cells = append(row.Cells, cell)
|
||
}
|
||
case xml.EndElement:
|
||
if t.Name.Local == "tr" {
|
||
return row, nil
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
func parseCell(decoder *xml.Decoder) (Cell, error) {
|
||
cell := Cell{ColSpan: 1}
|
||
var b strings.Builder
|
||
inText := false
|
||
tcDepth := 1
|
||
for {
|
||
token, err := decoder.Token()
|
||
if err != nil {
|
||
return cell, err
|
||
}
|
||
switch t := token.(type) {
|
||
case xml.StartElement:
|
||
switch t.Name.Local {
|
||
case "tc":
|
||
tcDepth++
|
||
case "gridSpan":
|
||
if tcDepth == 1 {
|
||
if span, err := strconv.Atoi(attr(t, "val")); err == nil && span > 0 {
|
||
cell.ColSpan = span
|
||
}
|
||
}
|
||
case "vMerge":
|
||
if tcDepth == 1 {
|
||
cell.VMerge = attr(t, "val")
|
||
if cell.VMerge == "" {
|
||
cell.VMerge = "continue"
|
||
}
|
||
}
|
||
case "t":
|
||
inText = true
|
||
case "tab":
|
||
b.WriteByte('\t')
|
||
case "br", "cr":
|
||
b.WriteByte('\n')
|
||
}
|
||
case xml.CharData:
|
||
if inText {
|
||
b.Write([]byte(t))
|
||
}
|
||
case xml.EndElement:
|
||
switch t.Name.Local {
|
||
case "t":
|
||
inText = false
|
||
case "p":
|
||
if b.Len() > 0 && !strings.HasSuffix(b.String(), "\n") {
|
||
b.WriteByte('\n')
|
||
}
|
||
case "tc":
|
||
tcDepth--
|
||
if tcDepth == 0 {
|
||
cell.Text = normalizeCellText(b.String())
|
||
return cell, nil
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
func attr(start xml.StartElement, local string) string {
|
||
for _, a := range start.Attr {
|
||
if a.Name.Local == local {
|
||
return a.Value
|
||
}
|
||
}
|
||
return ""
|
||
}
|
||
|
||
func normalizeCellText(s string) string {
|
||
s = strings.ReplaceAll(s, "\r\n", "\n")
|
||
s = strings.ReplaceAll(s, "\r", "\n")
|
||
lines := strings.Split(s, "\n")
|
||
out := make([]string, 0, len(lines))
|
||
for _, line := range lines {
|
||
line = strings.TrimSpace(strings.Join(strings.Fields(line), " "))
|
||
if line != "" {
|
||
out = append(out, line)
|
||
}
|
||
}
|
||
return strings.Join(out, "\n")
|
||
}
|
||
|
||
func inferFields(table Table) []Field {
|
||
headerIndex := detectHeaderRow(table)
|
||
if headerIndex >= 0 {
|
||
return inferHeaderFields(table, headerIndex)
|
||
}
|
||
return inferKeyValueFields(table)
|
||
}
|
||
|
||
func detectHeaderRow(table Table) int {
|
||
for i := 0; i < len(table.Rows)-1; i++ {
|
||
row := table.Rows[i]
|
||
if isTitleRow(row) {
|
||
continue
|
||
}
|
||
if nonEmptyCellCount(row) >= 2 && nonEmptyCellCount(table.Rows[i+1]) >= 2 {
|
||
return i
|
||
}
|
||
}
|
||
return -1
|
||
}
|
||
|
||
func isTitleRow(row Row) bool {
|
||
nonEmpty := nonEmptyCellCount(row)
|
||
if nonEmpty != 1 || len(row.Cells) == 0 {
|
||
return false
|
||
}
|
||
return len(row.Cells) == 1 || row.Cells[0].ColSpan > 1
|
||
}
|
||
|
||
func nonEmptyCellCount(row Row) int {
|
||
count := 0
|
||
for _, cell := range row.Cells {
|
||
if strings.TrimSpace(cell.Text) != "" {
|
||
count++
|
||
}
|
||
}
|
||
return count
|
||
}
|
||
|
||
func inferHeaderFields(table Table, headerIndex int) []Field {
|
||
headers := table.Rows[headerIndex].Cells
|
||
var fields []Field
|
||
recordIndex := 0
|
||
for rowIndex := headerIndex + 1; rowIndex < len(table.Rows); rowIndex++ {
|
||
row := table.Rows[rowIndex]
|
||
if nonEmptyCellCount(row) == 0 {
|
||
continue
|
||
}
|
||
for i, header := range headers {
|
||
if i >= len(row.Cells) {
|
||
continue
|
||
}
|
||
key := cleanFieldKey(header.Text)
|
||
if key == "" {
|
||
continue
|
||
}
|
||
value := strings.TrimSpace(row.Cells[i].Text)
|
||
fields = append(fields, Field{
|
||
RecordIndex: recordIndex,
|
||
RowIndex: rowIndex,
|
||
ColIndex: row.Cells[i].ColIndex,
|
||
Key: key,
|
||
Value: value,
|
||
SourceKind: "header_row",
|
||
})
|
||
}
|
||
recordIndex++
|
||
}
|
||
return fields
|
||
}
|
||
|
||
func inferKeyValueFields(table Table) []Field {
|
||
var fields []Field
|
||
recordIndex := 0
|
||
for rowIndex, row := range table.Rows {
|
||
cells := nonEmptyCells(row)
|
||
if len(cells) < 2 || len(cells)%2 != 0 {
|
||
continue
|
||
}
|
||
for i := 0; i < len(cells); i += 2 {
|
||
key := cleanFieldKey(cells[i].Text)
|
||
if key == "" {
|
||
continue
|
||
}
|
||
fields = append(fields, Field{
|
||
RecordIndex: recordIndex,
|
||
RowIndex: rowIndex,
|
||
ColIndex: cells[i+1].ColIndex,
|
||
Key: key,
|
||
Value: cells[i+1].Text,
|
||
SourceKind: "key_value",
|
||
})
|
||
}
|
||
recordIndex++
|
||
}
|
||
return fields
|
||
}
|
||
|
||
func nonEmptyCells(row Row) []Cell {
|
||
out := make([]Cell, 0, len(row.Cells))
|
||
for _, cell := range row.Cells {
|
||
if strings.TrimSpace(cell.Text) != "" {
|
||
out = append(out, cell)
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
func cleanFieldKey(s string) string {
|
||
s = strings.TrimSpace(s)
|
||
s = strings.TrimRight(s, "：: \t\r\n")
|
||
return strings.TrimSpace(s)
|
||
}
|
||
|
||
func fileSHA256(path string) (string, error) {
|
||
f, err := os.Open(path)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
defer f.Close()
|
||
h := sha256.New()
|
||
if _, err := io.Copy(h, f); err != nil {
|
||
return "", err
|
||
}
|
||
return hex.EncodeToString(h.Sum(nil)), nil
|
||
}
|