wordtable2sqlite/main.go

package main

import (
	"archive/zip"
	"bytes"
	"context"
	"crypto/sha256"
	"database/sql"
	"encoding/csv"
	"encoding/hex"
	"encoding/xml"
	"errors"
	"flag"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	_ "modernc.org/sqlite"
)

type Table struct {
	Index int
	Rows  []Row
}

type Row struct {
	Index int
	Cells []Cell
}

type Cell struct {
	RowIndex  int
	CellIndex int
	ColIndex  int
	ColSpan   int
	VMerge    string
	Text      string
}

type Field struct {
	RecordIndex int
	RowIndex    int
	ColIndex    int
	Key         string
	Value       string
	SourceKind  string
}

func main() {
	if err := run(os.Args[1:]); err != nil {
		fmt.Fprintln(os.Stderr, "error:", err)
		os.Exit(1)
	}
}

func run(args []string) error {
	if len(args) > 0 {
		switch args[0] {
		case "export-fields":
			return runExportFields(args[1:])
		case "export-cells":
			return runExportCells(args[1:])
		}
	}

	fs := flag.NewFlagSet("wordtable2sqlite", flag.ContinueOnError)
	dbPath := fs.String("db", "word_tables.sqlite", "SQLite database path")
	reset := fs.Bool("reset", false, "remove the target database before importing")
	quiet := fs.Bool("quiet", false, "suppress per-file summary output")
	if err := fs.Parse(args); err != nil {
		return err
	}
	docxPaths := fs.Args()
	if len(docxPaths) == 0 {
		return errors.New("usage: wordtable2sqlite -db output.sqlite [-reset] file1.docx [file2.docx ...]")
	}

	if *reset {
		if err := os.Remove(*dbPath); err != nil && !errors.Is(err, os.ErrNotExist) {
			return fmt.Errorf("reset database: %w", err)
		}
	}
	if dir := filepath.Dir(*dbPath); dir != "." {
		if err := os.MkdirAll(dir, 0o755); err != nil {
			return fmt.Errorf("create database directory: %w", err)
		}
	}

	db, err := sql.Open("sqlite", *dbPath)
	if err != nil {
		return fmt.Errorf("open sqlite database: %w", err)
	}
	defer db.Close()

	ctx := context.Background()
	for _, docxPath := range docxPaths {
		before, err := dbStats(ctx, db)
		if err != nil {
			return err
		}
		if err := importDocument(ctx, db, docxPath); err != nil {
			return err
		}
		after, err := dbStats(ctx, db)
		if err != nil {
			return err
		}
		if !*quiet {
			fmt.Printf("imported %s: %d tables, %d cells, %d fields\n",
				docxPath,
				after.Tables-before.Tables,
				after.Cells-before.Cells,
				after.Fields-before.Fields,
			)
		}
	}
	if !*quiet {
		fmt.Printf("database: %s\n", *dbPath)
	}
	return nil
}

func runExportFields(args []string) error {
	fs := flag.NewFlagSet("wordtable2sqlite export-fields", flag.ContinueOnError)
	dbPath := fs.String("db", "word_tables.sqlite", "SQLite database path")
	outPath := fs.String("out", "", "CSV output path")
	if err := fs.Parse(args); err != nil {
		return err
	}
	if *outPath == "" {
		return errors.New("usage: wordtable2sqlite export-fields -db report.sqlite -out report-fields.csv")
	}

	db, err := sql.Open("sqlite", *dbPath)
	if err != nil {
		return fmt.Errorf("open sqlite database: %w", err)
	}
	defer db.Close()

	rows, err := db.Query(`SELECT d.filename, f.key, f.value
		FROM fields f
		JOIN documents d ON d.id = f.document_id
		ORDER BY d.id, f.id`)
	if err != nil {
		return fmt.Errorf("query fields: %w", err)
	}
	defer rows.Close()

	return writeCSV(*outPath, []string{"filename", "key", "value"}, rows, 3)
}

func runExportCells(args []string) error {
	fs := flag.NewFlagSet("wordtable2sqlite export-cells", flag.ContinueOnError)
	dbPath := fs.String("db", "word_tables.sqlite", "SQLite database path")
	outPath := fs.String("out", "", "CSV output path")
	if err := fs.Parse(args); err != nil {
		return err
	}
	if *outPath == "" {
		return errors.New("usage: wordtable2sqlite export-cells -db report.sqlite -out report-cells.csv")
	}

	db, err := sql.Open("sqlite", *dbPath)
	if err != nil {
		return fmt.Errorf("open sqlite database: %w", err)
	}
	defer db.Close()

	rows, err := db.Query(`SELECT d.filename, c.table_index, c.row_index, c.cell_index, c.col_index, c.col_span, c.v_merge, c.text
		FROM cells c
		JOIN documents d ON d.id = c.document_id
		ORDER BY d.id, c.table_index, c.row_index, c.cell_index`)
	if err != nil {
		return fmt.Errorf("query cells: %w", err)
	}
	defer rows.Close()

	return writeCSV(*outPath, []string{"filename", "table_index", "row_index", "cell_index", "col_index", "col_span", "v_merge", "text"}, rows, 8)
}

func writeCSV(outPath string, header []string, rows *sql.Rows, columns int) error {
	if dir := filepath.Dir(outPath); dir != "." {
		if err := os.MkdirAll(dir, 0o755); err != nil {
			return fmt.Errorf("create csv directory: %w", err)
		}
	}
	f, err := os.Create(outPath)
	if err != nil {
		return fmt.Errorf("create csv: %w", err)
	}
	defer f.Close()

	w := csv.NewWriter(f)
	if err := w.Write(header); err != nil {
		return err
	}
	for rows.Next() {
		values := make([]string, columns)
		dest := make([]any, columns)
		for i := range values {
			dest[i] = &values[i]
		}
		if err := rows.Scan(dest...); err != nil {
			return fmt.Errorf("scan csv row: %w", err)
		}
		if err := w.Write(values); err != nil {
			return err
		}
	}
	if err := rows.Err(); err != nil {
		return err
	}
	w.Flush()
	return w.Error()
}

type stats struct {
	Tables int
	Cells  int
	Fields int
}

func dbStats(ctx context.Context, db *sql.DB) (stats, error) {
	if err := ensureSchema(ctx, db); err != nil {
		return stats{}, err
	}
	var s stats
	for _, item := range []struct {
		query string
		dst   *int
	}{
		{`SELECT COUNT(*) FROM tables`, &s.Tables},
		{`SELECT COUNT(*) FROM cells`, &s.Cells},
		{`SELECT COUNT(*) FROM fields`, &s.Fields},
	} {
		if err := db.QueryRowContext(ctx, item.query).Scan(item.dst); err != nil {
			return stats{}, err
		}
	}
	return s, nil
}

func importDocument(ctx context.Context, db *sql.DB, docxPath string) error {
	if err := ensureSchema(ctx, db); err != nil {
		return err
	}
	tables, err := readDocxTables(docxPath)
	if err != nil {
		return err
	}
	sum, err := fileSHA256(docxPath)
	if err != nil {
		return err
	}

	tx, err := db.BeginTx(ctx, nil)
	if err != nil {
		return err
	}
	defer tx.Rollback()

	res, err := tx.ExecContext(ctx,
		`INSERT INTO documents(path, filename, sha256, imported_at) VALUES (?, ?, ?, ?)`,
		docxPath,
		filepath.Base(docxPath),
		sum,
		time.Now().Format(time.RFC3339),
	)
	if err != nil {
		return fmt.Errorf("insert document: %w", err)
	}
	documentID, err := res.LastInsertId()
	if err != nil {
		return err
	}

	for tableIndex, table := range tables {
		cellCount := 0
		for _, row := range table.Rows {
			cellCount += len(row.Cells)
		}
		res, err := tx.ExecContext(ctx,
			`INSERT INTO tables(document_id, table_index, row_count, cell_count) VALUES (?, ?, ?, ?)`,
			documentID,
			tableIndex,
			len(table.Rows),
			cellCount,
		)
		if err != nil {
			return fmt.Errorf("insert table %d: %w", tableIndex, err)
		}
		tableID, err := res.LastInsertId()
		if err != nil {
			return err
		}

		for rowIndex, row := range table.Rows {
			for cellIndex, cell := range row.Cells {
				if _, err := tx.ExecContext(ctx,
					`INSERT INTO cells(document_id, table_id, table_index, row_index, cell_index, col_index, col_span, v_merge, text)
					 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
					documentID,
					tableID,
					tableIndex,
					rowIndex,
					cellIndex,
					cell.ColIndex,
					cell.ColSpan,
					cell.VMerge,
					cell.Text,
				); err != nil {
					return fmt.Errorf("insert cell table=%d row=%d cell=%d: %w", tableIndex, rowIndex, cellIndex, err)
				}
			}
		}

		for _, field := range inferFields(table) {
			if _, err := tx.ExecContext(ctx,
				`INSERT INTO fields(document_id, table_id, table_index, record_index, row_index, col_index, key, value, source_kind)
				 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
				documentID,
				tableID,
				tableIndex,
				field.RecordIndex,
				field.RowIndex,
				field.ColIndex,
				field.Key,
				field.Value,
				field.SourceKind,
			); err != nil {
				return fmt.Errorf("insert field %q: %w", field.Key, err)
			}
		}
	}

	return tx.Commit()
}

func ensureSchema(ctx context.Context, db execer) error {
	statements := []string{
		`PRAGMA foreign_keys = ON`,
		`CREATE TABLE IF NOT EXISTS documents (
			id INTEGER PRIMARY KEY AUTOINCREMENT,
			path TEXT NOT NULL,
			filename TEXT NOT NULL,
			sha256 TEXT NOT NULL,
			imported_at TEXT NOT NULL
		)`,
		`CREATE TABLE IF NOT EXISTS tables (
			id INTEGER PRIMARY KEY AUTOINCREMENT,
			document_id INTEGER NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
			table_index INTEGER NOT NULL,
			row_count INTEGER NOT NULL,
			cell_count INTEGER NOT NULL
		)`,
		`CREATE TABLE IF NOT EXISTS cells (
			id INTEGER PRIMARY KEY AUTOINCREMENT,
			document_id INTEGER NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
			table_id INTEGER NOT NULL REFERENCES tables(id) ON DELETE CASCADE,
			table_index INTEGER NOT NULL,
			row_index INTEGER NOT NULL,
			cell_index INTEGER NOT NULL,
			col_index INTEGER NOT NULL,
			col_span INTEGER NOT NULL,
			v_merge TEXT NOT NULL DEFAULT '',
			text TEXT NOT NULL
		)`,
		`CREATE TABLE IF NOT EXISTS fields (
			id INTEGER PRIMARY KEY AUTOINCREMENT,
			document_id INTEGER NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
			table_id INTEGER NOT NULL REFERENCES tables(id) ON DELETE CASCADE,
			table_index INTEGER NOT NULL,
			record_index INTEGER NOT NULL,
			row_index INTEGER NOT NULL,
			col_index INTEGER NOT NULL,
			key TEXT NOT NULL,
			value TEXT NOT NULL,
			source_kind TEXT NOT NULL
		)`,
		`CREATE INDEX IF NOT EXISTS idx_cells_doc_table ON cells(document_id, table_index, row_index, cell_index)`,
		`CREATE INDEX IF NOT EXISTS idx_fields_doc_key ON fields(document_id, key)`,
	}
	for _, statement := range statements {
		if _, err := db.ExecContext(ctx, statement); err != nil {
			return fmt.Errorf("schema statement failed: %w", err)
		}
	}
	return nil
}

type execer interface {
	ExecContext(context.Context, string, ...any) (sql.Result, error)
}

func readDocxTables(path string) ([]Table, error) {
	zr, err := zip.OpenReader(path)
	if err != nil {
		return nil, fmt.Errorf("open docx zip: %w", err)
	}
	defer zr.Close()

	for _, file := range zr.File {
		if file.Name != "word/document.xml" {
			continue
		}
		rc, err := file.Open()
		if err != nil {
			return nil, err
		}
		defer rc.Close()
		data, err := io.ReadAll(rc)
		if err != nil {
			return nil, err
		}
		return parseDocumentXML(data)
	}
	return nil, errors.New("word/document.xml not found in docx")
}

func parseDocumentXML(data []byte) ([]Table, error) {
	decoder := xml.NewDecoder(bytes.NewReader(data))
	var tables []Table
	for {
		token, err := decoder.Token()
		if errors.Is(err, io.EOF) {
			return tables, nil
		}
		if err != nil {
			return nil, err
		}
		start, ok := token.(xml.StartElement)
		if !ok || start.Name.Local != "tbl" {
			continue
		}
		table, err := parseTable(decoder)
		if err != nil {
			return nil, err
		}
		table.Index = len(tables)
		tables = append(tables, table)
	}
}

func parseTable(decoder *xml.Decoder) (Table, error) {
	var table Table
	for {
		token, err := decoder.Token()
		if err != nil {
			return table, err
		}
		switch t := token.(type) {
		case xml.StartElement:
			if t.Name.Local == "tr" {
				row, err := parseRow(decoder, len(table.Rows))
				if err != nil {
					return table, err
				}
				table.Rows = append(table.Rows, row)
			}
		case xml.EndElement:
			if t.Name.Local == "tbl" {
				return table, nil
			}
		}
	}
}

func parseRow(decoder *xml.Decoder, rowIndex int) (Row, error) {
	row := Row{Index: rowIndex}
	logicalCol := 0
	for {
		token, err := decoder.Token()
		if err != nil {
			return row, err
		}
		switch t := token.(type) {
		case xml.StartElement:
			if t.Name.Local == "tc" {
				cell, err := parseCell(decoder)
				if err != nil {
					return row, err
				}
				cell.RowIndex = rowIndex
				cell.CellIndex = len(row.Cells)
				cell.ColIndex = logicalCol
				if cell.ColSpan < 1 {
					cell.ColSpan = 1
				}
				logicalCol += cell.ColSpan
				row.Cells = append(row.Cells, cell)
			}
		case xml.EndElement:
			if t.Name.Local == "tr" {
				return row, nil
			}
		}
	}
}

func parseCell(decoder *xml.Decoder) (Cell, error) {
	cell := Cell{ColSpan: 1}
	var b strings.Builder
	inText := false
	tcDepth := 1
	for {
		token, err := decoder.Token()
		if err != nil {
			return cell, err
		}
		switch t := token.(type) {
		case xml.StartElement:
			switch t.Name.Local {
			case "tc":
				tcDepth++
			case "gridSpan":
				if tcDepth == 1 {
					if span, err := strconv.Atoi(attr(t, "val")); err == nil && span > 0 {
						cell.ColSpan = span
					}
				}
			case "vMerge":
				if tcDepth == 1 {
					cell.VMerge = attr(t, "val")
					if cell.VMerge == "" {
						cell.VMerge = "continue"
					}
				}
			case "t":
				inText = true
			case "tab":
				b.WriteByte('\t')
			case "br", "cr":
				b.WriteByte('\n')
			}
		case xml.CharData:
			if inText {
				b.Write([]byte(t))
			}
		case xml.EndElement:
			switch t.Name.Local {
			case "t":
				inText = false
			case "p":
				if b.Len() > 0 && !strings.HasSuffix(b.String(), "\n") {
					b.WriteByte('\n')
				}
			case "tc":
				tcDepth--
				if tcDepth == 0 {
					cell.Text = normalizeCellText(b.String())
					return cell, nil
				}
			}
		}
	}
}

func attr(start xml.StartElement, local string) string {
	for _, a := range start.Attr {
		if a.Name.Local == local {
			return a.Value
		}
	}
	return ""
}

func normalizeCellText(s string) string {
	s = strings.ReplaceAll(s, "\r\n", "\n")
	s = strings.ReplaceAll(s, "\r", "\n")
	lines := strings.Split(s, "\n")
	out := make([]string, 0, len(lines))
	for _, line := range lines {
		line = strings.TrimSpace(strings.Join(strings.Fields(line), " "))
		if line != "" {
			out = append(out, line)
		}
	}
	return strings.Join(out, "\n")
}

func inferFields(table Table) []Field {
	headerIndex := detectHeaderRow(table)
	if headerIndex >= 0 {
		return inferHeaderFields(table, headerIndex)
	}
	return inferKeyValueFields(table)
}

func detectHeaderRow(table Table) int {
	for i := 0; i < len(table.Rows)-1; i++ {
		row := table.Rows[i]
		if isTitleRow(row) {
			continue
		}
		if nonEmptyCellCount(row) >= 2 && nonEmptyCellCount(table.Rows[i+1]) >= 2 {
			return i
		}
	}
	return -1
}

func isTitleRow(row Row) bool {
	nonEmpty := nonEmptyCellCount(row)
	if nonEmpty != 1 || len(row.Cells) == 0 {
		return false
	}
	return len(row.Cells) == 1 || row.Cells[0].ColSpan > 1
}

func nonEmptyCellCount(row Row) int {
	count := 0
	for _, cell := range row.Cells {
		if strings.TrimSpace(cell.Text) != "" {
			count++
		}
	}
	return count
}

func inferHeaderFields(table Table, headerIndex int) []Field {
	headers := cellsWithLogicalColumns(table.Rows[headerIndex])
	var fields []Field
	recordIndex := 0
	for rowIndex := headerIndex + 1; rowIndex < len(table.Rows); rowIndex++ {
		row := table.Rows[rowIndex]
		if nonEmptyCellCount(row) == 0 {
			continue
		}
		rowCells := cellsWithLogicalColumns(row)
		for _, header := range headers {
			key := cleanFieldKey(header.Text)
			if key == "" {
				continue
			}
			value := cellsTextForRange(rowCells, header.ColIndex, header.ColIndex+span(header))
			fields = append(fields, Field{
				RecordIndex: recordIndex,
				RowIndex:    rowIndex,
				ColIndex:    header.ColIndex,
				Key:         key,
				Value:       value,
				SourceKind:  "header_row",
			})
		}
		recordIndex++
	}
	return fields
}

func cellsWithLogicalColumns(row Row) []Cell {
	out := make([]Cell, len(row.Cells))
	copy(out, row.Cells)

	needsReindex := false
	nextCol := 0
	for i := range out {
		if out[i].ColSpan < 1 {
			out[i].ColSpan = 1
		}
		if i == 0 {
			needsReindex = out[i].ColIndex != 0
		} else if out[i].ColIndex < nextCol {
			needsReindex = true
		}
		nextCol = out[i].ColIndex + out[i].ColSpan
	}
	if !needsReindex {
		return out
	}

	nextCol = 0
	for i := range out {
		out[i].ColIndex = nextCol
		nextCol += out[i].ColSpan
	}
	return out
}

func cellsTextForRange(cells []Cell, start, end int) string {
	var values []string
	for _, cell := range cells {
		cellStart := cell.ColIndex
		cellEnd := cell.ColIndex + span(cell)
		if cellStart >= end || cellEnd <= start {
			continue
		}
		text := strings.TrimSpace(cell.Text)
		if text == "" {
			continue
		}
		if len(values) == 0 || values[len(values)-1] != text {
			values = append(values, text)
		}
	}
	return strings.Join(values, "\n")
}

func span(cell Cell) int {
	if cell.ColSpan < 1 {
		return 1
	}
	return cell.ColSpan
}

func inferKeyValueFields(table Table) []Field {
	var fields []Field
	recordIndex := 0
	for rowIndex, row := range table.Rows {
		cells := nonEmptyCells(row)
		if len(cells) < 2 || len(cells)%2 != 0 {
			continue
		}
		for i := 0; i < len(cells); i += 2 {
			key := cleanFieldKey(cells[i].Text)
			if key == "" {
				continue
			}
			fields = append(fields, Field{
				RecordIndex: recordIndex,
				RowIndex:    rowIndex,
				ColIndex:    cells[i+1].ColIndex,
				Key:         key,
				Value:       cells[i+1].Text,
				SourceKind:  "key_value",
			})
		}
		recordIndex++
	}
	return fields
}

func nonEmptyCells(row Row) []Cell {
	out := make([]Cell, 0, len(row.Cells))
	for _, cell := range row.Cells {
		if strings.TrimSpace(cell.Text) != "" {
			out = append(out, cell)
		}
	}
	return out
}

func cleanFieldKey(s string) string {
	s = strings.TrimSpace(s)
	s = strings.Trim(s, "：: \t\r\n")
	return strings.TrimSpace(s)
}

func fileSHA256(path string) (string, error) {
	f, err := os.Open(path)
	if err != nil {
		return "", err
	}
	defer f.Close()
	h := sha256.New()
	if _, err := io.Copy(h, f); err != nil {
		return "", err
	}
	return hex.EncodeToString(h.Sum(nil)), nil
}