Implement internal mail skill MVP

This commit is contained in:
赵义仑
2026-06-06 22:28:36 +08:00
commit 097df73495
22 changed files with 4059 additions and 0 deletions

331
internal/api/api.go Normal file
View File

@@ -0,0 +1,331 @@
package api
import (
"encoding/json"
"errors"
"fmt"
"net/http"
"strconv"
"time"
"internal-mail-skill/internal/config"
"internal-mail-skill/internal/db"
"internal-mail-skill/internal/service"
)
const Version = "0.1.0"
type Server struct {
cfg *config.Config
store *db.Store
svc *service.Service
}
func New(cfg *config.Config, store *db.Store, svc *service.Service) *Server {
return &Server{cfg: cfg, store: store, svc: svc}
}
func (s *Server) Handler() http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("GET /health", s.health)
mux.HandleFunc("POST /sync", s.sync)
mux.HandleFunc("GET /emails/search", s.searchEmails)
mux.HandleFunc("GET /emails/{id}", s.getEmail)
mux.HandleFunc("GET /emails/{id}/thread", s.getThread)
mux.HandleFunc("POST /emails/{id}/mark", s.markEmail)
mux.HandleFunc("POST /labels", s.createLabel)
mux.HandleFunc("POST /emails/labels/apply", s.applyLabels)
mux.HandleFunc("POST /drafts", s.createDraft)
mux.HandleFunc("GET /drafts", s.listDrafts)
mux.HandleFunc("PATCH /drafts/{id}", s.updateDraft)
mux.HandleFunc("POST /drafts/{id}/send", s.sendDraft)
mux.HandleFunc("POST /send", s.sendDirect)
mux.HandleFunc("POST /emails/{id}/forward", s.forwardEmail)
return withJSON(mux)
}
func (s *Server) health(w http.ResponseWriter, r *http.Request) {
dbStatus := "ok"
if err := s.store.Ping(r.Context()); err != nil {
dbStatus = "down"
}
writeJSON(w, http.StatusOK, map[string]any{
"status": dbStatus,
"version": Version,
"database": dbStatus,
"account_id": s.cfg.AccountID,
"time": time.Now().UTC().Format(time.RFC3339),
})
}
func (s *Server) sync(w http.ResponseWriter, r *http.Request) {
result, err := s.svc.Sync(r.Context())
if err != nil {
writeError(w, http.StatusBadGateway, "sync_failed", err.Error(), nil)
return
}
writeJSON(w, http.StatusOK, result)
}
func (s *Server) searchEmails(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query()
filters := service.SearchFilters{
Q: q.Get("q"),
From: q.Get("from"),
To: q.Get("to"),
After: q.Get("after"),
Before: q.Get("before"),
Limit: intQuery(q.Get("limit"), 50),
Offset: intQuery(q.Get("offset"), 0),
}
filters.HasAttachment = boolQuery(q.Get("has_attachment"))
filters.Read = boolQuery(q.Get("read"))
filters.Processed = boolQuery(q.Get("processed"))
filters.Archived = boolQuery(q.Get("archived"))
filters.Replied = boolQuery(q.Get("replied"))
results, err := s.svc.SearchEmails(r.Context(), filters)
if err != nil {
writeError(w, http.StatusInternalServerError, "search_failed", err.Error(), nil)
return
}
writeJSON(w, http.StatusOK, map[string]any{"emails": results, "limit": filters.Limit, "offset": filters.Offset})
}
func (s *Server) getEmail(w http.ResponseWriter, r *http.Request) {
id, ok := pathID(w, r)
if !ok {
return
}
email, err := s.svc.GetEmail(r.Context(), id)
if err != nil {
handleServiceError(w, err)
return
}
writeJSON(w, http.StatusOK, email)
}
func (s *Server) getThread(w http.ResponseWriter, r *http.Request) {
id, ok := pathID(w, r)
if !ok {
return
}
thread, err := s.svc.GetThread(r.Context(), id)
if err != nil {
handleServiceError(w, err)
return
}
writeJSON(w, http.StatusOK, map[string]any{"thread": thread})
}
func (s *Server) markEmail(w http.ResponseWriter, r *http.Request) {
id, ok := pathID(w, r)
if !ok {
return
}
var input service.MarkInput
if !decodeBody(w, r, &input) {
return
}
if err := s.svc.MarkEmail(r.Context(), id, input); err != nil {
handleServiceError(w, err)
return
}
writeJSON(w, http.StatusOK, map[string]any{"ok": true})
}
func (s *Server) createLabel(w http.ResponseWriter, r *http.Request) {
var input struct {
Name string `json:"name"`
Color string `json:"color"`
}
if !decodeBody(w, r, &input) {
return
}
label, err := s.svc.CreateLabel(r.Context(), input.Name, input.Color)
if err != nil {
writeError(w, http.StatusBadRequest, "label_failed", err.Error(), nil)
return
}
writeJSON(w, http.StatusCreated, label)
}
func (s *Server) applyLabels(w http.ResponseWriter, r *http.Request) {
var input struct {
EmailIDs []int64 `json:"email_ids"`
LabelIDs []int64 `json:"label_ids"`
Op string `json:"op"`
}
if !decodeBody(w, r, &input) {
return
}
if err := s.svc.ApplyLabels(r.Context(), input.EmailIDs, input.LabelIDs, input.Op); err != nil {
writeError(w, http.StatusBadRequest, "label_apply_failed", err.Error(), nil)
return
}
writeJSON(w, http.StatusOK, map[string]any{"ok": true})
}
func (s *Server) createDraft(w http.ResponseWriter, r *http.Request) {
var input service.DraftInput
if !decodeBody(w, r, &input) {
return
}
draft, err := s.svc.CreateDraft(r.Context(), input)
if err != nil {
writeError(w, http.StatusBadRequest, "draft_create_failed", err.Error(), nil)
return
}
writeJSON(w, http.StatusCreated, draft)
}
func (s *Server) listDrafts(w http.ResponseWriter, r *http.Request) {
drafts, err := s.svc.ListDrafts(r.Context())
if err != nil {
writeError(w, http.StatusInternalServerError, "draft_list_failed", err.Error(), nil)
return
}
writeJSON(w, http.StatusOK, map[string]any{"drafts": drafts})
}
func (s *Server) updateDraft(w http.ResponseWriter, r *http.Request) {
id, ok := pathID(w, r)
if !ok {
return
}
var input service.DraftInput
if !decodeBody(w, r, &input) {
return
}
draft, err := s.svc.UpdateDraft(r.Context(), id, input)
if err != nil {
handleServiceError(w, err)
return
}
writeJSON(w, http.StatusOK, draft)
}
func (s *Server) sendDraft(w http.ResponseWriter, r *http.Request) {
id, ok := pathID(w, r)
if !ok {
return
}
outcome, err := s.svc.SendDraft(r.Context(), id)
if err != nil {
writeError(w, http.StatusBadGateway, "send_failed", err.Error(), outcome)
return
}
writeJSON(w, http.StatusOK, outcome)
}
func (s *Server) sendDirect(w http.ResponseWriter, r *http.Request) {
var input service.DraftInput
if !decodeBody(w, r, &input) {
return
}
outcome, err := s.svc.SendDirect(r.Context(), input)
if err != nil {
writeError(w, http.StatusBadGateway, "send_failed", err.Error(), outcome)
return
}
writeJSON(w, http.StatusOK, outcome)
}
func (s *Server) forwardEmail(w http.ResponseWriter, r *http.Request) {
id, ok := pathID(w, r)
if !ok {
return
}
var input service.ForwardInput
if !decodeBody(w, r, &input) {
return
}
outcome, err := s.svc.ForwardEmail(r.Context(), id, input)
if err != nil {
handleServiceError(w, err)
return
}
writeJSON(w, http.StatusOK, outcome)
}
func decodeBody(w http.ResponseWriter, r *http.Request, dst any) bool {
defer r.Body.Close()
if err := json.NewDecoder(r.Body).Decode(dst); err != nil {
writeError(w, http.StatusBadRequest, "invalid_json", err.Error(), nil)
return false
}
return true
}
func pathID(w http.ResponseWriter, r *http.Request) (int64, bool) {
id, err := strconv.ParseInt(r.PathValue("id"), 10, 64)
if err != nil || id <= 0 {
writeError(w, http.StatusBadRequest, "invalid_id", "id must be a positive integer", nil)
return 0, false
}
return id, true
}
func intQuery(value string, def int) int {
if value == "" {
return def
}
n, err := strconv.Atoi(value)
if err != nil {
return def
}
return n
}
func boolQuery(value string) *bool {
if value == "" {
return nil
}
switch value {
case "1", "true", "TRUE", "yes", "on":
v := true
return &v
case "0", "false", "FALSE", "no", "off":
v := false
return &v
default:
return nil
}
}
func handleServiceError(w http.ResponseWriter, err error) {
switch {
case errors.Is(err, service.ErrNotFound):
writeError(w, http.StatusNotFound, "not_found", err.Error(), nil)
case errors.Is(err, service.ErrAttachmentsForwardUnsupported):
writeError(w, http.StatusBadRequest, "attachments_forward_unsupported", err.Error(), nil)
default:
writeError(w, http.StatusInternalServerError, "internal_error", err.Error(), nil)
}
}
func writeJSON(w http.ResponseWriter, status int, payload any) {
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(payload)
}
func writeError(w http.ResponseWriter, status int, code, message string, details any) {
writeJSON(w, status, map[string]any{
"error": map[string]any{
"code": code,
"message": message,
"details": details,
},
})
}
func withJSON(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("X-Content-Type-Options", "nosniff")
next.ServeHTTP(w, r)
})
}
func ListenAddr(cfg *config.Config) string {
return fmt.Sprintf("%s:%d", cfg.HTTP.Host, cfg.HTTP.Port)
}

177
internal/config/config.go Normal file
View File

@@ -0,0 +1,177 @@
package config
import (
"bufio"
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"time"
)
type Config struct {
AccountID string
POP3 POP3Config
SMTP SMTPConfig
Data DataConfig
HTTP HTTPConfig
}
type POP3Config struct {
Host string
Port int
Username string
Password string
UseTLS bool
TimeoutSeconds int
}
type SMTPConfig struct {
Host string
Port int
Username string
Password string
UseTLS bool
UseSTARTTLS bool
AuthRequired bool
AllowInsecureAuth bool
FromAddress string
FromName string
TimeoutSeconds int
}
type DataConfig struct {
DataDir string
DatabasePath string
RawMailDir string
AttachmentDir string
}
type HTTPConfig struct {
Host string
Port int
}
func (c POP3Config) Timeout() time.Duration {
return time.Duration(c.TimeoutSeconds) * time.Second
}
func (c SMTPConfig) Timeout() time.Duration {
return time.Duration(c.TimeoutSeconds) * time.Second
}
func Load(envPath string) (*Config, error) {
values := map[string]string{}
if envPath == "" {
envPath = ".env"
}
if err := readEnvFile(envPath, values); err != nil {
return nil, err
}
for _, item := range os.Environ() {
key, val, ok := strings.Cut(item, "=")
if ok {
values[key] = val
}
}
dataDir := getString(values, "DATA_DIR", "./data")
cfg := &Config{
AccountID: getString(values, "MAIL_ACCOUNT_ID", "default"),
POP3: POP3Config{
Host: getString(values, "POP3_HOST", ""),
Port: getInt(values, "POP3_PORT", 110),
Username: getString(values, "POP3_USERNAME", ""),
Password: getString(values, "POP3_PASSWORD", ""),
UseTLS: getBool(values, "POP3_USE_TLS", false),
TimeoutSeconds: getInt(values, "POP3_TIMEOUT_SECONDS", 30),
},
SMTP: SMTPConfig{
Host: getString(values, "SMTP_HOST", ""),
Port: getInt(values, "SMTP_PORT", 25),
Username: getString(values, "SMTP_USERNAME", ""),
Password: getString(values, "SMTP_PASSWORD", ""),
UseTLS: getBool(values, "SMTP_USE_TLS", false),
UseSTARTTLS: getBool(values, "SMTP_USE_STARTTLS", false),
AuthRequired: getBool(values, "SMTP_AUTH_REQUIRED", false),
AllowInsecureAuth: getBool(values, "SMTP_ALLOW_INSECURE_AUTH", false),
FromAddress: getString(values, "SMTP_FROM_ADDRESS", ""),
FromName: getString(values, "SMTP_FROM_NAME", ""),
TimeoutSeconds: getInt(values, "SMTP_TIMEOUT_SECONDS", 30),
},
Data: DataConfig{
DataDir: dataDir,
DatabasePath: getString(values, "DATABASE_PATH", filepath.Join(dataDir, "internal_mail.db")),
RawMailDir: getString(values, "RAW_MAIL_DIR", filepath.Join(dataDir, "raw")),
AttachmentDir: getString(values, "ATTACHMENT_DIR", filepath.Join(dataDir, "attachments")),
},
HTTP: HTTPConfig{
Host: getString(values, "HTTP_HOST", "127.0.0.1"),
Port: getInt(values, "HTTP_PORT", 8765),
},
}
return cfg, nil
}
func readEnvFile(path string, values map[string]string) error {
file, err := os.Open(path)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return fmt.Errorf("read env file: %w", err)
}
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" || strings.HasPrefix(line, "#") {
continue
}
key, value, ok := strings.Cut(line, "=")
if !ok {
continue
}
key = strings.TrimSpace(key)
value = strings.TrimSpace(value)
value = strings.Trim(value, `"'`)
values[key] = value
}
return scanner.Err()
}
func getString(values map[string]string, key, def string) string {
if v, ok := values[key]; ok {
return v
}
return def
}
func getInt(values map[string]string, key string, def int) int {
v, ok := values[key]
if !ok || strings.TrimSpace(v) == "" {
return def
}
n, err := strconv.Atoi(v)
if err != nil {
return def
}
return n
}
func getBool(values map[string]string, key string, def bool) bool {
v, ok := values[key]
if !ok || strings.TrimSpace(v) == "" {
return def
}
switch strings.ToLower(strings.TrimSpace(v)) {
case "1", "true", "yes", "y", "on":
return true
case "0", "false", "no", "n", "off":
return false
default:
return def
}
}

View File

@@ -0,0 +1,42 @@
package config
import (
"os"
"path/filepath"
"testing"
)
func TestLoadDefaultsAndEnvFile(t *testing.T) {
dir := t.TempDir()
envPath := filepath.Join(dir, ".env")
if err := os.WriteFile(envPath, []byte("POP3_HOST=mail.local\nSMTP_PORT=2525\nSMTP_AUTH_REQUIRED=true\nSMTP_ALLOW_INSECURE_AUTH=true\nDATA_DIR="+dir+"\n"), 0600); err != nil {
t.Fatal(err)
}
cfg, err := Load(envPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if cfg.AccountID != "default" {
t.Fatalf("AccountID = %q, want default", cfg.AccountID)
}
if cfg.POP3.Host != "mail.local" {
t.Fatalf("POP3.Host = %q", cfg.POP3.Host)
}
if cfg.POP3.Port != 110 {
t.Fatalf("POP3.Port = %d, want 110", cfg.POP3.Port)
}
if cfg.SMTP.Port != 2525 {
t.Fatalf("SMTP.Port = %d, want 2525", cfg.SMTP.Port)
}
if !cfg.SMTP.AuthRequired {
t.Fatal("SMTP.AuthRequired = false, want true")
}
if !cfg.SMTP.AllowInsecureAuth {
t.Fatal("SMTP.AllowInsecureAuth = false, want true")
}
if cfg.HTTP.Host != "127.0.0.1" || cfg.HTTP.Port != 8765 {
t.Fatalf("HTTP default = %s:%d", cfg.HTTP.Host, cfg.HTTP.Port)
}
}

164
internal/db/db.go Normal file
View File

@@ -0,0 +1,164 @@
package db
import (
"context"
"database/sql"
"fmt"
"os"
"path/filepath"
_ "modernc.org/sqlite"
)
type Store struct {
DB *sql.DB
}
func Open(path string) (*Store, error) {
if path != ":memory:" {
if err := os.MkdirAll(filepath.Dir(path), 0750); err != nil {
return nil, fmt.Errorf("create database directory: %w", err)
}
}
db, err := sql.Open("sqlite", path)
if err != nil {
return nil, fmt.Errorf("open sqlite: %w", err)
}
if _, err := db.Exec("PRAGMA foreign_keys = ON"); err != nil {
_ = db.Close()
return nil, fmt.Errorf("enable foreign keys: %w", err)
}
if path == ":memory:" {
db.SetMaxOpenConns(1)
}
return &Store{DB: db}, nil
}
func (s *Store) Close() error {
if s == nil || s.DB == nil {
return nil
}
return s.DB.Close()
}
func (s *Store) Ping(ctx context.Context) error {
return s.DB.PingContext(ctx)
}
func (s *Store) Migrate(ctx context.Context) error {
_, err := s.DB.ExecContext(ctx, schemaSQL)
if err != nil {
return fmt.Errorf("migrate sqlite: %w", err)
}
return nil
}
const schemaSQL = `
CREATE TABLE IF NOT EXISTS mail_messages (
id INTEGER PRIMARY KEY AUTOINCREMENT,
account_id TEXT NOT NULL,
pop3_uid TEXT NOT NULL,
message_id_header TEXT,
in_reply_to TEXT,
references_header TEXT,
subject TEXT,
sender_name TEXT,
sender_email TEXT,
to_json TEXT NOT NULL DEFAULT '[]',
cc_json TEXT NOT NULL DEFAULT '[]',
date_header TEXT,
sent_at TEXT,
received_at TEXT NOT NULL,
body_text TEXT,
body_html TEXT,
raw_eml_path TEXT,
has_attachment INTEGER NOT NULL DEFAULT 0,
local_read INTEGER NOT NULL DEFAULT 0,
local_processed INTEGER NOT NULL DEFAULT 0,
local_archived INTEGER NOT NULL DEFAULT 0,
local_deleted INTEGER NOT NULL DEFAULT 0,
local_replied INTEGER NOT NULL DEFAULT 0,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
UNIQUE(account_id, pop3_uid)
);
CREATE INDEX IF NOT EXISTS idx_mail_messages_account_uid ON mail_messages(account_id, pop3_uid);
CREATE INDEX IF NOT EXISTS idx_mail_messages_message_id ON mail_messages(message_id_header);
CREATE INDEX IF NOT EXISTS idx_mail_messages_sender ON mail_messages(sender_email);
CREATE INDEX IF NOT EXISTS idx_mail_messages_sent_at ON mail_messages(sent_at);
CREATE INDEX IF NOT EXISTS idx_mail_messages_status ON mail_messages(local_read, local_processed, local_archived, local_deleted, local_replied);
CREATE TABLE IF NOT EXISTS attachments (
id INTEGER PRIMARY KEY AUTOINCREMENT,
message_id INTEGER NOT NULL,
file_name TEXT NOT NULL,
content_type TEXT,
content_id TEXT,
size_bytes INTEGER NOT NULL DEFAULT 0,
storage_path TEXT NOT NULL,
created_at TEXT NOT NULL,
FOREIGN KEY(message_id) REFERENCES mail_messages(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_attachments_message ON attachments(message_id);
CREATE TABLE IF NOT EXISTS labels (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL UNIQUE,
color TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS message_labels (
message_id INTEGER NOT NULL,
label_id INTEGER NOT NULL,
created_at TEXT NOT NULL,
PRIMARY KEY(message_id, label_id),
FOREIGN KEY(message_id) REFERENCES mail_messages(id) ON DELETE CASCADE,
FOREIGN KEY(label_id) REFERENCES labels(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_message_labels_label ON message_labels(label_id);
CREATE TABLE IF NOT EXISTS drafts (
id INTEGER PRIMARY KEY AUTOINCREMENT,
to_json TEXT NOT NULL DEFAULT '[]',
cc_json TEXT NOT NULL DEFAULT '[]',
bcc_json TEXT NOT NULL DEFAULT '[]',
subject TEXT,
body TEXT,
reply_to_email_id INTEGER,
status TEXT NOT NULL DEFAULT 'draft',
last_error TEXT,
sent_message_id INTEGER,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
FOREIGN KEY(reply_to_email_id) REFERENCES mail_messages(id) ON DELETE SET NULL
);
CREATE INDEX IF NOT EXISTS idx_drafts_status ON drafts(status);
CREATE TABLE IF NOT EXISTS sent_messages (
id INTEGER PRIMARY KEY AUTOINCREMENT,
draft_id INTEGER,
to_json TEXT NOT NULL DEFAULT '[]',
cc_json TEXT NOT NULL DEFAULT '[]',
bcc_json TEXT NOT NULL DEFAULT '[]',
subject TEXT,
body TEXT,
message_id_header TEXT,
in_reply_to TEXT,
references_header TEXT,
smtp_from TEXT,
status TEXT NOT NULL,
error_message TEXT,
sent_at TEXT,
created_at TEXT NOT NULL,
FOREIGN KEY(draft_id) REFERENCES drafts(id) ON DELETE SET NULL
);
CREATE INDEX IF NOT EXISTS idx_sent_messages_draft ON sent_messages(draft_id);
CREATE INDEX IF NOT EXISTS idx_sent_messages_status ON sent_messages(status);
`

33
internal/db/db_test.go Normal file
View File

@@ -0,0 +1,33 @@
package db
import (
"context"
"database/sql"
"testing"
)
func TestOpenAndMigrateCreatesCoreTables(t *testing.T) {
store, err := Open(":memory:")
if err != nil {
t.Fatalf("Open() error = %v", err)
}
defer store.Close()
if err := store.Migrate(context.Background()); err != nil {
t.Fatalf("Migrate() error = %v", err)
}
var name string
err = store.DB.QueryRowContext(context.Background(), "SELECT name FROM sqlite_master WHERE type='table' AND name='mail_messages'").Scan(&name)
if err != nil {
t.Fatalf("mail_messages table missing: %v", err)
}
err = store.DB.QueryRowContext(context.Background(), "SELECT name FROM sqlite_master WHERE type='table' AND name='sent_messages'").Scan(&name)
if err != nil && err != sql.ErrNoRows {
t.Fatalf("sent_messages lookup failed: %v", err)
}
if name != "sent_messages" {
t.Fatalf("sent_messages table missing, got %q", name)
}
}

View File

@@ -0,0 +1,571 @@
package e2e
import (
"bufio"
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"mime/multipart"
"net"
"net/http"
"net/http/httptest"
"net/textproto"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"testing"
"time"
"internal-mail-skill/internal/api"
"internal-mail-skill/internal/config"
"internal-mail-skill/internal/db"
"internal-mail-skill/internal/pop3"
"internal-mail-skill/internal/service"
mailsmtp "internal-mail-skill/internal/smtp"
)
func TestLocalPOP3SMTPHTTPFlow(t *testing.T) {
ctx := context.Background()
now := time.Date(2026, 6, 6, 9, 30, 0, 0, time.UTC)
raw := buildTestMessage(t, now)
popServer := newFakePOP3Server(t, []popMessage{{uid: "uid-local-1", raw: raw}})
smtpServer := newFakeSMTPServer(t)
dataDir := t.TempDir()
store, err := db.Open(filepath.Join(dataDir, "mail.db"))
if err != nil {
t.Fatalf("open db: %v", err)
}
t.Cleanup(func() { _ = store.Close() })
if err := store.Migrate(ctx); err != nil {
t.Fatalf("migrate db: %v", err)
}
cfg := &config.Config{
AccountID: "local-e2e",
POP3: config.POP3Config{
Host: popServer.host,
Port: popServer.port,
Username: "local-user",
Password: "local-pass",
TimeoutSeconds: 5,
},
SMTP: config.SMTPConfig{
Host: smtpServer.host,
Port: smtpServer.port,
FromAddress: "agent@example.local",
FromName: "Local Agent",
TimeoutSeconds: 5,
AuthRequired: true,
Username: "smtp-user",
Password: "smtp-pass",
AllowInsecureAuth: true,
},
Data: config.DataConfig{
DataDir: dataDir,
DatabasePath: filepath.Join(dataDir, "mail.db"),
RawMailDir: filepath.Join(dataDir, "raw"),
AttachmentDir: filepath.Join(dataDir, "attachments"),
},
}
svc := service.NewWithOptions(
store,
pop3.NewFetcher(cfg.POP3),
mailsmtp.NewSender(cfg.SMTP),
cfg.AccountID,
service.Options{RawDir: cfg.Data.RawMailDir, AttachmentDir: cfg.Data.AttachmentDir, FromAddress: cfg.SMTP.FromAddress, FromName: cfg.SMTP.FromName},
)
server := httptest.NewServer(api.New(cfg, store, svc).Handler())
t.Cleanup(server.Close)
health := getJSON[map[string]any](t, server.URL+"/health")
if health["status"] != "ok" || health["database"] != "ok" {
t.Fatalf("unexpected health: %#v", health)
}
syncResult := postJSON[map[string]any](t, server.URL+"/sync", nil, http.StatusOK)
if got := int(syncResult["stored"].(float64)); got != 1 {
t.Fatalf("stored = %d, want 1; result=%#v", got, syncResult)
}
syncAgain := postJSON[map[string]any](t, server.URL+"/sync", nil, http.StatusOK)
if got := int(syncAgain["skipped"].(float64)); got != 1 {
t.Fatalf("skipped after second sync = %d, want 1; result=%#v", got, syncAgain)
}
search := getJSON[map[string]any](t, server.URL+"/emails/search?q=Quarterly+Report&has_attachment=true")
emails := search["emails"].([]any)
if len(emails) != 1 {
t.Fatalf("search returned %d emails, want 1: %#v", len(emails), search)
}
emailID := int64(emails[0].(map[string]any)["id"].(float64))
detail := getJSON[map[string]any](t, fmt.Sprintf("%s/emails/%d", server.URL, emailID))
if detail["subject"] != "Quarterly Report" {
t.Fatalf("subject = %q", detail["subject"])
}
if !strings.Contains(detail["body_text"].(string), "The report is attached.") {
t.Fatalf("body_text missing expected content: %q", detail["body_text"])
}
attachments := detail["attachments"].([]any)
if len(attachments) != 1 {
t.Fatalf("attachments = %d, want 1", len(attachments))
}
if _, ok := attachments[0].(map[string]any)["attachment_id"]; !ok {
t.Fatalf("attachment id not exposed in detail: %#v", attachments[0])
}
if _, ok := attachments[0].(map[string]any)["storage_path"]; ok {
t.Fatalf("attachment storage path leaked: %#v", attachments[0])
}
label := postJSON[map[string]any](t, server.URL+"/labels", map[string]any{"name": "Local E2E", "color": "#336699"}, http.StatusCreated)
postJSON[map[string]any](t, server.URL+"/emails/labels/apply", map[string]any{
"email_ids": []int64{emailID},
"label_ids": []int64{int64(label["id"].(float64))},
"op": "add",
}, http.StatusOK)
postJSON[map[string]any](t, fmt.Sprintf("%s/emails/%d/mark", server.URL, emailID), map[string]any{"read": true, "processed": true}, http.StatusOK)
marked := getJSON[map[string]any](t, fmt.Sprintf("%s/emails/%d", server.URL, emailID))
status := marked["local_status"].(map[string]any)
if status["read"] != true || status["processed"] != true {
t.Fatalf("local status not updated: %#v", status)
}
if got := len(marked["labels"].([]any)); got != 1 {
t.Fatalf("labels after apply = %d, want 1", got)
}
draft := postJSON[map[string]any](t, server.URL+"/drafts", map[string]any{
"to": []string{"recipient@example.local"},
"cc": []string{"copy@example.local"},
"bcc": []string{"audit@example.local"},
"subject": "Re: Quarterly Report",
"body": "Thanks, received.",
"reply_to_email_id": emailID,
}, http.StatusCreated)
outcome := postJSON[map[string]any](t, fmt.Sprintf("%s/drafts/%d/send", server.URL, int64(draft["id"].(float64))), nil, http.StatusOK)
if sent := outcome["sent_message"].(map[string]any); sent["status"] != "sent" {
t.Fatalf("draft send status = %#v", sent)
}
sentMessages := smtpServer.messages()
if len(sentMessages) != 1 {
t.Fatalf("smtp captured %d messages, want 1", len(sentMessages))
}
if !containsAll(sentMessages[0].recipients, []string{"recipient@example.local", "copy@example.local", "audit@example.local"}) {
t.Fatalf("smtp recipients = %#v", sentMessages[0].recipients)
}
rawSent := string(sentMessages[0].data)
if strings.Contains(rawSent, "audit@example.local") {
t.Fatalf("Bcc recipient leaked into message headers/body:\n%s", rawSent)
}
if !strings.Contains(rawSent, "In-Reply-To: <local-1@example.local>") {
t.Fatalf("reply headers missing from SMTP message:\n%s", rawSent)
}
replied := getJSON[map[string]any](t, fmt.Sprintf("%s/emails/%d", server.URL, emailID))
if replied["local_status"].(map[string]any)["replied"] != true {
t.Fatalf("original message was not marked replied: %#v", replied["local_status"])
}
postJSON[map[string]any](t, server.URL+"/send", map[string]any{
"to": []string{"new@example.local"},
"subject": "Direct local send",
"body": "Hello from direct send.",
}, http.StatusOK)
if len(smtpServer.messages()) != 2 {
t.Fatalf("smtp captured %d messages after /send, want 2", len(smtpServer.messages()))
}
var sentCount int
if err := store.DB.QueryRowContext(ctx, "SELECT COUNT(*) FROM sent_messages WHERE status = 'sent'").Scan(&sentCount); err != nil {
t.Fatalf("count sent messages: %v", err)
}
if sentCount != 2 {
t.Fatalf("sent_messages rows = %d, want 2", sentCount)
}
assertNoStoredAttachmentPathInAPI(t, detail)
}
func buildTestMessage(t *testing.T, date time.Time) []byte {
t.Helper()
var body bytes.Buffer
writer := multipart.NewWriter(&body)
textPart, err := writer.CreatePart(textproto.MIMEHeader{
"Content-Type": {"text/plain; charset=utf-8"},
"Content-Transfer-Encoding": {"quoted-printable"},
})
if err != nil {
t.Fatalf("create text part: %v", err)
}
_, _ = io.WriteString(textPart, "Hello Agent,\r\nThe report is attached.\r\n")
attachmentPart, err := writer.CreatePart(textproto.MIMEHeader{
"Content-Type": {"text/plain; charset=utf-8"},
"Content-Disposition": {`attachment; filename="../report.txt"`},
"Content-Transfer-Encoding": {"base64"},
})
if err != nil {
t.Fatalf("create attachment part: %v", err)
}
_, _ = io.WriteString(attachmentPart, "cmVwb3J0LWJvZHk=")
if err := writer.Close(); err != nil {
t.Fatalf("close multipart: %v", err)
}
headers := strings.Join([]string{
"Message-ID: <local-1@example.local>",
"From: Alice Example <alice@example.local>",
"To: Agent <agent@example.local>",
"Subject: Quarterly Report",
"Date: " + date.Format(time.RFC1123Z),
"Content-Type: multipart/mixed; boundary=" + writer.Boundary(),
"MIME-Version: 1.0",
"",
"",
}, "\r\n")
return []byte(headers + body.String())
}
type popMessage struct {
uid string
raw []byte
}
type fakePOP3Server struct {
host string
port int
ln net.Listener
}
func newFakePOP3Server(t *testing.T, messages []popMessage) *fakePOP3Server {
t.Helper()
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("listen pop3: %v", err)
}
host, portString, _ := net.SplitHostPort(ln.Addr().String())
port, _ := strconv.Atoi(portString)
server := &fakePOP3Server{host: host, port: port, ln: ln}
t.Cleanup(func() { _ = ln.Close() })
go func() {
for {
conn, err := ln.Accept()
if err != nil {
return
}
go handlePOP3Conn(conn, messages)
}
}()
return server
}
func handlePOP3Conn(conn net.Conn, messages []popMessage) {
defer conn.Close()
_ = conn.SetDeadline(time.Now().Add(10 * time.Second))
reader := bufio.NewReader(conn)
writer := bufio.NewWriter(conn)
writeLine(writer, "+OK local POP3 ready")
for {
line, err := reader.ReadString('\n')
if err != nil {
return
}
line = strings.TrimRight(line, "\r\n")
parts := strings.Fields(line)
if len(parts) == 0 {
writeLine(writer, "-ERR empty command")
continue
}
switch strings.ToUpper(parts[0]) {
case "USER", "PASS", "NOOP":
writeLine(writer, "+OK")
case "UIDL":
if len(parts) > 1 {
idx, _ := strconv.Atoi(parts[1])
if idx <= 0 || idx > len(messages) {
writeLine(writer, "-ERR no such message")
continue
}
writeLine(writer, fmt.Sprintf("+OK %d %s", idx, messages[idx-1].uid))
continue
}
writeLine(writer, "+OK uid list follows")
for i, msg := range messages {
writeLine(writer, fmt.Sprintf("%d %s", i+1, msg.uid))
}
writeLine(writer, ".")
case "RETR":
if len(parts) < 2 {
writeLine(writer, "-ERR message id required")
continue
}
idx, _ := strconv.Atoi(parts[1])
if idx <= 0 || idx > len(messages) {
writeLine(writer, "-ERR no such message")
continue
}
writeLine(writer, fmt.Sprintf("+OK %d octets", len(messages[idx-1].raw)))
writeMultiline(writer, messages[idx-1].raw)
case "QUIT":
writeLine(writer, "+OK bye")
return
default:
writeLine(writer, "-ERR unsupported command")
}
}
}
func writeLine(w *bufio.Writer, line string) {
_, _ = w.WriteString(line + "\r\n")
_ = w.Flush()
}
func writeMultiline(w *bufio.Writer, raw []byte) {
for _, line := range strings.Split(strings.ReplaceAll(string(raw), "\r\n", "\n"), "\n") {
if strings.HasPrefix(line, ".") {
line = "." + line
}
_, _ = w.WriteString(line + "\r\n")
}
_, _ = w.WriteString(".\r\n")
_ = w.Flush()
}
type smtpMessage struct {
from string
recipients []string
data []byte
}
type fakeSMTPServer struct {
host string
port int
ln net.Listener
mu sync.Mutex
sent []smtpMessage
}
func newFakeSMTPServer(t *testing.T) *fakeSMTPServer {
t.Helper()
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("listen smtp: %v", err)
}
host, portString, _ := net.SplitHostPort(ln.Addr().String())
port, _ := strconv.Atoi(portString)
server := &fakeSMTPServer{host: host, port: port, ln: ln}
t.Cleanup(func() { _ = ln.Close() })
go func() {
for {
conn, err := ln.Accept()
if err != nil {
return
}
go server.handleConn(conn)
}
}()
return server
}
func (s *fakeSMTPServer) messages() []smtpMessage {
s.mu.Lock()
defer s.mu.Unlock()
out := make([]smtpMessage, len(s.sent))
copy(out, s.sent)
return out
}
func (s *fakeSMTPServer) handleConn(conn net.Conn) {
defer conn.Close()
_ = conn.SetDeadline(time.Now().Add(10 * time.Second))
reader := bufio.NewReader(conn)
writer := bufio.NewWriter(conn)
writeLine(writer, "220 local SMTP ready")
var current smtpMessage
for {
line, err := reader.ReadString('\n')
if err != nil {
return
}
line = strings.TrimRight(line, "\r\n")
upper := strings.ToUpper(line)
switch {
case strings.HasPrefix(upper, "EHLO ") || strings.HasPrefix(upper, "HELO "):
writeLine(writer, "250-localhost")
writeLine(writer, "250-AUTH PLAIN LOGIN")
writeLine(writer, "250 HELP")
case strings.HasPrefix(upper, "AUTH "):
writeLine(writer, "235 2.7.0 Authentication successful")
case strings.HasPrefix(upper, "MAIL FROM:"):
current = smtpMessage{from: extractSMTPPath(line)}
writeLine(writer, "250 OK")
case strings.HasPrefix(upper, "RCPT TO:"):
current.recipients = append(current.recipients, extractSMTPPath(line))
writeLine(writer, "250 OK")
case upper == "DATA":
writeLine(writer, "354 End data with <CR><LF>.<CR><LF>")
data, err := readSMTPData(reader)
if err != nil {
return
}
current.data = data
s.mu.Lock()
s.sent = append(s.sent, current)
s.mu.Unlock()
writeLine(writer, "250 OK queued")
case upper == "RSET":
current = smtpMessage{}
writeLine(writer, "250 OK")
case upper == "QUIT":
writeLine(writer, "221 bye")
return
default:
writeLine(writer, "250 OK")
}
}
}
func extractSMTPPath(line string) string {
_, value, ok := strings.Cut(line, ":")
if !ok {
return ""
}
value = strings.TrimSpace(value)
value = strings.TrimPrefix(value, "<")
value = strings.TrimSuffix(value, ">")
return value
}
func readSMTPData(reader *bufio.Reader) ([]byte, error) {
var out bytes.Buffer
for {
line, err := reader.ReadString('\n')
if err != nil {
return nil, err
}
if line == ".\r\n" || line == ".\n" {
return out.Bytes(), nil
}
if strings.HasPrefix(line, "..") {
line = line[1:]
}
out.WriteString(line)
}
}
func getJSON[T any](t *testing.T, url string) T {
t.Helper()
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
t.Fatalf("new GET request: %v", err)
}
return doJSON[T](t, req, http.StatusOK)
}
func postJSON[T any](t *testing.T, url string, body any, status int) T {
t.Helper()
var reader io.Reader
if body != nil {
payload, err := json.Marshal(body)
if err != nil {
t.Fatalf("marshal body: %v", err)
}
reader = bytes.NewReader(payload)
}
req, err := http.NewRequest(http.MethodPost, url, reader)
if err != nil {
t.Fatalf("new POST request: %v", err)
}
req.Header.Set("Content-Type", "application/json")
return doJSON[T](t, req, status)
}
func doJSON[T any](t *testing.T, req *http.Request, status int) T {
t.Helper()
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("%s %s: %v", req.Method, req.URL, err)
}
defer resp.Body.Close()
payload, err := io.ReadAll(resp.Body)
if err != nil {
t.Fatalf("read response body: %v", err)
}
if resp.StatusCode != status {
t.Fatalf("%s %s status=%d want=%d body=%s", req.Method, req.URL, resp.StatusCode, status, payload)
}
var out T
if err := json.Unmarshal(payload, &out); err != nil {
t.Fatalf("decode response JSON: %v body=%s", err, payload)
}
return out
}
func containsAll(got, want []string) bool {
seen := map[string]bool{}
for _, item := range got {
seen[item] = true
}
for _, item := range want {
if !seen[item] {
return false
}
}
return true
}
func assertNoStoredAttachmentPathInAPI(t *testing.T, detail map[string]any) {
t.Helper()
encoded, err := json.Marshal(detail)
if err != nil {
t.Fatalf("marshal detail: %v", err)
}
if strings.Contains(string(encoded), string(os.PathSeparator)+"attachments"+string(os.PathSeparator)) {
t.Fatalf("API response appears to expose attachment storage path: %s", encoded)
}
}
func TestSentMessagesTableStoresFailedSendWithoutMarkingDraftSent(t *testing.T) {
ctx := context.Background()
dataDir := t.TempDir()
store, err := db.Open(filepath.Join(dataDir, "mail.db"))
if err != nil {
t.Fatalf("open db: %v", err)
}
t.Cleanup(func() { _ = store.Close() })
if err := store.Migrate(ctx); err != nil {
t.Fatalf("migrate db: %v", err)
}
cfg := &config.Config{AccountID: "local-fail"}
svc := service.NewWithOptions(store, nil, failingMailer{}, cfg.AccountID, service.Options{FromAddress: "agent@example.local"})
server := httptest.NewServer(api.New(cfg, store, svc).Handler())
t.Cleanup(server.Close)
draft := postJSON[map[string]any](t, server.URL+"/drafts", map[string]any{
"to": []string{"recipient@example.local"},
"subject": "Will fail",
"body": "Body",
}, http.StatusCreated)
postJSON[map[string]any](t, fmt.Sprintf("%s/drafts/%d/send", server.URL, int64(draft["id"].(float64))), nil, http.StatusBadGateway)
var status, lastError, sentStatus, errorMessage string
err = store.DB.QueryRowContext(ctx, `
SELECT d.status, COALESCE(d.last_error, ''), sm.status, COALESCE(sm.error_message, '')
FROM drafts d JOIN sent_messages sm ON sm.draft_id = d.id
WHERE d.id = ?`, int64(draft["id"].(float64))).Scan(&status, &lastError, &sentStatus, &errorMessage)
if err != nil {
t.Fatalf("query failed send rows: %v", err)
}
if status != "draft" || sentStatus != "failed" || lastError == "" || errorMessage == "" {
t.Fatalf("unexpected failed send persistence: draft=%q lastError=%q sent=%q error=%q", status, lastError, sentStatus, errorMessage)
}
}
type failingMailer struct{}
func (failingMailer) Send(context.Context, mailsmtp.SendRequest) (mailsmtp.SendResult, error) {
return mailsmtp.SendResult{}, fmt.Errorf("local smtp failure")
}

124
internal/model/model.go Normal file
View File

@@ -0,0 +1,124 @@
package model
import "time"
type Address struct {
Name string `json:"name"`
Email string `json:"email"`
}
type ParsedAttachment struct {
Filename string `json:"filename"`
ContentType string `json:"content_type"`
ContentID string `json:"content_id,omitempty"`
SizeBytes int64 `json:"size_bytes"`
Data []byte `json:"-"`
}
type ParsedEmail struct {
MessageIDHeader string `json:"message_id_header"`
InReplyTo string `json:"in_reply_to"`
References string `json:"references_header"`
Subject string `json:"subject"`
SenderName string `json:"sender_name"`
SenderEmail string `json:"sender_email"`
To []Address `json:"to"`
Cc []Address `json:"cc"`
DateHeader string `json:"date_header"`
SentAt *time.Time `json:"sent_at,omitempty"`
BodyText string `json:"body_text"`
BodyHTML string `json:"body_html"`
Attachments []ParsedAttachment `json:"attachments"`
}
type Attachment struct {
ID int64 `json:"attachment_id"`
MessageID int64 `json:"-"`
Filename string `json:"filename"`
ContentType string `json:"content_type"`
ContentID string `json:"content_id,omitempty"`
SizeBytes int64 `json:"size_bytes"`
}
type Label struct {
ID int64 `json:"id"`
Name string `json:"name"`
Color string `json:"color,omitempty"`
}
type LocalStatus struct {
Read bool `json:"read"`
Processed bool `json:"processed"`
Archived bool `json:"archived"`
Deleted bool `json:"deleted"`
Replied bool `json:"replied"`
}
type EmailSummary struct {
ID int64 `json:"id"`
Subject string `json:"subject"`
SenderName string `json:"sender_name"`
SenderEmail string `json:"sender_email"`
To []Address `json:"to"`
Cc []Address `json:"cc,omitempty"`
DateHeader string `json:"date"`
SentAt *time.Time `json:"sent_at,omitempty"`
ReceivedAt *time.Time `json:"received_at,omitempty"`
Snippet string `json:"snippet"`
HasAttachment bool `json:"has_attachment"`
AttachmentCount int `json:"attachment_count"`
LocalStatus LocalStatus `json:"local_status"`
Labels []Label `json:"labels"`
}
type EmailDetail struct {
ID int64 `json:"id"`
MessageIDHeader string `json:"message_id_header"`
InReplyTo string `json:"in_reply_to,omitempty"`
References string `json:"references_header,omitempty"`
Subject string `json:"subject"`
SenderName string `json:"from_name"`
SenderEmail string `json:"from_email"`
To []Address `json:"to"`
Cc []Address `json:"cc"`
DateHeader string `json:"date"`
SentAt *time.Time `json:"sent_at,omitempty"`
ReceivedAt *time.Time `json:"received_at,omitempty"`
BodyText string `json:"body_text"`
BodyHTML string `json:"body_html"`
Attachments []Attachment `json:"attachments"`
LocalStatus LocalStatus `json:"local_status"`
Labels []Label `json:"labels"`
}
type Draft struct {
ID int64 `json:"id"`
To []string `json:"to"`
Cc []string `json:"cc"`
Bcc []string `json:"bcc"`
Subject string `json:"subject"`
Body string `json:"body"`
ReplyToEmailID *int64 `json:"reply_to_email_id,omitempty"`
Status string `json:"status"`
LastError string `json:"last_error,omitempty"`
SentMessageID *int64 `json:"sent_message_id,omitempty"`
CreatedAt *time.Time `json:"created_at,omitempty"`
UpdatedAt *time.Time `json:"updated_at,omitempty"`
}
type FetchedMessage struct {
ServerID int `json:"server_id"`
UID string `json:"uid"`
Raw []byte `json:"-"`
Error string `json:"error,omitempty"`
}
type SentMessage struct {
ID int64 `json:"id"`
DraftID *int64 `json:"draft_id,omitempty"`
Subject string `json:"subject"`
MessageIDHeader string `json:"message_id_header,omitempty"`
Status string `json:"status"`
ErrorMessage string `json:"error_message,omitempty"`
SentAt *time.Time `json:"sent_at,omitempty"`
}

228
internal/parser/parser.go Normal file
View File

@@ -0,0 +1,228 @@
package parser
import (
"bytes"
"encoding/base64"
"fmt"
"io"
"mime"
"mime/multipart"
"mime/quotedprintable"
"net/mail"
"path/filepath"
"strings"
"time"
"golang.org/x/net/html/charset"
"internal-mail-skill/internal/model"
)
func Parse(raw []byte) (model.ParsedEmail, error) {
msg, err := mail.ReadMessage(bytes.NewReader(raw))
if err != nil {
return model.ParsedEmail{}, fmt.Errorf("read message: %w", err)
}
parsed := model.ParsedEmail{
MessageIDHeader: strings.TrimSpace(msg.Header.Get("Message-ID")),
InReplyTo: strings.TrimSpace(msg.Header.Get("In-Reply-To")),
References: strings.TrimSpace(msg.Header.Get("References")),
Subject: decodeHeader(msg.Header.Get("Subject")),
DateHeader: strings.TrimSpace(msg.Header.Get("Date")),
}
if sentAt, err := mail.ParseDate(parsed.DateHeader); err == nil {
parsed.SentAt = &sentAt
}
if from, err := mail.ParseAddress(msg.Header.Get("From")); err == nil {
parsed.SenderName = decodeHeader(from.Name)
parsed.SenderEmail = strings.ToLower(from.Address)
}
parsed.To = parseAddressList(msg.Header.Get("To"))
parsed.Cc = parseAddressList(msg.Header.Get("Cc"))
if err := parsePart(mail.Header(msg.Header), msg.Body, &parsed); err != nil {
return model.ParsedEmail{}, err
}
return parsed, nil
}
func parsePart(header mail.Header, body io.Reader, out *model.ParsedEmail) error {
mediaType, params, err := mime.ParseMediaType(header.Get("Content-Type"))
if err != nil || mediaType == "" {
mediaType = "text/plain"
}
mediaType = strings.ToLower(mediaType)
if strings.HasPrefix(mediaType, "multipart/") {
boundary := params["boundary"]
if boundary == "" {
return fmt.Errorf("multipart message missing boundary")
}
mr := multipart.NewReader(body, boundary)
for {
part, err := mr.NextPart()
if err == io.EOF {
return nil
}
if err != nil {
return fmt.Errorf("read multipart: %w", err)
}
if err := parsePart(mail.Header(part.Header), part, out); err != nil {
return err
}
}
}
decoded := transferDecodedReader(header, body)
filename := partFilename(header)
disposition := strings.ToLower(header.Get("Content-Disposition"))
isAttachment := filename != "" || strings.Contains(disposition, "attachment")
if !isAttachment && strings.HasPrefix(mediaType, "text/") {
text, err := readText(decoded, params["charset"])
if err != nil {
return fmt.Errorf("read text part: %w", err)
}
switch mediaType {
case "text/html":
appendText(&out.BodyHTML, text)
default:
appendText(&out.BodyText, text)
}
return nil
}
data, err := io.ReadAll(decoded)
if err != nil {
return fmt.Errorf("read attachment: %w", err)
}
if filename == "" {
filename = "attachment.bin"
}
out.Attachments = append(out.Attachments, model.ParsedAttachment{
Filename: sanitizeFilename(filename),
ContentType: mediaType,
ContentID: trimAngle(header.Get("Content-ID")),
SizeBytes: int64(len(data)),
Data: data,
})
return nil
}
func decodeHeader(value string) string {
value = strings.TrimSpace(value)
if value == "" {
return ""
}
decoder := &mime.WordDecoder{CharsetReader: charset.NewReaderLabel}
decoded, err := decoder.DecodeHeader(value)
if err != nil {
return value
}
return decoded
}
func parseAddressList(value string) []model.Address {
value = strings.TrimSpace(value)
if value == "" {
return nil
}
list, err := mail.ParseAddressList(value)
if err != nil {
decoded := decodeHeader(value)
list, err = mail.ParseAddressList(decoded)
if err != nil {
return nil
}
}
out := make([]model.Address, 0, len(list))
for _, addr := range list {
out = append(out, model.Address{
Name: decodeHeader(addr.Name),
Email: strings.ToLower(addr.Address),
})
}
return out
}
func transferDecodedReader(header mail.Header, r io.Reader) io.Reader {
switch strings.ToLower(strings.TrimSpace(header.Get("Content-Transfer-Encoding"))) {
case "base64":
return base64.NewDecoder(base64.StdEncoding, r)
case "quoted-printable":
return quotedprintable.NewReader(r)
default:
return r
}
}
func readText(r io.Reader, label string) (string, error) {
if label != "" {
cr, err := charset.NewReaderLabel(label, r)
if err == nil {
r = cr
}
}
data, err := io.ReadAll(r)
if err != nil {
return "", err
}
return string(data), nil
}
func partFilename(header mail.Header) string {
if disp := header.Get("Content-Disposition"); disp != "" {
_, params, err := mime.ParseMediaType(disp)
if err == nil && params["filename"] != "" {
return decodeHeader(params["filename"])
}
}
if ct := header.Get("Content-Type"); ct != "" {
_, params, err := mime.ParseMediaType(ct)
if err == nil && params["name"] != "" {
return decodeHeader(params["name"])
}
}
return ""
}
func appendText(dst *string, value string) {
if strings.TrimSpace(value) == "" {
return
}
if strings.TrimSpace(*dst) == "" {
*dst = value
return
}
*dst += "\n" + value
}
func sanitizeFilename(name string) string {
name = filepath.Base(strings.TrimSpace(name))
name = strings.ReplaceAll(name, "\x00", "")
name = strings.ReplaceAll(name, "/", "_")
name = strings.ReplaceAll(name, "\\", "_")
if name == "." || name == "" {
return "attachment.bin"
}
return name
}
func trimAngle(value string) string {
value = strings.TrimSpace(value)
value = strings.TrimPrefix(value, "<")
value = strings.TrimSuffix(value, ">")
return value
}
func ParseTimeRFC3339(value string) *time.Time {
if value == "" {
return nil
}
t, err := time.Parse(time.RFC3339Nano, value)
if err != nil {
return nil
}
return &t
}

View File

@@ -0,0 +1,63 @@
package parser
import (
"strings"
"testing"
)
func TestParseMultipartEmailWithAttachment(t *testing.T) {
raw := strings.Join([]string{
"Message-ID: <m1@example.com>",
"Subject: =?UTF-8?B?5rWL6K+V6YKu5Lu2?=",
"From: =?UTF-8?B?5byg5LiJ?= <sender@example.com>",
"To: receiver@example.com",
"Cc: cc@example.com",
"Date: Tue, 04 Jun 2024 12:00:00 +0800",
"MIME-Version: 1.0",
"Content-Type: multipart/mixed; boundary=mix",
"",
"--mix",
"Content-Type: multipart/alternative; boundary=alt",
"",
"--alt",
"Content-Type: text/plain; charset=utf-8",
"",
"plain body",
"--alt",
"Content-Type: text/html; charset=utf-8",
"",
"<p>html body</p>",
"--alt--",
"--mix",
"Content-Type: text/plain; name=\"note.txt\"",
"Content-Disposition: attachment; filename=\"note.txt\"",
"Content-Transfer-Encoding: base64",
"",
"YXR0YWNobWVudA==",
"--mix--",
"",
}, "\r\n")
msg, err := Parse([]byte(raw))
if err != nil {
t.Fatalf("Parse() error = %v", err)
}
if msg.Subject != "测试邮件" {
t.Fatalf("Subject = %q", msg.Subject)
}
if msg.SenderName != "张三" || msg.SenderEmail != "sender@example.com" {
t.Fatalf("sender = %q <%s>", msg.SenderName, msg.SenderEmail)
}
if strings.TrimSpace(msg.BodyText) != "plain body" {
t.Fatalf("BodyText = %q", msg.BodyText)
}
if !strings.Contains(msg.BodyHTML, "html body") {
t.Fatalf("BodyHTML = %q", msg.BodyHTML)
}
if len(msg.Attachments) != 1 {
t.Fatalf("attachments = %d", len(msg.Attachments))
}
if msg.Attachments[0].Filename != "note.txt" || string(msg.Attachments[0].Data) != "attachment" {
t.Fatalf("attachment = %#v", msg.Attachments[0])
}
}

69
internal/pop3/pop3.go Normal file
View File

@@ -0,0 +1,69 @@
package pop3
import (
"context"
"fmt"
gopop3 "github.com/knadh/go-pop3"
"internal-mail-skill/internal/config"
"internal-mail-skill/internal/model"
)
type Fetcher struct {
cfg config.POP3Config
}
func NewFetcher(cfg config.POP3Config) *Fetcher {
return &Fetcher{cfg: cfg}
}
func (f *Fetcher) Fetch(ctx context.Context) ([]model.FetchedMessage, error) {
if f.cfg.Host == "" {
return nil, fmt.Errorf("POP3_HOST is required")
}
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
client := gopop3.New(gopop3.Opt{
Host: f.cfg.Host,
Port: f.cfg.Port,
TLSEnabled: f.cfg.UseTLS,
DialTimeout: f.cfg.Timeout(),
})
conn, err := client.NewConn()
if err != nil {
return nil, fmt.Errorf("connect POP3: %w", err)
}
defer conn.Quit()
if err := conn.Auth(f.cfg.Username, f.cfg.Password); err != nil {
return nil, fmt.Errorf("POP3 auth: %w", err)
}
ids, err := conn.Uidl(0)
if err != nil {
return nil, fmt.Errorf("POP3 UIDL: %w", err)
}
out := make([]model.FetchedMessage, 0, len(ids))
for _, item := range ids {
select {
case <-ctx.Done():
return out, ctx.Err()
default:
}
raw, err := conn.RetrRaw(item.ID)
if err != nil {
out = append(out, model.FetchedMessage{ServerID: item.ID, UID: item.UID, Error: fmt.Sprintf("POP3 RETR %d: %v", item.ID, err)})
continue
}
out = append(out, model.FetchedMessage{
ServerID: item.ID,
UID: item.UID,
Raw: raw.Bytes(),
})
}
return out, nil
}

1127
internal/service/service.go Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,210 @@
package service
import (
"context"
"errors"
"os"
"path/filepath"
"testing"
"time"
"internal-mail-skill/internal/db"
"internal-mail-skill/internal/model"
mailsmtp "internal-mail-skill/internal/smtp"
)
func TestThreadAssemblyLinksReferences(t *testing.T) {
ctx := context.Background()
store, err := db.Open(":memory:")
if err != nil {
t.Fatal(err)
}
defer store.Close()
if err := store.Migrate(ctx); err != nil {
t.Fatal(err)
}
svc := New(store, nil, nil, "default")
root, err := svc.SaveParsedEmail(ctx, SaveEmailInput{
POP3UID: "uid-root",
Parsed: model.ParsedEmail{
MessageIDHeader: "<root@example.com>",
Subject: "Root",
SenderEmail: "a@example.com",
},
})
if err != nil {
t.Fatal(err)
}
reply, err := svc.SaveParsedEmail(ctx, SaveEmailInput{
POP3UID: "uid-reply",
Parsed: model.ParsedEmail{
MessageIDHeader: "<reply@example.com>",
InReplyTo: "<root@example.com>",
References: "<root@example.com>",
Subject: "Re: Root",
SenderEmail: "b@example.com",
},
})
if err != nil {
t.Fatal(err)
}
thread, err := svc.GetThread(ctx, reply)
if err != nil {
t.Fatal(err)
}
if len(thread) != 2 || thread[0].ID != root || thread[1].ID != reply {
t.Fatalf("thread = %#v", thread)
}
}
func TestSearchMarkLabelsAndDraftLifecycle(t *testing.T) {
ctx := context.Background()
store, err := db.Open(":memory:")
if err != nil {
t.Fatal(err)
}
defer store.Close()
if err := store.Migrate(ctx); err != nil {
t.Fatal(err)
}
svc := New(store, nil, nil, "default")
emailID, err := svc.SaveParsedEmail(ctx, SaveEmailInput{
POP3UID: "uid-1",
Parsed: model.ParsedEmail{
MessageIDHeader: "<m@example.com>",
Subject: "Quarterly Plan",
BodyText: "Please review",
SenderEmail: "boss@example.com",
},
})
if err != nil {
t.Fatal(err)
}
if err := svc.MarkEmail(ctx, emailID, MarkInput{Read: boolPtr(true), Processed: boolPtr(true)}); err != nil {
t.Fatal(err)
}
label, err := svc.CreateLabel(ctx, "work", "#336699")
if err != nil {
t.Fatal(err)
}
if err := svc.ApplyLabels(ctx, []int64{emailID}, []int64{label.ID}, "add"); err != nil {
t.Fatal(err)
}
results, err := svc.SearchEmails(ctx, SearchFilters{Q: "Quarterly", Read: boolPtr(true), Limit: 10})
if err != nil {
t.Fatal(err)
}
if len(results) != 1 || results[0].Labels[0].Name != "work" {
t.Fatalf("search results = %#v", results)
}
draft, err := svc.CreateDraft(ctx, DraftInput{To: []string{"x@example.com"}, Subject: "Draft", Body: "Hi"})
if err != nil {
t.Fatal(err)
}
updated, err := svc.UpdateDraft(ctx, draft.ID, DraftInput{To: []string{"x@example.com"}, Subject: "Updated", Body: "Hello"})
if err != nil {
t.Fatal(err)
}
if updated.Subject != "Updated" {
t.Fatalf("updated subject = %q", updated.Subject)
}
}
func TestSaveParsedEmailRollsBackWhenAttachmentStorageFails(t *testing.T) {
ctx := context.Background()
store, err := db.Open(":memory:")
if err != nil {
t.Fatal(err)
}
defer store.Close()
if err := store.Migrate(ctx); err != nil {
t.Fatal(err)
}
dir := t.TempDir()
blocker := filepath.Join(dir, "attachments-blocker")
if err := os.WriteFile(blocker, []byte("not a directory"), 0600); err != nil {
t.Fatal(err)
}
svc := NewWithOptions(store, nil, nil, "default", Options{AttachmentDir: blocker})
_, err = svc.SaveParsedEmail(ctx, SaveEmailInput{
POP3UID: "uid-broken-attachment",
Parsed: model.ParsedEmail{
MessageIDHeader: "<broken@example.com>",
Subject: "Broken Attachment",
SenderEmail: "a@example.com",
Attachments: []model.ParsedAttachment{{
Filename: "note.txt",
ContentType: "text/plain",
SizeBytes: 4,
Data: []byte("nope"),
}},
},
})
if err == nil {
t.Fatal("SaveParsedEmail() error = nil, want attachment storage failure")
}
results, err := svc.SearchEmails(ctx, SearchFilters{Q: "Broken Attachment", Limit: 10})
if err != nil {
t.Fatal(err)
}
if len(results) != 0 {
t.Fatalf("failed save left searchable message: %#v", results)
}
}
func TestSendDraftIsIdempotentAfterSuccess(t *testing.T) {
ctx := context.Background()
store, err := db.Open(":memory:")
if err != nil {
t.Fatal(err)
}
defer store.Close()
if err := store.Migrate(ctx); err != nil {
t.Fatal(err)
}
mailer := &fakeMailer{}
svc := NewWithOptions(store, nil, mailer, "default", Options{FromAddress: "sender@example.com"})
draft, err := svc.CreateDraft(ctx, DraftInput{To: []string{"x@example.com"}, Subject: "Once", Body: "hello"})
if err != nil {
t.Fatal(err)
}
first, err := svc.SendDraft(ctx, draft.ID)
if err != nil {
t.Fatal(err)
}
second, err := svc.SendDraft(ctx, draft.ID)
if err != nil {
t.Fatal(err)
}
if mailer.calls != 1 {
t.Fatalf("mailer calls = %d, want 1", mailer.calls)
}
if second.SentMessage.ID != first.SentMessage.ID {
t.Fatalf("second send message id = %d, want %d", second.SentMessage.ID, first.SentMessage.ID)
}
}
func boolPtr(v bool) *bool { return &v }
type fakeMailer struct {
calls int
err error
}
func (f *fakeMailer) Send(ctx context.Context, req mailsmtp.SendRequest) (mailsmtp.SendResult, error) {
f.calls++
if f.err != nil {
return mailsmtp.SendResult{}, f.err
}
if req.FromAddress == "" {
return mailsmtp.SendResult{}, errors.New("from required")
}
return mailsmtp.SendResult{MessageIDHeader: "<sent@example.com>", SentAt: time.Now().UTC()}, nil
}

View File

@@ -0,0 +1,90 @@
package smtp
import (
stdsmtp "net/smtp"
"strings"
"testing"
"internal-mail-skill/internal/config"
)
func TestBuildMessageOmitsBccHeaderButKeepsRecipients(t *testing.T) {
req := SendRequest{
FromAddress: "agent@example.com",
FromName: "Agent",
To: []string{"to@example.com"},
Cc: []string{"cc@example.com"},
Bcc: []string{"hidden@example.com"},
Subject: "Hello",
Body: "Body",
}
msg, recipients, err := BuildMessage(req)
if err != nil {
t.Fatalf("BuildMessage() error = %v", err)
}
raw := string(msg)
if strings.Contains(raw, "Bcc:") || strings.Contains(raw, "hidden@example.com") {
t.Fatalf("raw message leaked Bcc: %s", raw)
}
if len(recipients) != 3 {
t.Fatalf("recipients = %#v", recipients)
}
if recipients[2] != "hidden@example.com" {
t.Fatalf("Bcc missing from envelope recipients: %#v", recipients)
}
}
func TestAuthForConfigRequiresExplicitOptInForCleartextAuth(t *testing.T) {
cfg := config.SMTPConfig{
Host: "smtp.example.local",
Username: "user",
Password: "secret",
AuthRequired: true,
}
if _, err := authForConfig(cfg); err == nil {
t.Fatal("authForConfig() error = nil, want cleartext auth rejection")
}
cfg.AllowInsecureAuth = true
auth, err := authForConfig(cfg)
if err != nil {
t.Fatalf("authForConfig() with opt-in error = %v", err)
}
proto, _, err := auth.Start(&stdsmtp.ServerInfo{Name: cfg.Host, TLS: false})
if err != nil {
t.Fatalf("insecure auth Start() error = %v", err)
}
if proto != "PLAIN" {
t.Fatalf("proto = %q, want PLAIN", proto)
}
cfg.AllowInsecureAuth = false
cfg.UseSTARTTLS = true
auth, err = authForConfig(cfg)
if err != nil {
t.Fatalf("authForConfig() over STARTTLS error = %v", err)
}
if _, _, err := auth.Start(&stdsmtp.ServerInfo{Name: cfg.Host, TLS: true}); err != nil {
t.Fatalf("TLS auth Start() error = %v", err)
}
}
func TestAuthForConfigAllowsImplicitTLSAuth(t *testing.T) {
cfg := config.SMTPConfig{
Host: "smtp.example.local",
Username: "user",
Password: "secret",
AuthRequired: true,
UseTLS: true,
}
auth, err := authForConfig(cfg)
if err != nil {
t.Fatalf("authForConfig() error = %v", err)
}
if _, _, err := auth.Start(&stdsmtp.ServerInfo{Name: cfg.Host, TLS: false}); err != nil {
t.Fatalf("implicit TLS auth Start() error = %v", err)
}
}

252
internal/smtp/smtp.go Normal file
View File

@@ -0,0 +1,252 @@
package smtp
import (
"bytes"
"context"
"crypto/rand"
"crypto/tls"
"encoding/hex"
"fmt"
"io"
"mime"
"net"
"net/mail"
stdsmtp "net/smtp"
"os"
"strings"
"time"
"internal-mail-skill/internal/config"
)
type SendRequest struct {
FromAddress string
FromName string
To []string
Cc []string
Bcc []string
Subject string
Body string
InReplyTo string
References string
MessageIDHeader string
}
type SendResult struct {
MessageIDHeader string
Raw []byte
Recipients []string
SentAt time.Time
}
type Sender struct {
cfg config.SMTPConfig
}
func NewSender(cfg config.SMTPConfig) *Sender {
return &Sender{cfg: cfg}
}
func (s *Sender) Send(ctx context.Context, req SendRequest) (SendResult, error) {
if req.FromAddress == "" {
req.FromAddress = s.cfg.FromAddress
}
if req.FromName == "" {
req.FromName = s.cfg.FromName
}
raw, recipients, err := BuildMessage(req)
if err != nil {
return SendResult{}, err
}
if len(recipients) == 0 {
return SendResult{}, fmt.Errorf("no recipients")
}
if s.cfg.Host == "" {
return SendResult{}, fmt.Errorf("SMTP_HOST is required")
}
auth, err := authForConfig(s.cfg)
if err != nil {
return SendResult{}, err
}
if err := s.sendRaw(ctx, req.FromAddress, recipients, raw, auth); err != nil {
return SendResult{}, err
}
return SendResult{
MessageIDHeader: extractHeader(raw, "Message-ID"),
Raw: raw,
Recipients: recipients,
SentAt: time.Now().UTC(),
}, nil
}
func BuildMessage(req SendRequest) ([]byte, []string, error) {
if req.FromAddress == "" {
return nil, nil, fmt.Errorf("from address is required")
}
recipients := append([]string{}, req.To...)
recipients = append(recipients, req.Cc...)
recipients = append(recipients, req.Bcc...)
if len(recipients) == 0 {
return nil, nil, fmt.Errorf("at least one recipient is required")
}
messageID := req.MessageIDHeader
if messageID == "" {
messageID = newMessageID()
}
var b bytes.Buffer
writeHeader(&b, "From", (&mail.Address{Name: req.FromName, Address: req.FromAddress}).String())
writeHeader(&b, "To", strings.Join(req.To, ", "))
if len(req.Cc) > 0 {
writeHeader(&b, "Cc", strings.Join(req.Cc, ", "))
}
writeHeader(&b, "Subject", mime.QEncoding.Encode("utf-8", req.Subject))
writeHeader(&b, "Date", time.Now().Format(time.RFC1123Z))
writeHeader(&b, "Message-ID", messageID)
if req.InReplyTo != "" {
writeHeader(&b, "In-Reply-To", req.InReplyTo)
}
if req.References != "" {
writeHeader(&b, "References", req.References)
}
writeHeader(&b, "MIME-Version", "1.0")
writeHeader(&b, "Content-Type", `text/plain; charset="utf-8"`)
writeHeader(&b, "Content-Transfer-Encoding", "8bit")
b.WriteString("\r\n")
body := strings.ReplaceAll(req.Body, "\n", "\r\n")
body = strings.ReplaceAll(body, "\r\r\n", "\r\n")
b.WriteString(body)
if !strings.HasSuffix(body, "\r\n") {
b.WriteString("\r\n")
}
return b.Bytes(), recipients, nil
}
func (s *Sender) sendRaw(ctx context.Context, from string, recipients []string, raw []byte, auth stdsmtp.Auth) error {
timeout := s.cfg.Timeout()
if timeout <= 0 {
timeout = 30 * time.Second
}
address := net.JoinHostPort(s.cfg.Host, fmt.Sprintf("%d", s.cfg.Port))
dialer := &net.Dialer{Timeout: timeout}
var conn net.Conn
var err error
if s.cfg.UseTLS {
conn, err = tls.DialWithDialer(dialer, "tcp", address, &tls.Config{ServerName: s.cfg.Host, MinVersion: tls.VersionTLS12})
} else {
conn, err = dialer.DialContext(ctx, "tcp", address)
}
if err != nil {
return fmt.Errorf("connect SMTP: %w", err)
}
defer conn.Close()
client, err := stdsmtp.NewClient(conn, s.cfg.Host)
if err != nil {
return fmt.Errorf("create SMTP client: %w", err)
}
defer client.Close()
if s.cfg.UseSTARTTLS {
if err := client.StartTLS(&tls.Config{ServerName: s.cfg.Host, MinVersion: tls.VersionTLS12}); err != nil {
return fmt.Errorf("starttls: %w", err)
}
}
if auth != nil {
if err := client.Auth(auth); err != nil {
return fmt.Errorf("smtp auth: %w", err)
}
}
if err := client.Mail(from); err != nil {
return fmt.Errorf("smtp MAIL FROM: %w", err)
}
for _, recipient := range recipients {
if strings.TrimSpace(recipient) == "" {
continue
}
if err := client.Rcpt(recipient); err != nil {
return fmt.Errorf("smtp RCPT TO %s: %w", recipient, err)
}
}
writer, err := client.Data()
if err != nil {
return fmt.Errorf("smtp DATA: %w", err)
}
if _, err := io.Copy(writer, bytes.NewReader(raw)); err != nil {
_ = writer.Close()
return fmt.Errorf("write SMTP body: %w", err)
}
if err := writer.Close(); err != nil {
return fmt.Errorf("close SMTP body: %w", err)
}
if err := client.Quit(); err != nil {
return fmt.Errorf("smtp quit: %w", err)
}
return nil
}
func authForConfig(cfg config.SMTPConfig) (stdsmtp.Auth, error) {
if !cfg.AuthRequired {
return nil, nil
}
if !cfg.UseTLS && !cfg.UseSTARTTLS {
if cfg.AllowInsecureAuth {
return plainAuthNoTLSCheck{username: cfg.Username, password: cfg.Password}, nil
}
if cfg.Host != "localhost" && cfg.Host != "127.0.0.1" && cfg.Host != "::1" {
return nil, fmt.Errorf("SMTP auth over cleartext requires SMTP_USE_TLS=true, SMTP_USE_STARTTLS=true, or SMTP_ALLOW_INSECURE_AUTH=true")
}
}
if cfg.UseTLS {
return plainAuthNoTLSCheck{username: cfg.Username, password: cfg.Password}, nil
}
return stdsmtp.PlainAuth("", cfg.Username, cfg.Password, cfg.Host), nil
}
type plainAuthNoTLSCheck struct {
username string
password string
}
func (a plainAuthNoTLSCheck) Start(server *stdsmtp.ServerInfo) (string, []byte, error) {
resp := []byte("\x00" + a.username + "\x00" + a.password)
return "PLAIN", resp, nil
}
func (a plainAuthNoTLSCheck) Next(fromServer []byte, more bool) ([]byte, error) {
if more {
return nil, fmt.Errorf("unexpected server challenge for PLAIN auth")
}
return nil, nil
}
func writeHeader(b *bytes.Buffer, key, value string) {
if strings.TrimSpace(value) == "" {
return
}
b.WriteString(key)
b.WriteString(": ")
b.WriteString(value)
b.WriteString("\r\n")
}
func newMessageID() string {
host, err := os.Hostname()
if err != nil || host == "" {
host = "localhost"
}
var random [8]byte
if _, err := rand.Read(random[:]); err != nil {
return fmt.Sprintf("<%d@%s>", time.Now().UnixNano(), host)
}
return fmt.Sprintf("<%d.%s@%s>", time.Now().UnixNano(), hex.EncodeToString(random[:]), host)
}
func extractHeader(raw []byte, name string) string {
prefix := strings.ToLower(name) + ":"
for _, line := range strings.Split(string(raw), "\r\n") {
if strings.HasPrefix(strings.ToLower(line), prefix) {
return strings.TrimSpace(line[len(name)+1:])
}
}
return ""
}