chore: initialize qiming workspace repository
This commit is contained in:
8
qimingcode/packages/opencode/src/storage/db.bun.ts
Normal file
8
qimingcode/packages/opencode/src/storage/db.bun.ts
Normal file
@@ -0,0 +1,8 @@
|
||||
import { Database } from "bun:sqlite"
|
||||
import { drizzle } from "drizzle-orm/bun-sqlite"
|
||||
|
||||
export function init(path: string) {
|
||||
const sqlite = new Database(path, { create: true })
|
||||
const db = drizzle({ client: sqlite })
|
||||
return db
|
||||
}
|
||||
8
qimingcode/packages/opencode/src/storage/db.node.ts
Normal file
8
qimingcode/packages/opencode/src/storage/db.node.ts
Normal file
@@ -0,0 +1,8 @@
|
||||
import { DatabaseSync } from "node:sqlite"
|
||||
import { drizzle } from "drizzle-orm/node-sqlite"
|
||||
|
||||
export function init(path: string) {
|
||||
const sqlite = new DatabaseSync(path)
|
||||
const db = drizzle({ client: sqlite })
|
||||
return db
|
||||
}
|
||||
172
qimingcode/packages/opencode/src/storage/db.ts
Normal file
172
qimingcode/packages/opencode/src/storage/db.ts
Normal file
@@ -0,0 +1,172 @@
|
||||
import { type SQLiteBunDatabase } from "drizzle-orm/bun-sqlite"
|
||||
import { migrate } from "drizzle-orm/bun-sqlite/migrator"
|
||||
import { type SQLiteTransaction } from "drizzle-orm/sqlite-core"
|
||||
export * from "drizzle-orm"
|
||||
import { LocalContext } from "../util"
|
||||
import { lazy } from "../util/lazy"
|
||||
import { Global } from "@opencode-ai/core/global"
|
||||
import { Log } from "../util"
|
||||
import { NamedError } from "@opencode-ai/core/util/error"
|
||||
import z from "zod"
|
||||
import path from "path"
|
||||
import { readFileSync, readdirSync, existsSync } from "fs"
|
||||
import { Flag } from "@opencode-ai/core/flag/flag"
|
||||
import { InstallationChannel } from "@opencode-ai/core/installation/version"
|
||||
import { InstanceState } from "@/effect"
|
||||
import { iife } from "@/util/iife"
|
||||
import { init } from "#db"
|
||||
|
||||
declare const OPENCODE_MIGRATIONS: { sql: string; timestamp: number; name: string }[] | undefined
|
||||
|
||||
export const NotFoundError = NamedError.create(
|
||||
"NotFoundError",
|
||||
z.object({
|
||||
message: z.string(),
|
||||
}),
|
||||
)
|
||||
|
||||
const log = Log.create({ service: "db" })
|
||||
|
||||
export function getChannelPath() {
|
||||
if (["latest", "beta", "prod"].includes(InstallationChannel) || Flag.OPENCODE_DISABLE_CHANNEL_DB)
|
||||
return path.join(Global.Path.data, "opencode.db")
|
||||
const safe = InstallationChannel.replace(/[^a-zA-Z0-9._-]/g, "-")
|
||||
return path.join(Global.Path.data, `opencode-${safe}.db`)
|
||||
}
|
||||
|
||||
export const Path = iife(() => {
|
||||
if (Flag.OPENCODE_DB) {
|
||||
if (Flag.OPENCODE_DB === ":memory:" || path.isAbsolute(Flag.OPENCODE_DB)) return Flag.OPENCODE_DB
|
||||
return path.join(Global.Path.data, Flag.OPENCODE_DB)
|
||||
}
|
||||
return getChannelPath()
|
||||
})
|
||||
|
||||
export type Transaction = SQLiteTransaction<"sync", void>
|
||||
|
||||
type Client = SQLiteBunDatabase
|
||||
|
||||
type Journal = { sql: string; timestamp: number; name: string }[]
|
||||
|
||||
function time(tag: string) {
|
||||
const match = /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})/.exec(tag)
|
||||
if (!match) return 0
|
||||
return Date.UTC(
|
||||
Number(match[1]),
|
||||
Number(match[2]) - 1,
|
||||
Number(match[3]),
|
||||
Number(match[4]),
|
||||
Number(match[5]),
|
||||
Number(match[6]),
|
||||
)
|
||||
}
|
||||
|
||||
function migrations(dir: string): Journal {
|
||||
const dirs = readdirSync(dir, { withFileTypes: true })
|
||||
.filter((entry) => entry.isDirectory())
|
||||
.map((entry) => entry.name)
|
||||
|
||||
const sql = dirs
|
||||
.map((name) => {
|
||||
const file = path.join(dir, name, "migration.sql")
|
||||
if (!existsSync(file)) return
|
||||
return {
|
||||
sql: readFileSync(file, "utf-8"),
|
||||
timestamp: time(name),
|
||||
name,
|
||||
}
|
||||
})
|
||||
.filter(Boolean) as Journal
|
||||
|
||||
return sql.sort((a, b) => a.timestamp - b.timestamp)
|
||||
}
|
||||
|
||||
export const Client = lazy(() => {
|
||||
log.info("opening database", { path: Path })
|
||||
|
||||
const db = init(Path)
|
||||
|
||||
db.run("PRAGMA journal_mode = WAL")
|
||||
db.run("PRAGMA synchronous = NORMAL")
|
||||
db.run("PRAGMA busy_timeout = 5000")
|
||||
db.run("PRAGMA cache_size = -64000")
|
||||
db.run("PRAGMA foreign_keys = ON")
|
||||
db.run("PRAGMA wal_checkpoint(PASSIVE)")
|
||||
|
||||
// Apply schema migrations
|
||||
const entries =
|
||||
typeof OPENCODE_MIGRATIONS !== "undefined"
|
||||
? OPENCODE_MIGRATIONS
|
||||
: migrations(path.join(import.meta.dirname, "../../migration"))
|
||||
if (entries.length > 0) {
|
||||
log.info("applying migrations", {
|
||||
count: entries.length,
|
||||
mode: typeof OPENCODE_MIGRATIONS !== "undefined" ? "bundled" : "dev",
|
||||
})
|
||||
if (Flag.OPENCODE_SKIP_MIGRATIONS) {
|
||||
for (const item of entries) {
|
||||
item.sql = "select 1;"
|
||||
}
|
||||
}
|
||||
migrate(db, entries)
|
||||
}
|
||||
|
||||
return db
|
||||
})
|
||||
|
||||
export function close() {
|
||||
Client().$client.close()
|
||||
Client.reset()
|
||||
}
|
||||
|
||||
export type TxOrDb = Transaction | Client
|
||||
|
||||
const ctx = LocalContext.create<{
|
||||
tx: TxOrDb
|
||||
effects: (() => void | Promise<void>)[]
|
||||
}>("database")
|
||||
|
||||
export function use<T>(callback: (trx: TxOrDb) => T): T {
|
||||
try {
|
||||
return callback(ctx.use().tx)
|
||||
} catch (err) {
|
||||
if (err instanceof LocalContext.NotFound) {
|
||||
const effects: (() => void | Promise<void>)[] = []
|
||||
const result = ctx.provide({ effects, tx: Client() }, () => callback(Client()))
|
||||
for (const effect of effects) effect()
|
||||
return result
|
||||
}
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
export function effect(fn: () => any | Promise<any>) {
|
||||
const bound = InstanceState.bind(fn)
|
||||
try {
|
||||
ctx.use().effects.push(bound)
|
||||
} catch {
|
||||
bound()
|
||||
}
|
||||
}
|
||||
|
||||
type NotPromise<T> = T extends Promise<any> ? never : T
|
||||
|
||||
export function transaction<T>(
|
||||
callback: (tx: TxOrDb) => NotPromise<T>,
|
||||
options?: {
|
||||
behavior?: "deferred" | "immediate" | "exclusive"
|
||||
},
|
||||
): NotPromise<T> {
|
||||
try {
|
||||
return callback(ctx.use().tx)
|
||||
} catch (err) {
|
||||
if (err instanceof LocalContext.NotFound) {
|
||||
const effects: (() => void | Promise<void>)[] = []
|
||||
const txCallback = InstanceState.bind((tx: TxOrDb) => ctx.provide({ tx, effects }, () => callback(tx)))
|
||||
const result = Client().transaction(txCallback, { behavior: options?.behavior })
|
||||
for (const effect of effects) effect()
|
||||
return result as NotPromise<T>
|
||||
}
|
||||
throw err
|
||||
}
|
||||
}
|
||||
26
qimingcode/packages/opencode/src/storage/index.ts
Normal file
26
qimingcode/packages/opencode/src/storage/index.ts
Normal file
@@ -0,0 +1,26 @@
|
||||
export * as JsonMigration from "./json-migration"
|
||||
export * as Database from "./db"
|
||||
export * as Storage from "./storage"
|
||||
export {
|
||||
asc,
|
||||
eq,
|
||||
and,
|
||||
or,
|
||||
inArray,
|
||||
desc,
|
||||
not,
|
||||
sql,
|
||||
isNull,
|
||||
isNotNull,
|
||||
count,
|
||||
like,
|
||||
exists,
|
||||
between,
|
||||
gt,
|
||||
gte,
|
||||
lt,
|
||||
lte,
|
||||
ne,
|
||||
} from "drizzle-orm"
|
||||
export type { SQL } from "drizzle-orm"
|
||||
export { NotFoundError } from "./storage"
|
||||
428
qimingcode/packages/opencode/src/storage/json-migration.ts
Normal file
428
qimingcode/packages/opencode/src/storage/json-migration.ts
Normal file
@@ -0,0 +1,428 @@
|
||||
import type { SQLiteBunDatabase } from "drizzle-orm/bun-sqlite"
|
||||
import type { NodeSQLiteDatabase } from "drizzle-orm/node-sqlite"
|
||||
import { Global } from "@opencode-ai/core/global"
|
||||
import { Log } from "../util"
|
||||
import { ProjectTable } from "../project/project.sql"
|
||||
import { SessionTable, MessageTable, PartTable, TodoTable, PermissionTable } from "../session/session.sql"
|
||||
import { SessionShareTable } from "../share/share.sql"
|
||||
import path from "path"
|
||||
import { existsSync } from "fs"
|
||||
import { Filesystem } from "../util"
|
||||
import { Glob } from "@opencode-ai/core/util/glob"
|
||||
|
||||
const log = Log.create({ service: "json-migration" })
|
||||
|
||||
export type Progress = {
|
||||
current: number
|
||||
total: number
|
||||
label: string
|
||||
}
|
||||
|
||||
type Options = {
|
||||
progress?: (event: Progress) => void
|
||||
}
|
||||
|
||||
export async function run(db: SQLiteBunDatabase<any, any> | NodeSQLiteDatabase<any, any>, options?: Options) {
|
||||
const storageDir = path.join(Global.Path.data, "storage")
|
||||
|
||||
if (!existsSync(storageDir)) {
|
||||
log.info("storage directory does not exist, skipping migration")
|
||||
return {
|
||||
projects: 0,
|
||||
sessions: 0,
|
||||
messages: 0,
|
||||
parts: 0,
|
||||
todos: 0,
|
||||
permissions: 0,
|
||||
shares: 0,
|
||||
errors: [] as string[],
|
||||
}
|
||||
}
|
||||
|
||||
log.info("starting json to sqlite migration", { storageDir })
|
||||
const start = performance.now()
|
||||
|
||||
// const db = drizzle({ client: sqlite })
|
||||
|
||||
// Optimize SQLite for bulk inserts
|
||||
db.run("PRAGMA journal_mode = WAL")
|
||||
db.run("PRAGMA synchronous = OFF")
|
||||
db.run("PRAGMA cache_size = 10000")
|
||||
db.run("PRAGMA temp_store = MEMORY")
|
||||
const stats = {
|
||||
projects: 0,
|
||||
sessions: 0,
|
||||
messages: 0,
|
||||
parts: 0,
|
||||
todos: 0,
|
||||
permissions: 0,
|
||||
shares: 0,
|
||||
errors: [] as string[],
|
||||
}
|
||||
const orphans = {
|
||||
sessions: 0,
|
||||
todos: 0,
|
||||
permissions: 0,
|
||||
shares: 0,
|
||||
}
|
||||
const errs = stats.errors
|
||||
|
||||
const batchSize = 1000
|
||||
const now = Date.now()
|
||||
|
||||
async function list(pattern: string) {
|
||||
return Glob.scan(pattern, { cwd: storageDir, absolute: true })
|
||||
}
|
||||
|
||||
async function read(files: string[], start: number, end: number) {
|
||||
const count = end - start
|
||||
// oxlint-disable-next-line unicorn/no-new-array -- pre-allocated for index-based batch fill
|
||||
const tasks = new Array(count)
|
||||
for (let i = 0; i < count; i++) {
|
||||
tasks[i] = Filesystem.readJson(files[start + i])
|
||||
}
|
||||
const results = await Promise.allSettled(tasks)
|
||||
// oxlint-disable-next-line unicorn/no-new-array -- pre-allocated for index-based batch fill
|
||||
const items = new Array(count)
|
||||
for (let i = 0; i < results.length; i++) {
|
||||
const result = results[i]
|
||||
if (result.status === "fulfilled") {
|
||||
items[i] = result.value
|
||||
continue
|
||||
}
|
||||
errs.push(`failed to read ${files[start + i]}: ${result.reason}`)
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
function insert(values: unknown[], table: Parameters<typeof db.insert>[0], label: string) {
|
||||
if (values.length === 0) return 0
|
||||
try {
|
||||
db.insert(table).values(values).onConflictDoNothing().run()
|
||||
return values.length
|
||||
} catch (e) {
|
||||
errs.push(`failed to migrate ${label} batch: ${e}`)
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
// Pre-scan all files upfront to avoid repeated glob operations
|
||||
log.info("scanning files...")
|
||||
const [projectFiles, sessionFiles, messageFiles, partFiles, todoFiles, permFiles, shareFiles] = await Promise.all([
|
||||
list("project/*.json"),
|
||||
list("session/*/*.json"),
|
||||
list("message/*/*.json"),
|
||||
list("part/*/*.json"),
|
||||
list("todo/*.json"),
|
||||
list("permission/*.json"),
|
||||
list("session_share/*.json"),
|
||||
])
|
||||
|
||||
log.info("file scan complete", {
|
||||
projects: projectFiles.length,
|
||||
sessions: sessionFiles.length,
|
||||
messages: messageFiles.length,
|
||||
parts: partFiles.length,
|
||||
todos: todoFiles.length,
|
||||
permissions: permFiles.length,
|
||||
shares: shareFiles.length,
|
||||
})
|
||||
|
||||
const total = Math.max(
|
||||
1,
|
||||
projectFiles.length +
|
||||
sessionFiles.length +
|
||||
messageFiles.length +
|
||||
partFiles.length +
|
||||
todoFiles.length +
|
||||
permFiles.length +
|
||||
shareFiles.length,
|
||||
)
|
||||
const progress = options?.progress
|
||||
let current = 0
|
||||
const step = (label: string, count: number) => {
|
||||
current = Math.min(total, current + count)
|
||||
progress?.({ current, total, label })
|
||||
}
|
||||
|
||||
progress?.({ current, total, label: "starting" })
|
||||
|
||||
db.run("BEGIN TRANSACTION")
|
||||
|
||||
// Migrate projects first (no FK deps)
|
||||
// Derive all IDs from file paths, not JSON content
|
||||
const projectIds = new Set<string>()
|
||||
const projectValues: unknown[] = []
|
||||
for (let i = 0; i < projectFiles.length; i += batchSize) {
|
||||
const end = Math.min(i + batchSize, projectFiles.length)
|
||||
const batch = await read(projectFiles, i, end)
|
||||
projectValues.length = 0
|
||||
for (let j = 0; j < batch.length; j++) {
|
||||
const data = batch[j]
|
||||
if (!data) continue
|
||||
const id = path.basename(projectFiles[i + j], ".json")
|
||||
projectIds.add(id)
|
||||
projectValues.push({
|
||||
id,
|
||||
worktree: data.worktree ?? "/",
|
||||
vcs: data.vcs,
|
||||
name: data.name ?? undefined,
|
||||
icon_url: data.icon?.url,
|
||||
icon_url_override: data.icon?.override,
|
||||
icon_color: data.icon?.color,
|
||||
time_created: data.time?.created ?? now,
|
||||
time_updated: data.time?.updated ?? now,
|
||||
time_initialized: data.time?.initialized,
|
||||
sandboxes: data.sandboxes ?? [],
|
||||
commands: data.commands,
|
||||
})
|
||||
}
|
||||
stats.projects += insert(projectValues, ProjectTable, "project")
|
||||
step("projects", end - i)
|
||||
}
|
||||
log.info("migrated projects", { count: stats.projects, duration: Math.round(performance.now() - start) })
|
||||
|
||||
// Migrate sessions (depends on projects)
|
||||
// Derive all IDs from directory/file paths, not JSON content, since earlier
|
||||
// migrations may have moved sessions to new directories without updating the JSON
|
||||
const sessionProjects = sessionFiles.map((file) => path.basename(path.dirname(file)))
|
||||
const sessionIds = new Set<string>()
|
||||
const sessionValues: unknown[] = []
|
||||
for (let i = 0; i < sessionFiles.length; i += batchSize) {
|
||||
const end = Math.min(i + batchSize, sessionFiles.length)
|
||||
const batch = await read(sessionFiles, i, end)
|
||||
sessionValues.length = 0
|
||||
for (let j = 0; j < batch.length; j++) {
|
||||
const data = batch[j]
|
||||
if (!data) continue
|
||||
const id = path.basename(sessionFiles[i + j], ".json")
|
||||
const projectID = sessionProjects[i + j]
|
||||
if (!projectIds.has(projectID)) {
|
||||
orphans.sessions++
|
||||
continue
|
||||
}
|
||||
sessionIds.add(id)
|
||||
sessionValues.push({
|
||||
id,
|
||||
project_id: projectID,
|
||||
parent_id: data.parentID ?? null,
|
||||
slug: data.slug ?? "",
|
||||
directory: data.directory ?? "",
|
||||
title: data.title ?? "",
|
||||
version: data.version ?? "",
|
||||
share_url: data.share?.url ?? null,
|
||||
summary_additions: data.summary?.additions ?? null,
|
||||
summary_deletions: data.summary?.deletions ?? null,
|
||||
summary_files: data.summary?.files ?? null,
|
||||
summary_diffs: data.summary?.diffs ?? null,
|
||||
revert: data.revert ?? null,
|
||||
permission: data.permission ?? null,
|
||||
time_created: data.time?.created ?? now,
|
||||
time_updated: data.time?.updated ?? now,
|
||||
time_compacting: data.time?.compacting ?? null,
|
||||
time_archived: data.time?.archived ?? null,
|
||||
})
|
||||
}
|
||||
stats.sessions += insert(sessionValues, SessionTable, "session")
|
||||
step("sessions", end - i)
|
||||
}
|
||||
log.info("migrated sessions", { count: stats.sessions })
|
||||
if (orphans.sessions > 0) {
|
||||
log.warn("skipped orphaned sessions", { count: orphans.sessions })
|
||||
}
|
||||
|
||||
// Migrate messages using pre-scanned file map
|
||||
const allMessageFiles = [] as string[]
|
||||
const allMessageSessions = [] as string[]
|
||||
const messageSessions = new Map<string, string>()
|
||||
for (const file of messageFiles) {
|
||||
const sessionID = path.basename(path.dirname(file))
|
||||
if (!sessionIds.has(sessionID)) continue
|
||||
allMessageFiles.push(file)
|
||||
allMessageSessions.push(sessionID)
|
||||
}
|
||||
|
||||
for (let i = 0; i < allMessageFiles.length; i += batchSize) {
|
||||
const end = Math.min(i + batchSize, allMessageFiles.length)
|
||||
const batch = await read(allMessageFiles, i, end)
|
||||
// oxlint-disable-next-line unicorn/no-new-array -- pre-allocated for index-based batch fill
|
||||
const values = new Array(batch.length)
|
||||
let count = 0
|
||||
for (let j = 0; j < batch.length; j++) {
|
||||
const data = batch[j]
|
||||
if (!data) continue
|
||||
const file = allMessageFiles[i + j]
|
||||
const id = path.basename(file, ".json")
|
||||
const sessionID = allMessageSessions[i + j]
|
||||
messageSessions.set(id, sessionID)
|
||||
const rest = data
|
||||
delete rest.id
|
||||
delete rest.sessionID
|
||||
values[count++] = {
|
||||
id,
|
||||
session_id: sessionID,
|
||||
time_created: data.time?.created ?? now,
|
||||
time_updated: data.time?.updated ?? now,
|
||||
data: rest,
|
||||
}
|
||||
}
|
||||
values.length = count
|
||||
stats.messages += insert(values, MessageTable, "message")
|
||||
step("messages", end - i)
|
||||
}
|
||||
log.info("migrated messages", { count: stats.messages })
|
||||
|
||||
// Migrate parts using pre-scanned file map
|
||||
for (let i = 0; i < partFiles.length; i += batchSize) {
|
||||
const end = Math.min(i + batchSize, partFiles.length)
|
||||
const batch = await read(partFiles, i, end)
|
||||
// oxlint-disable-next-line unicorn/no-new-array -- pre-allocated for index-based batch fill
|
||||
const values = new Array(batch.length)
|
||||
let count = 0
|
||||
for (let j = 0; j < batch.length; j++) {
|
||||
const data = batch[j]
|
||||
if (!data) continue
|
||||
const file = partFiles[i + j]
|
||||
const id = path.basename(file, ".json")
|
||||
const messageID = path.basename(path.dirname(file))
|
||||
const sessionID = messageSessions.get(messageID)
|
||||
if (!sessionID) {
|
||||
errs.push(`part missing message session: ${file}`)
|
||||
continue
|
||||
}
|
||||
if (!sessionIds.has(sessionID)) continue
|
||||
const rest = data
|
||||
delete rest.id
|
||||
delete rest.messageID
|
||||
delete rest.sessionID
|
||||
values[count++] = {
|
||||
id,
|
||||
message_id: messageID,
|
||||
session_id: sessionID,
|
||||
time_created: data.time?.created ?? now,
|
||||
time_updated: data.time?.updated ?? now,
|
||||
data: rest,
|
||||
}
|
||||
}
|
||||
values.length = count
|
||||
stats.parts += insert(values, PartTable, "part")
|
||||
step("parts", end - i)
|
||||
}
|
||||
log.info("migrated parts", { count: stats.parts })
|
||||
|
||||
// Migrate todos
|
||||
const todoSessions = todoFiles.map((file) => path.basename(file, ".json"))
|
||||
for (let i = 0; i < todoFiles.length; i += batchSize) {
|
||||
const end = Math.min(i + batchSize, todoFiles.length)
|
||||
const batch = await read(todoFiles, i, end)
|
||||
const values: unknown[] = []
|
||||
for (let j = 0; j < batch.length; j++) {
|
||||
const data = batch[j]
|
||||
if (!data) continue
|
||||
const sessionID = todoSessions[i + j]
|
||||
if (!sessionIds.has(sessionID)) {
|
||||
orphans.todos++
|
||||
continue
|
||||
}
|
||||
if (!Array.isArray(data)) {
|
||||
errs.push(`todo not an array: ${todoFiles[i + j]}`)
|
||||
continue
|
||||
}
|
||||
for (let position = 0; position < data.length; position++) {
|
||||
const todo = data[position]
|
||||
if (!todo?.content || !todo?.status || !todo?.priority) continue
|
||||
values.push({
|
||||
session_id: sessionID,
|
||||
content: todo.content,
|
||||
status: todo.status,
|
||||
priority: todo.priority,
|
||||
position,
|
||||
time_created: now,
|
||||
time_updated: now,
|
||||
})
|
||||
}
|
||||
}
|
||||
stats.todos += insert(values, TodoTable, "todo")
|
||||
step("todos", end - i)
|
||||
}
|
||||
log.info("migrated todos", { count: stats.todos })
|
||||
if (orphans.todos > 0) {
|
||||
log.warn("skipped orphaned todos", { count: orphans.todos })
|
||||
}
|
||||
|
||||
// Migrate permissions
|
||||
const permProjects = permFiles.map((file) => path.basename(file, ".json"))
|
||||
const permValues: unknown[] = []
|
||||
for (let i = 0; i < permFiles.length; i += batchSize) {
|
||||
const end = Math.min(i + batchSize, permFiles.length)
|
||||
const batch = await read(permFiles, i, end)
|
||||
permValues.length = 0
|
||||
for (let j = 0; j < batch.length; j++) {
|
||||
const data = batch[j]
|
||||
if (!data) continue
|
||||
const projectID = permProjects[i + j]
|
||||
if (!projectIds.has(projectID)) {
|
||||
orphans.permissions++
|
||||
continue
|
||||
}
|
||||
permValues.push({ project_id: projectID, data })
|
||||
}
|
||||
stats.permissions += insert(permValues, PermissionTable, "permission")
|
||||
step("permissions", end - i)
|
||||
}
|
||||
log.info("migrated permissions", { count: stats.permissions })
|
||||
if (orphans.permissions > 0) {
|
||||
log.warn("skipped orphaned permissions", { count: orphans.permissions })
|
||||
}
|
||||
|
||||
// Migrate session shares
|
||||
const shareSessions = shareFiles.map((file) => path.basename(file, ".json"))
|
||||
const shareValues: unknown[] = []
|
||||
for (let i = 0; i < shareFiles.length; i += batchSize) {
|
||||
const end = Math.min(i + batchSize, shareFiles.length)
|
||||
const batch = await read(shareFiles, i, end)
|
||||
shareValues.length = 0
|
||||
for (let j = 0; j < batch.length; j++) {
|
||||
const data = batch[j]
|
||||
if (!data) continue
|
||||
const sessionID = shareSessions[i + j]
|
||||
if (!sessionIds.has(sessionID)) {
|
||||
orphans.shares++
|
||||
continue
|
||||
}
|
||||
if (!data?.id || !data?.secret || !data?.url) {
|
||||
errs.push(`session_share missing id/secret/url: ${shareFiles[i + j]}`)
|
||||
continue
|
||||
}
|
||||
shareValues.push({ session_id: sessionID, id: data.id, secret: data.secret, url: data.url })
|
||||
}
|
||||
stats.shares += insert(shareValues, SessionShareTable, "session_share")
|
||||
step("shares", end - i)
|
||||
}
|
||||
log.info("migrated session shares", { count: stats.shares })
|
||||
if (orphans.shares > 0) {
|
||||
log.warn("skipped orphaned session shares", { count: orphans.shares })
|
||||
}
|
||||
|
||||
db.run("COMMIT")
|
||||
|
||||
log.info("json migration complete", {
|
||||
projects: stats.projects,
|
||||
sessions: stats.sessions,
|
||||
messages: stats.messages,
|
||||
parts: stats.parts,
|
||||
todos: stats.todos,
|
||||
permissions: stats.permissions,
|
||||
shares: stats.shares,
|
||||
errorCount: stats.errors.length,
|
||||
duration: Math.round(performance.now() - start),
|
||||
})
|
||||
|
||||
if (stats.errors.length > 0) {
|
||||
log.warn("migration errors", { errors: stats.errors.slice(0, 20) })
|
||||
}
|
||||
|
||||
progress?.({ current: total, total, label: "complete" })
|
||||
|
||||
return stats
|
||||
}
|
||||
10
qimingcode/packages/opencode/src/storage/schema.sql.ts
Normal file
10
qimingcode/packages/opencode/src/storage/schema.sql.ts
Normal file
@@ -0,0 +1,10 @@
|
||||
import { integer } from "drizzle-orm/sqlite-core"
|
||||
|
||||
export const Timestamps = {
|
||||
time_created: integer()
|
||||
.notNull()
|
||||
.$default(() => Date.now()),
|
||||
time_updated: integer()
|
||||
.notNull()
|
||||
.$onUpdate(() => Date.now()),
|
||||
}
|
||||
5
qimingcode/packages/opencode/src/storage/schema.ts
Normal file
5
qimingcode/packages/opencode/src/storage/schema.ts
Normal file
@@ -0,0 +1,5 @@
|
||||
export { AccountTable, AccountStateTable, ControlAccountTable } from "../account/account.sql"
|
||||
export { ProjectTable } from "../project/project.sql"
|
||||
export { SessionTable, MessageTable, PartTable, TodoTable, PermissionTable } from "../session/session.sql"
|
||||
export { SessionShareTable } from "../share/share.sql"
|
||||
export { WorkspaceTable } from "../control-plane/workspace.sql"
|
||||
331
qimingcode/packages/opencode/src/storage/storage.ts
Normal file
331
qimingcode/packages/opencode/src/storage/storage.ts
Normal file
@@ -0,0 +1,331 @@
|
||||
import { Log } from "../util"
|
||||
import path from "path"
|
||||
import { Global } from "@opencode-ai/core/global"
|
||||
import { NamedError } from "@opencode-ai/core/util/error"
|
||||
import z from "zod"
|
||||
import { AppFileSystem } from "@opencode-ai/core/filesystem"
|
||||
import { Effect, Exit, Layer, Option, RcMap, Schema, Context, TxReentrantLock } from "effect"
|
||||
import { Git } from "@/git"
|
||||
|
||||
const log = Log.create({ service: "storage" })
|
||||
|
||||
type Migration = (
|
||||
dir: string,
|
||||
fs: AppFileSystem.Interface,
|
||||
git: Git.Interface,
|
||||
) => Effect.Effect<void, AppFileSystem.Error>
|
||||
|
||||
export const NotFoundError = NamedError.create(
|
||||
"NotFoundError",
|
||||
z.object({
|
||||
message: z.string(),
|
||||
}),
|
||||
)
|
||||
|
||||
export type Error = AppFileSystem.Error | InstanceType<typeof NotFoundError>
|
||||
|
||||
const RootFile = Schema.Struct({
|
||||
path: Schema.optional(
|
||||
Schema.Struct({
|
||||
root: Schema.optional(Schema.String),
|
||||
}),
|
||||
),
|
||||
})
|
||||
|
||||
const SessionFile = Schema.Struct({
|
||||
id: Schema.String,
|
||||
})
|
||||
|
||||
const MessageFile = Schema.Struct({
|
||||
id: Schema.String,
|
||||
})
|
||||
|
||||
const DiffFile = Schema.Struct({
|
||||
additions: Schema.Number,
|
||||
deletions: Schema.Number,
|
||||
})
|
||||
|
||||
const SummaryFile = Schema.Struct({
|
||||
id: Schema.String,
|
||||
projectID: Schema.String,
|
||||
summary: Schema.Struct({ diffs: Schema.Array(DiffFile) }),
|
||||
})
|
||||
|
||||
const decodeRoot = Schema.decodeUnknownOption(RootFile)
|
||||
const decodeSession = Schema.decodeUnknownOption(SessionFile)
|
||||
const decodeMessage = Schema.decodeUnknownOption(MessageFile)
|
||||
const decodeSummary = Schema.decodeUnknownOption(SummaryFile)
|
||||
|
||||
export interface Interface {
|
||||
readonly remove: (key: string[]) => Effect.Effect<void, AppFileSystem.Error>
|
||||
readonly read: <T>(key: string[]) => Effect.Effect<T, Error>
|
||||
readonly update: <T>(key: string[], fn: (draft: T) => void) => Effect.Effect<T, Error>
|
||||
readonly write: <T>(key: string[], content: T) => Effect.Effect<void, AppFileSystem.Error>
|
||||
readonly list: (prefix: string[]) => Effect.Effect<string[][], AppFileSystem.Error>
|
||||
}
|
||||
|
||||
export class Service extends Context.Service<Service, Interface>()("@opencode/Storage") {}
|
||||
|
||||
function file(dir: string, key: string[]) {
|
||||
return path.join(dir, ...key) + ".json"
|
||||
}
|
||||
|
||||
function missing(err: unknown) {
|
||||
if (!err || typeof err !== "object") return false
|
||||
if ("code" in err && err.code === "ENOENT") return true
|
||||
if ("reason" in err && err.reason && typeof err.reason === "object" && "_tag" in err.reason) {
|
||||
return err.reason._tag === "NotFound"
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
function parseMigration(text: string) {
|
||||
const value = Number.parseInt(text, 10)
|
||||
return Number.isNaN(value) ? 0 : value
|
||||
}
|
||||
|
||||
const MIGRATIONS: Migration[] = [
|
||||
Effect.fn("Storage.migration.1")(function* (dir: string, fs: AppFileSystem.Interface, git: Git.Interface) {
|
||||
const project = path.resolve(dir, "../project")
|
||||
if (!(yield* fs.isDir(project))) return
|
||||
const projectDirs = yield* fs.glob("*", {
|
||||
cwd: project,
|
||||
include: "all",
|
||||
})
|
||||
for (const projectDir of projectDirs) {
|
||||
const full = path.join(project, projectDir)
|
||||
if (!(yield* fs.isDir(full))) continue
|
||||
log.info(`migrating project ${projectDir}`)
|
||||
let projectID = projectDir
|
||||
let worktree = "/"
|
||||
|
||||
if (projectID !== "global") {
|
||||
for (const msgFile of yield* fs.glob("storage/session/message/*/*.json", {
|
||||
cwd: full,
|
||||
absolute: true,
|
||||
})) {
|
||||
const json = decodeRoot(yield* fs.readJson(msgFile), { onExcessProperty: "preserve" })
|
||||
const root = Option.isSome(json) ? json.value.path?.root : undefined
|
||||
if (!root) continue
|
||||
worktree = root
|
||||
break
|
||||
}
|
||||
if (!worktree) continue
|
||||
if (!(yield* fs.isDir(worktree))) continue
|
||||
const result = yield* git.run(["rev-list", "--max-parents=0", "--all"], {
|
||||
cwd: worktree,
|
||||
})
|
||||
const [id] = result
|
||||
.text()
|
||||
.split("\n")
|
||||
.filter(Boolean)
|
||||
.map((x) => x.trim())
|
||||
.toSorted()
|
||||
if (!id) continue
|
||||
projectID = id
|
||||
|
||||
yield* fs.writeWithDirs(
|
||||
path.join(dir, "project", projectID + ".json"),
|
||||
JSON.stringify(
|
||||
{
|
||||
id,
|
||||
vcs: "git",
|
||||
worktree,
|
||||
time: {
|
||||
created: Date.now(),
|
||||
initialized: Date.now(),
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
)
|
||||
|
||||
log.info(`migrating sessions for project ${projectID}`)
|
||||
for (const sessionFile of yield* fs.glob("storage/session/info/*.json", {
|
||||
cwd: full,
|
||||
absolute: true,
|
||||
})) {
|
||||
const dest = path.join(dir, "session", projectID, path.basename(sessionFile))
|
||||
log.info("copying", { sessionFile, dest })
|
||||
const session = yield* fs.readJson(sessionFile)
|
||||
const info = decodeSession(session, { onExcessProperty: "preserve" })
|
||||
yield* fs.writeWithDirs(dest, JSON.stringify(session, null, 2))
|
||||
if (Option.isNone(info)) continue
|
||||
log.info(`migrating messages for session ${info.value.id}`)
|
||||
for (const msgFile of yield* fs.glob(`storage/session/message/${info.value.id}/*.json`, {
|
||||
cwd: full,
|
||||
absolute: true,
|
||||
})) {
|
||||
const next = path.join(dir, "message", info.value.id, path.basename(msgFile))
|
||||
log.info("copying", {
|
||||
msgFile,
|
||||
dest: next,
|
||||
})
|
||||
const message = yield* fs.readJson(msgFile)
|
||||
const item = decodeMessage(message, { onExcessProperty: "preserve" })
|
||||
yield* fs.writeWithDirs(next, JSON.stringify(message, null, 2))
|
||||
if (Option.isNone(item)) continue
|
||||
|
||||
log.info(`migrating parts for message ${item.value.id}`)
|
||||
for (const partFile of yield* fs.glob(`storage/session/part/${info.value.id}/${item.value.id}/*.json`, {
|
||||
cwd: full,
|
||||
absolute: true,
|
||||
})) {
|
||||
const out = path.join(dir, "part", item.value.id, path.basename(partFile))
|
||||
const part = yield* fs.readJson(partFile)
|
||||
log.info("copying", {
|
||||
partFile,
|
||||
dest: out,
|
||||
})
|
||||
yield* fs.writeWithDirs(out, JSON.stringify(part, null, 2))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}),
|
||||
Effect.fn("Storage.migration.2")(function* (dir: string, fs: AppFileSystem.Interface) {
|
||||
for (const item of yield* fs.glob("session/*/*.json", {
|
||||
cwd: dir,
|
||||
absolute: true,
|
||||
})) {
|
||||
const raw = yield* fs.readJson(item)
|
||||
const session = decodeSummary(raw, { onExcessProperty: "preserve" })
|
||||
if (Option.isNone(session)) continue
|
||||
const diffs = session.value.summary.diffs
|
||||
yield* fs.writeWithDirs(
|
||||
path.join(dir, "session_diff", session.value.id + ".json"),
|
||||
JSON.stringify(diffs, null, 2),
|
||||
)
|
||||
yield* fs.writeWithDirs(
|
||||
path.join(dir, "session", session.value.projectID, session.value.id + ".json"),
|
||||
JSON.stringify(
|
||||
{
|
||||
...(raw as Record<string, unknown>),
|
||||
summary: {
|
||||
additions: diffs.reduce((sum, x) => sum + x.additions, 0),
|
||||
deletions: diffs.reduce((sum, x) => sum + x.deletions, 0),
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
)
|
||||
}
|
||||
}),
|
||||
]
|
||||
|
||||
export const layer = Layer.effect(
|
||||
Service,
|
||||
Effect.gen(function* () {
|
||||
const fs = yield* AppFileSystem.Service
|
||||
const git = yield* Git.Service
|
||||
const locks = yield* RcMap.make({
|
||||
lookup: () => TxReentrantLock.make(),
|
||||
idleTimeToLive: 0,
|
||||
})
|
||||
const state = yield* Effect.cached(
|
||||
Effect.gen(function* () {
|
||||
const dir = path.join(Global.Path.data, "storage")
|
||||
const marker = path.join(dir, "migration")
|
||||
const migration = yield* fs.readFileString(marker).pipe(
|
||||
Effect.map(parseMigration),
|
||||
Effect.catchIf(missing, () => Effect.succeed(0)),
|
||||
Effect.orElseSucceed(() => 0),
|
||||
)
|
||||
for (let i = migration; i < MIGRATIONS.length; i++) {
|
||||
log.info("running migration", { index: i })
|
||||
const step = MIGRATIONS[i]!
|
||||
const exit = yield* Effect.exit(step(dir, fs, git))
|
||||
if (Exit.isFailure(exit)) {
|
||||
log.error("failed to run migration", { index: i, cause: exit.cause })
|
||||
break
|
||||
}
|
||||
yield* fs.writeWithDirs(marker, String(i + 1))
|
||||
}
|
||||
return { dir }
|
||||
}),
|
||||
)
|
||||
|
||||
const fail = (target: string): Effect.Effect<never, InstanceType<typeof NotFoundError>> =>
|
||||
Effect.fail(new NotFoundError({ message: `Resource not found: ${target}` }))
|
||||
|
||||
const wrap = <A>(target: string, body: Effect.Effect<A, AppFileSystem.Error>) =>
|
||||
body.pipe(Effect.catchIf(missing, () => fail(target)))
|
||||
|
||||
const writeJson = Effect.fnUntraced(function* (target: string, content: unknown) {
|
||||
yield* fs.writeWithDirs(target, JSON.stringify(content, null, 2))
|
||||
})
|
||||
|
||||
const withResolved = <A, E>(
|
||||
key: string[],
|
||||
fn: (target: string, rw: TxReentrantLock.TxReentrantLock) => Effect.Effect<A, E>,
|
||||
): Effect.Effect<A, E | AppFileSystem.Error> =>
|
||||
Effect.scoped(
|
||||
Effect.gen(function* () {
|
||||
const target = file((yield* state).dir, key)
|
||||
return yield* fn(target, yield* RcMap.get(locks, target))
|
||||
}),
|
||||
)
|
||||
|
||||
const remove: Interface["remove"] = Effect.fn("Storage.remove")(function* (key: string[]) {
|
||||
yield* withResolved(key, (target, rw) =>
|
||||
TxReentrantLock.withWriteLock(rw, fs.remove(target).pipe(Effect.catchIf(missing, () => Effect.void))),
|
||||
)
|
||||
})
|
||||
|
||||
const read: Interface["read"] = <T>(key: string[]) =>
|
||||
Effect.gen(function* () {
|
||||
const value = yield* withResolved(key, (target, rw) =>
|
||||
TxReentrantLock.withReadLock(rw, wrap(target, fs.readJson(target))),
|
||||
)
|
||||
return value as T
|
||||
})
|
||||
|
||||
const update: Interface["update"] = <T>(key: string[], fn: (draft: T) => void) =>
|
||||
Effect.gen(function* () {
|
||||
const value = yield* withResolved(key, (target, rw) =>
|
||||
TxReentrantLock.withWriteLock(
|
||||
rw,
|
||||
Effect.gen(function* () {
|
||||
const content = yield* wrap(target, fs.readJson(target))
|
||||
fn(content as T)
|
||||
yield* writeJson(target, content)
|
||||
return content
|
||||
}),
|
||||
),
|
||||
)
|
||||
return value as T
|
||||
})
|
||||
|
||||
const write: Interface["write"] = (key: string[], content: unknown) =>
|
||||
Effect.gen(function* () {
|
||||
yield* withResolved(key, (target, rw) => TxReentrantLock.withWriteLock(rw, writeJson(target, content)))
|
||||
})
|
||||
|
||||
const list: Interface["list"] = Effect.fn("Storage.list")(function* (prefix: string[]) {
|
||||
const dir = (yield* state).dir
|
||||
const cwd = path.join(dir, ...prefix)
|
||||
const result = yield* fs
|
||||
.glob("**/*", {
|
||||
cwd,
|
||||
include: "file",
|
||||
})
|
||||
.pipe(Effect.catch(() => Effect.succeed<string[]>([])))
|
||||
return result
|
||||
.map((x) => [...prefix, ...x.slice(0, -5).split(path.sep)])
|
||||
.toSorted((a, b) => a.join("/").localeCompare(b.join("/")))
|
||||
})
|
||||
|
||||
return Service.of({
|
||||
remove,
|
||||
read,
|
||||
update,
|
||||
write,
|
||||
list,
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
export const defaultLayer = layer.pipe(Layer.provide(AppFileSystem.defaultLayer), Layer.provide(Git.defaultLayer))
|
||||
Reference in New Issue
Block a user