feat(client): track digital sync retry history
This commit is contained in:
@@ -1698,6 +1698,59 @@
|
||||
font-size: 13px;
|
||||
line-height: 1.55;
|
||||
}
|
||||
.de-sync-attempt-history {
|
||||
display: grid;
|
||||
gap: 8px;
|
||||
padding: 14px;
|
||||
border: 1px solid rgba(126,163,194,0.2);
|
||||
border-radius: 14px;
|
||||
background: rgba(247,251,255,0.82);
|
||||
}
|
||||
.de-sync-attempt-history > span {
|
||||
color: #6f8196;
|
||||
font-size: 12px;
|
||||
font-weight: 800;
|
||||
}
|
||||
.de-sync-attempt-list {
|
||||
display: grid;
|
||||
gap: 8px;
|
||||
}
|
||||
.de-sync-attempt-row {
|
||||
display: grid;
|
||||
gap: 5px;
|
||||
padding: 9px 10px;
|
||||
border-radius: 12px;
|
||||
background: rgba(255,255,255,0.72);
|
||||
}
|
||||
.de-sync-attempt-row > div {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
flex-wrap: wrap;
|
||||
gap: 8px;
|
||||
}
|
||||
.de-sync-attempt-row strong {
|
||||
color: #153754;
|
||||
font-size: 12px;
|
||||
}
|
||||
.de-sync-attempt-row em {
|
||||
font-size: 12px;
|
||||
font-style: normal;
|
||||
font-weight: 800;
|
||||
}
|
||||
.de-sync-attempt-row span,
|
||||
.de-sync-attempt-row small {
|
||||
color: #667b90;
|
||||
font-size: 12px;
|
||||
}
|
||||
.de-sync-attempt-row small {
|
||||
line-height: 1.45;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
.de-sync-attempt-row--success em { color: #23805d; }
|
||||
.de-sync-attempt-row--danger em { color: #bb3b3b; }
|
||||
.de-sync-attempt-row--info em { color: #2e6fa5; }
|
||||
.de-event-modal-actions {
|
||||
display: flex;
|
||||
justify-content: flex-end;
|
||||
|
||||
@@ -64,6 +64,15 @@ interface QimingclawSyncStatus {
|
||||
recentFailures?: QimingclawSyncFailure[];
|
||||
}
|
||||
|
||||
interface QimingclawSyncAttempt {
|
||||
attemptNo: number;
|
||||
status: string;
|
||||
error: string | null;
|
||||
endpoint?: string | null;
|
||||
startedAt: string;
|
||||
finishedAt: string;
|
||||
}
|
||||
|
||||
interface QimingclawSyncFailure {
|
||||
id: string;
|
||||
entityType: string;
|
||||
@@ -74,6 +83,7 @@ interface QimingclawSyncFailure {
|
||||
nextRetryAt?: string | null;
|
||||
dueForRetry?: boolean;
|
||||
managementRecordUrl?: string | null;
|
||||
attemptHistory?: QimingclawSyncAttempt[];
|
||||
error: string | null;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
|
||||
@@ -64,6 +64,15 @@ interface ManagementSyncStatus {
|
||||
recentFailures?: ManagementSyncFailure[];
|
||||
}
|
||||
|
||||
interface ManagementSyncAttempt {
|
||||
attemptNo: number;
|
||||
status: string;
|
||||
error: string | null;
|
||||
endpoint?: string | null;
|
||||
startedAt: string;
|
||||
finishedAt: string;
|
||||
}
|
||||
|
||||
interface ManagementSyncFailure {
|
||||
id: string;
|
||||
entityType: string;
|
||||
@@ -74,6 +83,7 @@ interface ManagementSyncFailure {
|
||||
nextRetryAt?: string | null;
|
||||
dueForRetry?: boolean;
|
||||
managementRecordUrl?: string | null;
|
||||
attemptHistory?: ManagementSyncAttempt[];
|
||||
error: string | null;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
@@ -495,6 +505,18 @@ function syncFailureRetryTime(failure: ManagementSyncFailure): string {
|
||||
return formatBeijingDateTime(failure.nextRetryAt) || '待计算';
|
||||
}
|
||||
|
||||
function syncAttemptStatusLabel(status: string): string {
|
||||
if (status === 'synced') return '已同步';
|
||||
if (status === 'failed') return '失败';
|
||||
return status || '未知';
|
||||
}
|
||||
|
||||
function syncAttemptTone(status: string): 'success' | 'danger' | 'info' {
|
||||
if (status === 'synced') return 'success';
|
||||
if (status === 'failed') return 'danger';
|
||||
return 'info';
|
||||
}
|
||||
|
||||
async function readManagementSyncStatus(): Promise<ManagementSyncStatus | null> {
|
||||
const bridge = window.QimingClawBridge?.digital;
|
||||
if (!bridge?.getSyncStatus) return null;
|
||||
@@ -1837,6 +1859,31 @@ export default function CommandDeck({ onNavigate }: { onNavigate: (target: Digit
|
||||
<span>失败原因</span>
|
||||
<p>{selectedSyncFailure.error || '暂无错误详情。'}</p>
|
||||
</div>
|
||||
{(selectedSyncFailure.attemptHistory?.length ?? 0) > 0 && (
|
||||
<div className="de-sync-attempt-history">
|
||||
<span>重试历史</span>
|
||||
<div className="de-sync-attempt-list">
|
||||
{selectedSyncFailure.attemptHistory?.map((attempt) => (
|
||||
<div
|
||||
key={`${attempt.attemptNo}-${attempt.finishedAt}`}
|
||||
className={`de-sync-attempt-row de-sync-attempt-row--${syncAttemptTone(
|
||||
attempt.status,
|
||||
)}`}
|
||||
>
|
||||
<div>
|
||||
<strong>第 {attempt.attemptNo} 次</strong>
|
||||
<em>{syncAttemptStatusLabel(attempt.status)}</em>
|
||||
<span>
|
||||
{formatBeijingDateTime(attempt.finishedAt) ||
|
||||
attempt.finishedAt}
|
||||
</span>
|
||||
</div>
|
||||
{attempt.error && <small>{attempt.error}</small>}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
<div className="de-event-modal-actions">
|
||||
<button
|
||||
type="button"
|
||||
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -13,8 +13,8 @@
|
||||
console.warn("[qimingclaw] digital employee auth bootstrap skipped", error);
|
||||
}
|
||||
</script>
|
||||
<script type="module" crossorigin src="./assets/index-BmpavZF5.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="./assets/index-DDOlQz2O.css">
|
||||
<script type="module" crossorigin src="./assets/index-CeOrqzZi.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="./assets/index-ReBtI0Fy.css">
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
|
||||
@@ -136,11 +136,26 @@ export function initDatabase(): void {
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS digital_sync_attempts (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
outbox_id TEXT NOT NULL,
|
||||
entity_type TEXT NOT NULL,
|
||||
entity_id TEXT NOT NULL,
|
||||
operation TEXT NOT NULL,
|
||||
attempt_no INTEGER NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
error TEXT,
|
||||
endpoint TEXT,
|
||||
started_at TEXT NOT NULL,
|
||||
finished_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_digital_plans_sync ON digital_plans(sync_status);
|
||||
CREATE INDEX IF NOT EXISTS idx_digital_tasks_plan ON digital_tasks(plan_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_digital_runs_task ON digital_runs(task_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_digital_events_plan ON digital_events(plan_id, occurred_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_digital_sync_outbox_status ON digital_sync_outbox(status, created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_digital_sync_attempts_outbox ON digital_sync_attempts(outbox_id, finished_at);
|
||||
`);
|
||||
log.info('Database tables created');
|
||||
} catch (error) {
|
||||
|
||||
@@ -11,6 +11,7 @@ const DEFAULT_BATCH_SIZE = 50;
|
||||
const REQUEST_TIMEOUT_MS = 15_000;
|
||||
const MAX_RETRY_DELAY_MS = 30 * 60_000;
|
||||
const FAILED_RETRY_CANDIDATE_MULTIPLIER = 4;
|
||||
const ATTEMPT_HISTORY_RETENTION_MS = 30 * 24 * 60 * 60_000;
|
||||
|
||||
let syncTimer: ReturnType<typeof setInterval> | null = null;
|
||||
let syncing = false;
|
||||
@@ -35,6 +36,15 @@ interface OutboxRow {
|
||||
last_attempt_at?: string | null;
|
||||
}
|
||||
|
||||
export interface DigitalEmployeeSyncAttempt {
|
||||
attemptNo: number;
|
||||
status: string;
|
||||
error: string | null;
|
||||
endpoint: string | null;
|
||||
startedAt: string;
|
||||
finishedAt: string;
|
||||
}
|
||||
|
||||
export interface DigitalEmployeeSyncFailure {
|
||||
id: string;
|
||||
entityType: string;
|
||||
@@ -45,6 +55,7 @@ export interface DigitalEmployeeSyncFailure {
|
||||
nextRetryAt: string | null;
|
||||
dueForRetry: boolean;
|
||||
managementRecordUrl: string | null;
|
||||
attemptHistory: DigitalEmployeeSyncAttempt[];
|
||||
error: string | null;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
@@ -134,17 +145,25 @@ export async function flushDigitalEmployeeSyncOutbox(
|
||||
}
|
||||
|
||||
syncing = true;
|
||||
const now = new Date().toISOString();
|
||||
markRowsSyncing(rows, now);
|
||||
const startedAt = new Date().toISOString();
|
||||
markRowsSyncing(rows, startedAt);
|
||||
|
||||
try {
|
||||
const response = await postOutbox(config, rows);
|
||||
const finishedAt = new Date().toISOString();
|
||||
const acked = response.data?.acked || response.acked || [];
|
||||
if (response.success === false) {
|
||||
throw new Error(response.message || "digital employee sync rejected");
|
||||
}
|
||||
const syncedRows = acked.length > 0
|
||||
? rows.filter((row) => acked.some((ack) => ackMatchesRow(ack, row)))
|
||||
: rows;
|
||||
const unackedRows = acked.length > 0
|
||||
? rows.filter((row) => !acked.some((ack) => ackMatchesRow(ack, row)))
|
||||
: [];
|
||||
recordSyncAttempts(syncedRows, config, "synced", null, startedAt, finishedAt);
|
||||
if (acked.length > 0) {
|
||||
markRowsSynced(acked, now);
|
||||
markRowsSynced(acked, finishedAt);
|
||||
} else {
|
||||
markRowsSynced(
|
||||
rows.map((row) => ({
|
||||
@@ -152,14 +171,30 @@ export async function flushDigitalEmployeeSyncOutbox(
|
||||
entity_type: row.entity_type,
|
||||
entity_id: row.entity_id,
|
||||
})),
|
||||
now,
|
||||
finishedAt,
|
||||
);
|
||||
}
|
||||
lastSyncAt = now;
|
||||
lastSyncError = null;
|
||||
if (unackedRows.length > 0) {
|
||||
const unackedError = "management sync response did not acknowledge item";
|
||||
recordSyncAttempts(
|
||||
unackedRows,
|
||||
config,
|
||||
"failed",
|
||||
unackedError,
|
||||
startedAt,
|
||||
finishedAt,
|
||||
);
|
||||
markRowsFailed(unackedRows, finishedAt, unackedError);
|
||||
lastSyncError = unackedError;
|
||||
} else {
|
||||
lastSyncError = null;
|
||||
}
|
||||
lastSyncAt = finishedAt;
|
||||
} catch (error) {
|
||||
const finishedAt = new Date().toISOString();
|
||||
lastSyncError = normalizeError(error);
|
||||
markRowsFailed(rows, now, lastSyncError);
|
||||
recordSyncAttempts(rows, config, "failed", lastSyncError, startedAt, finishedAt);
|
||||
markRowsFailed(rows, finishedAt, lastSyncError);
|
||||
log.warn("[DigitalEmployeeSync] Flush failed:", lastSyncError);
|
||||
} finally {
|
||||
syncing = false;
|
||||
@@ -335,6 +370,50 @@ function markRowsSynced(acks: SyncAck[], now: string): void {
|
||||
tx();
|
||||
}
|
||||
|
||||
function ackMatchesRow(ack: SyncAck, row: OutboxRow): boolean {
|
||||
const outboxId = ack.outbox_id || ack.id;
|
||||
if (outboxId) return outboxId === row.id;
|
||||
return ack.entity_type === row.entity_type && ack.entity_id === row.entity_id;
|
||||
}
|
||||
|
||||
function recordSyncAttempts(
|
||||
rows: OutboxRow[],
|
||||
config: DigitalEmployeeSyncConfig,
|
||||
status: "synced" | "failed",
|
||||
error: string | null,
|
||||
startedAt: string,
|
||||
finishedAt: string,
|
||||
): void {
|
||||
const db = getDb();
|
||||
if (!db) return;
|
||||
const tx = db.transaction(() => {
|
||||
const stmt = db.prepare(`
|
||||
INSERT INTO digital_sync_attempts (
|
||||
outbox_id, entity_type, entity_id, operation, attempt_no,
|
||||
status, error, endpoint, started_at, finished_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`);
|
||||
for (const row of rows) {
|
||||
stmt.run(
|
||||
row.id,
|
||||
row.entity_type,
|
||||
row.entity_id,
|
||||
row.operation,
|
||||
row.attempts + 1,
|
||||
status,
|
||||
error,
|
||||
config.endpoint,
|
||||
startedAt,
|
||||
finishedAt,
|
||||
);
|
||||
}
|
||||
db.prepare("DELETE FROM digital_sync_attempts WHERE finished_at < ?").run(
|
||||
new Date(Date.parse(finishedAt) - ATTEMPT_HISTORY_RETENTION_MS).toISOString(),
|
||||
);
|
||||
});
|
||||
tx();
|
||||
}
|
||||
|
||||
function updateEntitySyncStatus(ack: SyncAck, now: string): void {
|
||||
if (!ack.entity_type || !ack.entity_id) return;
|
||||
const table = entityTable(ack.entity_type);
|
||||
@@ -429,6 +508,7 @@ function readRecentFailures(
|
||||
nextRetryAt,
|
||||
dueForRetry: !nextRetryAt || Date.now() >= Date.parse(nextRetryAt),
|
||||
managementRecordUrl: buildManagementRecordUrl(config, row),
|
||||
attemptHistory: readAttemptHistory(row.id),
|
||||
error: row.error,
|
||||
createdAt: row.created_at,
|
||||
updatedAt: row.updated_at,
|
||||
@@ -436,6 +516,41 @@ function readRecentFailures(
|
||||
});
|
||||
}
|
||||
|
||||
function readAttemptHistory(
|
||||
outboxId: string,
|
||||
limit = 5,
|
||||
): DigitalEmployeeSyncAttempt[] {
|
||||
const db = getDb();
|
||||
if (!db) return [];
|
||||
const rows = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT attempt_no, status, error, endpoint, started_at, finished_at
|
||||
FROM digital_sync_attempts
|
||||
WHERE outbox_id = ?
|
||||
ORDER BY finished_at DESC, id DESC
|
||||
LIMIT ?
|
||||
`,
|
||||
)
|
||||
.all(outboxId, limit) as Array<{
|
||||
attempt_no: number;
|
||||
status: string;
|
||||
error: string | null;
|
||||
endpoint: string | null;
|
||||
started_at: string;
|
||||
finished_at: string;
|
||||
}>;
|
||||
|
||||
return rows.map((row) => ({
|
||||
attemptNo: row.attempt_no,
|
||||
status: row.status,
|
||||
error: row.error,
|
||||
endpoint: row.endpoint,
|
||||
startedAt: row.started_at,
|
||||
finishedAt: row.finished_at,
|
||||
}));
|
||||
}
|
||||
|
||||
function buildManagementRecordUrl(
|
||||
config: DigitalEmployeeSyncConfig,
|
||||
row: {
|
||||
|
||||
Reference in New Issue
Block a user