吸收数字员工PlanStep管理端强租约
This commit is contained in:
baiyanyun
@@ -2624,6 +2624,11 @@ function planStepRows(snapshot: QimingclawSnapshot | null): BusinessViewRow[] {
|
||||
lease_expires_at: stringValue(lease?.expires_at ?? lease?.expiresAt) || null,
|
||||
lease_released_at: stringValue(lease?.released_at ?? lease?.releasedAt) || null,
|
||||
lease_release_reason: stringValue(lease?.release_reason ?? lease?.releaseReason) || null,
|
||||
lease_source: stringValue(lease?.source) || null,
|
||||
remote_lease_id: stringValue(lease?.remote_lease_id ?? lease?.remoteLeaseId) || null,
|
||||
remote_lease_error: stringValue(lease?.remote_error ?? lease?.remoteError) || null,
|
||||
remote_release_error: stringValue(lease?.remote_release_error ?? lease?.remoteReleaseError) || null,
|
||||
remote_lease_rejected: stringValue(lease?.release_reason ?? lease?.releaseReason) === 'remote_lease_rejected',
|
||||
lease_active: Boolean(lease),
|
||||
sync_status: step.syncStatus,
|
||||
last_synced_at: step.lastSyncedAt ?? null,
|
||||
@@ -2664,6 +2669,11 @@ function planStepGraphRows(snapshot: QimingclawSnapshot | null): BusinessViewRow
|
||||
lease_expires_at: stringValue(row.lease_expires_at) || null,
|
||||
lease_released_at: stringValue(row.lease_released_at) || null,
|
||||
lease_release_reason: stringValue(row.lease_release_reason) || null,
|
||||
lease_source: stringValue(row.lease_source) || null,
|
||||
remote_lease_id: stringValue(row.remote_lease_id) || null,
|
||||
remote_lease_error: stringValue(row.remote_lease_error) || null,
|
||||
remote_release_error: stringValue(row.remote_release_error) || null,
|
||||
remote_lease_rejected: row.remote_lease_rejected === true,
|
||||
lease_active: row.lease_active === true,
|
||||
sync_status: stringValue(row.sync_status) || null,
|
||||
updated_at: stringValue(row.updated_at) || null,
|
||||
|
||||
@@ -239,6 +239,10 @@ export interface PlanStepLeaseSummary {
|
||||
trigger_source?: string | null;
|
||||
orchestration_id?: string | null;
|
||||
candidate_count?: number | null;
|
||||
source?: string | null;
|
||||
remote_lease_id?: string | null;
|
||||
remote_error?: string | null;
|
||||
remote_release_error?: string | null;
|
||||
}
|
||||
|
||||
export interface PlanStepBusinessRow {
|
||||
@@ -266,6 +270,11 @@ export interface PlanStepBusinessRow {
|
||||
lease_expires_at?: string | null;
|
||||
lease_released_at?: string | null;
|
||||
lease_release_reason?: string | null;
|
||||
lease_source?: string | null;
|
||||
remote_lease_id?: string | null;
|
||||
remote_lease_error?: string | null;
|
||||
remote_release_error?: string | null;
|
||||
remote_lease_rejected?: boolean;
|
||||
lease_active?: boolean;
|
||||
sync_status?: string | null;
|
||||
updated_at?: string | null;
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -16,7 +16,7 @@
|
||||
console.warn("[qimingclaw] digital employee auth bootstrap skipped", error);
|
||||
}
|
||||
</script>
|
||||
<script type="module" crossorigin src="./assets/index-BXlmpGIQ.js"></script>
|
||||
<script type="module" crossorigin src="./assets/index-LEtlcmt1.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="./assets/index-4TVkOCsf.css">
|
||||
</head>
|
||||
<body>
|
||||
|
||||
@@ -335,6 +335,7 @@ function checkDigitalPlanStepGraphDispatchPolicy() {
|
||||
[types, "PlanStepDispatchPolicyPullResult", "plan step dispatch policy pull result type"],
|
||||
[types, "last_pull_error", "plan step dispatch policy pull error field"],
|
||||
[types, "PlanStepLeaseSummary", "plan step lease summary type"],
|
||||
[types, "remote_lease_id", "plan step remote lease type field"],
|
||||
[types, "PlanStepDependencyDetail", "plan step blocked dependency detail type"],
|
||||
[types, "lease_active", "plan step lease active type field"],
|
||||
[types, "PlanStepBusinessRow", "plan step business row type"],
|
||||
@@ -358,6 +359,9 @@ function checkDigitalPlanStepGraphDispatchPolicy() {
|
||||
[stateService, "activeDigitalEmployeePlanStepLease", "plan step active lease helper"],
|
||||
[stateService, "acquireDigitalEmployeePlanStepLease", "plan step lease acquire helper"],
|
||||
[stateService, "releaseDigitalEmployeePlanStepLease", "plan step lease release helper"],
|
||||
[stateService, "recordDigitalEmployeePlanStepLeaseSkipped", "plan step remote lease rejected helper"],
|
||||
[stateService, "local_fallback", "plan step local fallback lease source"],
|
||||
[stateService, "remote_release_error", "plan step remote release error state"],
|
||||
[stateService, "plan_step_lease_acquired", "plan step lease acquired event"],
|
||||
[stateService, "plan_step_lease_released", "plan step lease released event"],
|
||||
[stateService, "plan_step_lease_skipped", "plan step lease skipped event"],
|
||||
@@ -365,6 +369,10 @@ function checkDigitalPlanStepGraphDispatchPolicy() {
|
||||
[schedulerService, "runDigitalEmployeePlanReadyStepsNow", "manual plan-scoped ready step dispatcher"],
|
||||
[schedulerService, "acquireDigitalEmployeePlanStepLease", "scheduler acquire plan step lease"],
|
||||
[schedulerService, "releaseDigitalEmployeePlanStepLease", "scheduler release plan step lease"],
|
||||
[schedulerService, "acquireDigitalEmployeePlanStepRemoteLease", "scheduler remote plan step lease acquire"],
|
||||
[schedulerService, "releaseDigitalEmployeePlanStepRemoteLease", "scheduler remote plan step lease release"],
|
||||
[schedulerService, "remote_lease_rejected", "scheduler remote lease rejected skip reason"],
|
||||
[schedulerService, "local_fallback", "scheduler remote lease fallback source"],
|
||||
[schedulerService, "triggerSource", "ready step dispatch trigger source"],
|
||||
[schedulerService, "readDigitalEmployeePlanStepDispatchPolicy", "scheduler plan step dispatch policy reader"],
|
||||
[schedulerService, "pullDigitalEmployeePlanStepDispatchPolicy", "scheduler remote plan step dispatch policy pull"],
|
||||
@@ -377,7 +385,13 @@ function checkDigitalPlanStepGraphDispatchPolicy() {
|
||||
[syncService, "pullDigitalEmployeePlanStepDispatchPolicy", "remote plan step dispatch policy pull service"],
|
||||
[syncService, "planStepDispatchPolicyEndpoint", "remote plan step dispatch policy endpoint config"],
|
||||
[syncService, "/api/digital-employee/policies/plan-step-dispatch", "remote plan step dispatch policy default path"],
|
||||
[syncService, "acquireDigitalEmployeePlanStepRemoteLease", "remote plan step dispatch lease acquire service"],
|
||||
[syncService, "releaseDigitalEmployeePlanStepRemoteLease", "remote plan step dispatch lease release service"],
|
||||
[syncService, "planStepDispatchLeaseEndpoint", "remote plan step dispatch lease endpoint config"],
|
||||
[syncService, "/api/digital-employee/leases/plan-step-dispatch", "remote plan step dispatch lease default path"],
|
||||
[syncService, "lease_active", "plan step sync lease active field"],
|
||||
[syncService, "remote_lease_error", "plan step sync remote lease error field"],
|
||||
[syncService, "remote_release_error", "plan step sync remote release error field"],
|
||||
[syncService, "planStepLeaseBusinessView", "plan step lease sync event view"],
|
||||
[syncService, "plan_step_lease_acquired", "plan step lease acquired sync event kind"],
|
||||
[syncService, "trigger_source", "plan step dispatch sync trigger source"],
|
||||
|
||||
@@ -10,6 +10,10 @@ const mockState = vi.hoisted(() => ({
|
||||
acquiredLeases: [] as Array<Record<string, unknown>>,
|
||||
releasedLeases: [] as Array<Record<string, unknown>>,
|
||||
policyPulls: [] as Array<Record<string, unknown>>,
|
||||
remoteLeaseAcquires: [] as Array<Record<string, unknown>>,
|
||||
remoteLeaseReleases: [] as Array<Record<string, unknown>>,
|
||||
remoteLeaseAcquireResult: null as Record<string, unknown> | null,
|
||||
remoteLeaseReleaseResult: null as Record<string, unknown> | null,
|
||||
settings: {
|
||||
enabled: true,
|
||||
catch_up_on_startup: true,
|
||||
@@ -69,13 +73,26 @@ vi.mock("./stateService", () => ({
|
||||
}
|
||||
return {
|
||||
ok: true,
|
||||
lease: { lease_id: `lease-${input.stepId}`, state: "active", owner_device_id: "device-001", acquired_at: input.occurredAt, expires_at: "2026-06-07T01:05:10.000Z" },
|
||||
lease: {
|
||||
lease_id: (input.remoteLease as Record<string, unknown> | undefined)?.lease_id ?? `lease-${input.stepId}`,
|
||||
state: "active",
|
||||
owner_device_id: "device-001",
|
||||
acquired_at: input.occurredAt,
|
||||
expires_at: "2026-06-07T01:05:10.000Z",
|
||||
source: input.source ?? "local",
|
||||
remote_lease_id: (input.remoteLease as Record<string, unknown> | undefined)?.remote_lease_id ?? null,
|
||||
remote_error: input.remoteError ?? null,
|
||||
},
|
||||
};
|
||||
},
|
||||
releaseDigitalEmployeePlanStepLease: (input: Record<string, unknown>) => {
|
||||
mockState.releasedLeases.push(input);
|
||||
return { ok: true, lease: { lease_id: input.leaseId, state: "released", owner_device_id: "device-001", acquired_at: "2026-06-07T01:00:10.000Z", expires_at: "2026-06-07T01:05:10.000Z", released_at: input.occurredAt, release_reason: input.reason } };
|
||||
},
|
||||
recordDigitalEmployeePlanStepLeaseSkipped: (input: Record<string, unknown>) => {
|
||||
mockState.acquiredLeases.push({ ...input, skipped: true });
|
||||
return { ok: false, reason: input.reason, lease: { lease_id: `remote-rejected-${input.stepId}`, state: "expired", owner_device_id: "device-remote", acquired_at: input.occurredAt, expires_at: input.occurredAt, source: "remote", remote_error: (input.remoteLease as Record<string, unknown> | undefined)?.remote_error } };
|
||||
},
|
||||
recordDigitalEmployeeGovernanceCommand: (input: Record<string, unknown>) => {
|
||||
mockState.governanceCommands.push(input);
|
||||
return {
|
||||
@@ -91,6 +108,25 @@ vi.mock("./syncService", () => ({
|
||||
mockState.policyPulls.push(input);
|
||||
return Promise.resolve({ ok: true, endpoint: "https://manage.example.com/api/digital-employee/policies/plan-step-dispatch" });
|
||||
},
|
||||
acquireDigitalEmployeePlanStepRemoteLease: (input: Record<string, unknown>) => {
|
||||
mockState.remoteLeaseAcquires.push(input);
|
||||
return Promise.resolve(mockState.remoteLeaseAcquireResult ?? {
|
||||
ok: true,
|
||||
endpoint: "https://manage.example.com/api/digital-employee/leases/plan-step-dispatch/acquire",
|
||||
lease: {
|
||||
lease_id: `remote-lease-${input.stepId}`,
|
||||
remote_lease_id: `remote-lease-${input.stepId}`,
|
||||
state: "active",
|
||||
owner_device_id: "device-001",
|
||||
acquired_at: input.occurredAt,
|
||||
expires_at: "2026-06-07T01:05:10.000Z",
|
||||
},
|
||||
});
|
||||
},
|
||||
releaseDigitalEmployeePlanStepRemoteLease: (input: Record<string, unknown>) => {
|
||||
mockState.remoteLeaseReleases.push(input);
|
||||
return Promise.resolve(mockState.remoteLeaseReleaseResult ?? { ok: true, endpoint: "https://manage.example.com/api/digital-employee/leases/plan-step-dispatch/release" });
|
||||
},
|
||||
}));
|
||||
|
||||
describe("digital employee scheduler service", () => {
|
||||
@@ -104,6 +140,10 @@ describe("digital employee scheduler service", () => {
|
||||
mockState.acquiredLeases = [];
|
||||
mockState.releasedLeases = [];
|
||||
mockState.policyPulls = [];
|
||||
mockState.remoteLeaseAcquires = [];
|
||||
mockState.remoteLeaseReleases = [];
|
||||
mockState.remoteLeaseAcquireResult = null;
|
||||
mockState.remoteLeaseReleaseResult = null;
|
||||
mockState.dispatchPolicy = {
|
||||
enabled: true,
|
||||
max_per_sweep: 3,
|
||||
@@ -189,8 +229,10 @@ describe("digital employee scheduler service", () => {
|
||||
);
|
||||
expect(mockState.dispatchEvents.map((event) => event.status)).toEqual(["started", "completed"]);
|
||||
expect(mockState.policyPulls).toEqual([{}]);
|
||||
expect(mockState.acquiredLeases).toEqual([expect.objectContaining({ stepId: "step-ready", triggerSource: "auto" })]);
|
||||
expect(mockState.releasedLeases).toEqual([expect.objectContaining({ stepId: "step-ready", leaseId: "lease-step-ready", reason: "completed" })]);
|
||||
expect(mockState.remoteLeaseAcquires).toEqual([expect.objectContaining({ stepId: "step-ready", triggerSource: "auto" })]);
|
||||
expect(mockState.acquiredLeases).toEqual([expect.objectContaining({ stepId: "step-ready", triggerSource: "auto", source: "remote" })]);
|
||||
expect(mockState.remoteLeaseReleases).toEqual([expect.objectContaining({ stepId: "step-ready", remoteLeaseId: "remote-lease-step-ready", reason: "completed" })]);
|
||||
expect(mockState.releasedLeases).toEqual([expect.objectContaining({ stepId: "step-ready", leaseId: "remote-lease-step-ready", reason: "completed" })]);
|
||||
expect(mockState.governanceCommands[0]).toMatchObject({
|
||||
commandKind: "start_run",
|
||||
planId: "plan-1",
|
||||
@@ -214,7 +256,8 @@ describe("digital employee scheduler service", () => {
|
||||
expect(result.readyStepDispatch).toMatchObject({ candidates: 1, dispatched: 0, failed: 1 });
|
||||
expect(result.readyStepDispatch.errors[0]).toMatchObject({ stepId: "step-ready", error: "ComputerServer 502: bad gateway" });
|
||||
expect(mockState.dispatchEvents.map((event) => event.status)).toEqual(["started", "failed"]);
|
||||
expect(mockState.releasedLeases).toEqual([expect.objectContaining({ stepId: "step-ready", leaseId: "lease-step-ready", reason: "failed" })]);
|
||||
expect(mockState.remoteLeaseReleases).toEqual([expect.objectContaining({ stepId: "step-ready", remoteLeaseId: "remote-lease-step-ready", reason: "failed" })]);
|
||||
expect(mockState.releasedLeases).toEqual([expect.objectContaining({ stepId: "step-ready", leaseId: "remote-lease-step-ready", reason: "failed" })]);
|
||||
expect(mockState.governanceCommands).toHaveLength(0);
|
||||
});
|
||||
|
||||
@@ -232,6 +275,46 @@ describe("digital employee scheduler service", () => {
|
||||
expect(mockState.dispatchEvents).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("skips ready plan steps when management rejects the remote lease", async () => {
|
||||
mockState.schedules = [];
|
||||
mockState.readyCandidates = [readyStepCandidate()];
|
||||
mockState.remoteLeaseAcquireResult = { ok: false, rejected: true, reason: "locked_by_other_device" };
|
||||
const { runDigitalEmployeeSchedulerCheck } = await import("./schedulerService");
|
||||
|
||||
const result = await runDigitalEmployeeSchedulerCheck(new Date("2026-06-07T01:00:10.000Z"));
|
||||
|
||||
expect(result.readyStepDispatch).toMatchObject({ candidates: 1, dispatched: 0, skipped: 1, failed: 0 });
|
||||
expect(fetch).not.toHaveBeenCalled();
|
||||
expect(mockState.acquiredLeases).toEqual([expect.objectContaining({ stepId: "step-ready", skipped: true, reason: "remote_lease_rejected" })]);
|
||||
expect(mockState.releasedLeases).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("falls back to the local soft lease when remote lease acquire fails", async () => {
|
||||
mockState.schedules = [];
|
||||
mockState.readyCandidates = [readyStepCandidate()];
|
||||
mockState.remoteLeaseAcquireResult = { ok: false, fallback: true, error: "lease service unavailable" };
|
||||
const { runDigitalEmployeeSchedulerCheck } = await import("./schedulerService");
|
||||
|
||||
const result = await runDigitalEmployeeSchedulerCheck(new Date("2026-06-07T01:00:10.000Z"));
|
||||
|
||||
expect(result.readyStepDispatch).toMatchObject({ candidates: 1, dispatched: 1, skipped: 0, failed: 0 });
|
||||
expect(mockState.acquiredLeases).toEqual([expect.objectContaining({ stepId: "step-ready", source: "local_fallback", remoteError: "lease service unavailable" })]);
|
||||
expect(mockState.remoteLeaseReleases).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("keeps local release when remote lease release fails", async () => {
|
||||
mockState.schedules = [];
|
||||
mockState.readyCandidates = [readyStepCandidate()];
|
||||
mockState.remoteLeaseReleaseResult = { ok: false, error: "release failed" };
|
||||
const { runDigitalEmployeeSchedulerCheck } = await import("./schedulerService");
|
||||
|
||||
const result = await runDigitalEmployeeSchedulerCheck(new Date("2026-06-07T01:00:10.000Z"));
|
||||
|
||||
expect(result.readyStepDispatch).toMatchObject({ candidates: 1, dispatched: 1, skipped: 0, failed: 0 });
|
||||
expect(mockState.remoteLeaseReleases).toHaveLength(1);
|
||||
expect(mockState.releasedLeases).toEqual([expect.objectContaining({ stepId: "step-ready", remoteReleaseError: "release failed" })]);
|
||||
});
|
||||
|
||||
it("honors disabled ready step dispatch policy", async () => {
|
||||
mockState.readyCandidates = [readyStepCandidate()];
|
||||
mockState.dispatchPolicy = { ...mockState.dispatchPolicy, enabled: false };
|
||||
@@ -271,6 +354,7 @@ describe("digital employee scheduler service", () => {
|
||||
expect(result).toMatchObject({ plan_id: "plan-1", trigger_source: "manual", candidates: 1, dispatched: 1, skipped: 0, failed: 0 });
|
||||
expect(mockState.dispatchEvents[0]).toMatchObject({ status: "started", payload: expect.objectContaining({ trigger_source: "manual" }) });
|
||||
expect(mockState.policyPulls).toHaveLength(0);
|
||||
expect(mockState.remoteLeaseAcquires).toEqual([expect.objectContaining({ stepId: "step-ready", triggerSource: "manual" })]);
|
||||
});
|
||||
|
||||
it("keeps the enabled switch as a manual plan dispatch kill switch", async () => {
|
||||
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
readDigitalEmployeeCronSettings,
|
||||
readDigitalEmployeePlanStepDispatchPolicy,
|
||||
readDigitalEmployeeScheduleRuns,
|
||||
recordDigitalEmployeePlanStepLeaseSkipped,
|
||||
recordDigitalEmployeeGovernanceCommand,
|
||||
recordDigitalEmployeePlanStepDispatch,
|
||||
recordDigitalEmployeeScheduleRunStarted,
|
||||
@@ -19,11 +20,16 @@ import {
|
||||
type DigitalEmployeeReadyPlanStepDispatchCandidate,
|
||||
type DigitalEmployeeScheduleRecord,
|
||||
} from "./stateService";
|
||||
import { pullDigitalEmployeePlanStepDispatchPolicy } from "./syncService";
|
||||
import {
|
||||
acquireDigitalEmployeePlanStepRemoteLease,
|
||||
pullDigitalEmployeePlanStepDispatchPolicy,
|
||||
releaseDigitalEmployeePlanStepRemoteLease,
|
||||
} from "./syncService";
|
||||
|
||||
const SWEEP_INTERVAL_MS = 30_000;
|
||||
const MAX_SCAN_MINUTES = 366 * 24 * 60;
|
||||
const RETRY_DELAYS_MS = [60_000, 5 * 60_000, 15 * 60_000];
|
||||
const PLAN_STEP_LEASE_TTL_MS = 5 * 60_000;
|
||||
|
||||
let schedulerTimer: ReturnType<typeof setInterval> | null = null;
|
||||
let schedulerSweeping = false;
|
||||
@@ -193,10 +199,7 @@ export async function dispatchReadyPlanSteps(
|
||||
result.skipped_step_ids?.push(candidate.stepId);
|
||||
continue;
|
||||
}
|
||||
const leaseResult = acquireDigitalEmployeePlanStepLease({
|
||||
stepId: candidate.stepId,
|
||||
planId: candidate.planId,
|
||||
taskId: candidate.taskId,
|
||||
const leaseResult = await acquireReadyPlanStepDispatchLease(candidate, {
|
||||
triggerSource,
|
||||
orchestrationId,
|
||||
candidateCount: candidates.length,
|
||||
@@ -217,9 +220,12 @@ export async function dispatchReadyPlanSteps(
|
||||
result.failed += 1;
|
||||
result.failed_step_ids?.push(candidate.stepId);
|
||||
result.errors.push({ stepId: candidate.stepId, error: message });
|
||||
releaseDigitalEmployeePlanStepLease({
|
||||
await releaseReadyPlanStepDispatchLease({
|
||||
stepId: candidate.stepId,
|
||||
planId: candidate.planId,
|
||||
leaseId: leaseResult.lease.lease_id,
|
||||
remoteLeaseId: leaseResult.lease.remote_lease_id,
|
||||
leaseSource: leaseResult.lease.source,
|
||||
reason: "failed",
|
||||
taskId: candidate.taskId,
|
||||
occurredAt: now.toISOString(),
|
||||
@@ -239,6 +245,121 @@ export async function dispatchReadyPlanSteps(
|
||||
return result;
|
||||
}
|
||||
|
||||
async function acquireReadyPlanStepDispatchLease(
|
||||
candidate: DigitalEmployeeReadyPlanStepDispatchCandidate,
|
||||
orchestration: { triggerSource: DigitalEmployeeReadyStepDispatchTriggerSource; orchestrationId: string; candidateCount: number; occurredAt: string },
|
||||
): Promise<ReturnType<typeof acquireDigitalEmployeePlanStepLease>> {
|
||||
const remote = await acquireDigitalEmployeePlanStepRemoteLease({
|
||||
planId: candidate.planId,
|
||||
stepId: candidate.stepId,
|
||||
taskId: candidate.taskId,
|
||||
triggerSource: orchestration.triggerSource,
|
||||
ttlMs: PLAN_STEP_LEASE_TTL_MS,
|
||||
occurredAt: orchestration.occurredAt,
|
||||
});
|
||||
if (remote.ok && remote.lease) {
|
||||
const localLease = acquireDigitalEmployeePlanStepLease({
|
||||
stepId: candidate.stepId,
|
||||
planId: candidate.planId,
|
||||
taskId: candidate.taskId,
|
||||
triggerSource: orchestration.triggerSource,
|
||||
orchestrationId: orchestration.orchestrationId,
|
||||
candidateCount: orchestration.candidateCount,
|
||||
occurredAt: orchestration.occurredAt,
|
||||
ttlMs: PLAN_STEP_LEASE_TTL_MS,
|
||||
source: "remote",
|
||||
remoteLease: {
|
||||
lease_id: remote.lease.lease_id,
|
||||
owner_device_id: remote.lease.owner_device_id,
|
||||
acquired_at: remote.lease.acquired_at,
|
||||
expires_at: remote.lease.expires_at,
|
||||
remote_lease_id: remote.lease.remote_lease_id,
|
||||
remote_owner_device_id: remote.lease.remote_owner_device_id,
|
||||
last_remote_checked_at: orchestration.occurredAt,
|
||||
},
|
||||
});
|
||||
if (!localLease.ok) {
|
||||
await releaseDigitalEmployeePlanStepRemoteLease({
|
||||
planId: candidate.planId,
|
||||
stepId: candidate.stepId,
|
||||
taskId: candidate.taskId,
|
||||
leaseId: remote.lease.lease_id,
|
||||
remoteLeaseId: remote.lease.remote_lease_id,
|
||||
reason: localLease.reason || "local_lease_unavailable",
|
||||
occurredAt: orchestration.occurredAt,
|
||||
});
|
||||
}
|
||||
return localLease;
|
||||
}
|
||||
if (remote.rejected) {
|
||||
return recordDigitalEmployeePlanStepLeaseSkipped({
|
||||
stepId: candidate.stepId,
|
||||
planId: candidate.planId,
|
||||
taskId: candidate.taskId,
|
||||
reason: "remote_lease_rejected",
|
||||
occurredAt: orchestration.occurredAt,
|
||||
remoteLease: {
|
||||
lease_id: remote.lease?.lease_id,
|
||||
owner_device_id: remote.lease?.owner_device_id,
|
||||
expires_at: remote.lease?.expires_at,
|
||||
remote_lease_id: remote.lease?.remote_lease_id,
|
||||
remote_owner_device_id: remote.lease?.remote_owner_device_id,
|
||||
remote_error: remote.reason || "remote_lease_rejected",
|
||||
last_remote_checked_at: orchestration.occurredAt,
|
||||
},
|
||||
});
|
||||
}
|
||||
return acquireDigitalEmployeePlanStepLease({
|
||||
stepId: candidate.stepId,
|
||||
planId: candidate.planId,
|
||||
taskId: candidate.taskId,
|
||||
triggerSource: orchestration.triggerSource,
|
||||
orchestrationId: orchestration.orchestrationId,
|
||||
candidateCount: orchestration.candidateCount,
|
||||
occurredAt: orchestration.occurredAt,
|
||||
ttlMs: PLAN_STEP_LEASE_TTL_MS,
|
||||
source: "local_fallback",
|
||||
remoteError: remote.error || remote.reason || "remote_lease_unavailable",
|
||||
});
|
||||
}
|
||||
|
||||
async function releaseReadyPlanStepDispatchLease(input: {
|
||||
stepId: string;
|
||||
planId?: string | null;
|
||||
taskId?: string | null;
|
||||
runId?: string | null;
|
||||
leaseId?: string | null;
|
||||
remoteLeaseId?: string | null;
|
||||
leaseSource?: DigitalEmployeePlanStepLease["source"] | null;
|
||||
reason: string;
|
||||
occurredAt: string;
|
||||
}): Promise<void> {
|
||||
let remoteReleaseError: string | null = null;
|
||||
if (input.leaseSource === "remote" || input.remoteLeaseId) {
|
||||
const released = await releaseDigitalEmployeePlanStepRemoteLease({
|
||||
planId: input.planId,
|
||||
stepId: input.stepId,
|
||||
taskId: input.taskId,
|
||||
runId: input.runId,
|
||||
leaseId: input.leaseId,
|
||||
remoteLeaseId: input.remoteLeaseId,
|
||||
reason: input.reason,
|
||||
occurredAt: input.occurredAt,
|
||||
});
|
||||
remoteReleaseError = released.ok ? null : released.error || released.reason || "remote_release_failed";
|
||||
}
|
||||
releaseDigitalEmployeePlanStepLease({
|
||||
stepId: input.stepId,
|
||||
leaseId: input.leaseId,
|
||||
reason: input.reason,
|
||||
taskId: input.taskId,
|
||||
runId: input.runId,
|
||||
occurredAt: input.occurredAt,
|
||||
remoteReleaseError,
|
||||
lastRemoteCheckedAt: remoteReleaseError ? input.occurredAt : undefined,
|
||||
});
|
||||
}
|
||||
|
||||
function emptyReadyStepDispatchResult(): DigitalEmployeeReadyStepDispatchResult {
|
||||
return {
|
||||
candidates: 0,
|
||||
@@ -318,9 +439,12 @@ async function dispatchReadyPlanStep(
|
||||
},
|
||||
occurredAt,
|
||||
});
|
||||
releaseDigitalEmployeePlanStepLease({
|
||||
await releaseReadyPlanStepDispatchLease({
|
||||
stepId: candidate.stepId,
|
||||
planId: candidate.planId,
|
||||
leaseId: lease.lease_id,
|
||||
remoteLeaseId: lease.remote_lease_id,
|
||||
leaseSource: lease.source,
|
||||
reason: "completed",
|
||||
taskId: candidate.taskId,
|
||||
runId: ids.runId,
|
||||
|
||||
@@ -1518,6 +1518,85 @@ describe("digital employee state service", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves remote and fallback metadata on plan step leases", async () => {
|
||||
const {
|
||||
acquireDigitalEmployeePlanStepLease,
|
||||
activeDigitalEmployeePlanStepLease,
|
||||
recordDigitalEmployeePlanStepLeaseSkipped,
|
||||
releaseDigitalEmployeePlanStepLease,
|
||||
} = await import("./stateService");
|
||||
|
||||
createGovernableTask({ taskStatus: "pending", stepStatus: "ready", runStatus: "completed", finishedAt: "2026-06-07T07:20:00.000Z" });
|
||||
|
||||
const remote = acquireDigitalEmployeePlanStepLease({
|
||||
stepId: "step-governance",
|
||||
planId: "plan-governance",
|
||||
taskId: "task-governance",
|
||||
occurredAt: "2026-06-07T08:20:00.000Z",
|
||||
source: "remote",
|
||||
remoteLease: {
|
||||
lease_id: "remote-lease-1",
|
||||
owner_device_id: "device-remote-owner",
|
||||
acquired_at: "2026-06-07T08:20:00.000Z",
|
||||
expires_at: "2026-06-07T08:25:00.000Z",
|
||||
remote_lease_id: "remote-lease-1",
|
||||
remote_owner_device_id: "device-remote-owner",
|
||||
last_remote_checked_at: "2026-06-07T08:20:00.000Z",
|
||||
},
|
||||
});
|
||||
|
||||
expect(remote).toMatchObject({
|
||||
ok: true,
|
||||
lease: expect.objectContaining({
|
||||
lease_id: "remote-lease-1",
|
||||
source: "remote",
|
||||
remote_lease_id: "remote-lease-1",
|
||||
remote_owner_device_id: "device-remote-owner",
|
||||
}),
|
||||
});
|
||||
expect(activeDigitalEmployeePlanStepLease(JSON.parse(mockState.db?.planSteps.get("step-governance")?.payload ?? "{}"), "2026-06-07T08:21:00.000Z")).toMatchObject({ source: "remote" });
|
||||
|
||||
const released = releaseDigitalEmployeePlanStepLease({
|
||||
stepId: "step-governance",
|
||||
leaseId: remote.lease?.lease_id,
|
||||
reason: "completed",
|
||||
occurredAt: "2026-06-07T08:22:00.000Z",
|
||||
remoteReleaseError: "release failed",
|
||||
lastRemoteCheckedAt: "2026-06-07T08:22:00.000Z",
|
||||
});
|
||||
expect(released).toMatchObject({ ok: true, lease: expect.objectContaining({ source: "remote", remote_release_error: "release failed" }) });
|
||||
|
||||
addReadyPlanStepWithTask({ planId: "plan-governance", stepId: "step-remote-rejected", taskId: "task-remote-rejected" });
|
||||
const skipped = recordDigitalEmployeePlanStepLeaseSkipped({
|
||||
stepId: "step-remote-rejected",
|
||||
planId: "plan-governance",
|
||||
taskId: "task-remote-rejected",
|
||||
reason: "remote_lease_rejected",
|
||||
occurredAt: "2026-06-07T08:23:00.000Z",
|
||||
remoteLease: { remote_error: "locked_by_other_device", last_remote_checked_at: "2026-06-07T08:23:00.000Z" },
|
||||
});
|
||||
expect(skipped).toMatchObject({ ok: false, reason: "remote_lease_rejected", lease: expect.objectContaining({ source: "remote", remote_error: "locked_by_other_device" }) });
|
||||
expect(Array.from(mockState.db?.events.values() ?? [])).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({ kind: "plan_step_lease_skipped", payload: expect.stringContaining("remote_lease_rejected") }),
|
||||
]));
|
||||
});
|
||||
|
||||
it("marks local fallback plan step leases with the remote acquire error", async () => {
|
||||
const { acquireDigitalEmployeePlanStepLease } = await import("./stateService");
|
||||
|
||||
createGovernableTask({ taskStatus: "pending", stepStatus: "ready", runStatus: "completed", finishedAt: "2026-06-07T07:20:00.000Z" });
|
||||
const fallback = acquireDigitalEmployeePlanStepLease({
|
||||
stepId: "step-governance",
|
||||
planId: "plan-governance",
|
||||
taskId: "task-governance",
|
||||
occurredAt: "2026-06-07T08:24:00.000Z",
|
||||
source: "local_fallback",
|
||||
remoteError: "lease service unavailable",
|
||||
});
|
||||
|
||||
expect(fallback).toMatchObject({ ok: true, lease: expect.objectContaining({ source: "local_fallback", remote_error: "lease service unavailable" }) });
|
||||
});
|
||||
|
||||
it("blocks dependent plan steps on cancellation and returns them to pending on retry", async () => {
|
||||
const { recordDigitalEmployeeGovernanceCommand } = await import("./stateService");
|
||||
|
||||
|
||||
@@ -317,6 +317,12 @@ export interface DigitalEmployeePlanStepLease {
|
||||
trigger_source?: string | null;
|
||||
orchestration_id?: string | null;
|
||||
candidate_count?: number | null;
|
||||
source?: "local" | "remote" | "local_fallback";
|
||||
remote_lease_id?: string | null;
|
||||
remote_owner_device_id?: string | null;
|
||||
remote_error?: string | null;
|
||||
remote_release_error?: string | null;
|
||||
last_remote_checked_at?: string | null;
|
||||
}
|
||||
|
||||
export interface DigitalEmployeePlanStepLeaseResult {
|
||||
@@ -325,6 +331,17 @@ export interface DigitalEmployeePlanStepLeaseResult {
|
||||
reason?: string | null;
|
||||
}
|
||||
|
||||
export interface DigitalEmployeePlanStepRemoteLeasePatch {
|
||||
lease_id?: string | null;
|
||||
owner_device_id?: string | null;
|
||||
acquired_at?: string | null;
|
||||
expires_at?: string | null;
|
||||
remote_lease_id?: string | null;
|
||||
remote_owner_device_id?: string | null;
|
||||
remote_error?: string | null;
|
||||
last_remote_checked_at?: string | null;
|
||||
}
|
||||
|
||||
export interface DigitalEmployeePlanStepLeaseInput {
|
||||
stepId: string;
|
||||
planId?: string | null;
|
||||
@@ -335,6 +352,9 @@ export interface DigitalEmployeePlanStepLeaseInput {
|
||||
candidateCount?: number | null;
|
||||
occurredAt?: string | null;
|
||||
ttlMs?: number | null;
|
||||
source?: DigitalEmployeePlanStepLease["source"] | null;
|
||||
remoteLease?: DigitalEmployeePlanStepRemoteLeasePatch | null;
|
||||
remoteError?: string | null;
|
||||
}
|
||||
|
||||
export interface DigitalEmployeePlanStepDispatchInput {
|
||||
@@ -1456,9 +1476,20 @@ export function activeDigitalEmployeePlanStepLease(
|
||||
trigger_source: stringValue(lease.trigger_source ?? lease.triggerSource) || null,
|
||||
orchestration_id: stringValue(lease.orchestration_id ?? lease.orchestrationId) || null,
|
||||
candidate_count: numberValue(lease.candidate_count ?? lease.candidateCount),
|
||||
source: normalizePlanStepLeaseSource(lease.source),
|
||||
remote_lease_id: stringValue(lease.remote_lease_id ?? lease.remoteLeaseId) || null,
|
||||
remote_owner_device_id: stringValue(lease.remote_owner_device_id ?? lease.remoteOwnerDeviceId) || null,
|
||||
remote_error: stringValue(lease.remote_error ?? lease.remoteError) || null,
|
||||
remote_release_error: stringValue(lease.remote_release_error ?? lease.remoteReleaseError) || null,
|
||||
last_remote_checked_at: stringValue(lease.last_remote_checked_at ?? lease.lastRemoteCheckedAt) || null,
|
||||
};
|
||||
}
|
||||
|
||||
function normalizePlanStepLeaseSource(value: unknown): DigitalEmployeePlanStepLease["source"] {
|
||||
if (value === "remote" || value === "local_fallback") return value;
|
||||
return "local";
|
||||
}
|
||||
|
||||
export function acquireDigitalEmployeePlanStepLease(
|
||||
input: DigitalEmployeePlanStepLeaseInput,
|
||||
): DigitalEmployeePlanStepLeaseResult {
|
||||
@@ -1481,14 +1512,20 @@ export function acquireDigitalEmployeePlanStepLease(
|
||||
}
|
||||
const ttlMs = Math.max(60_000, Math.min(Number(input.ttlMs) || PLAN_STEP_LEASE_TTL_MS, 30 * 60 * 1000));
|
||||
const lease: DigitalEmployeePlanStepLease = {
|
||||
lease_id: `plan-step-lease:${safeId(stepId)}:${safeId(now)}`,
|
||||
lease_id: stringValue(input.remoteLease?.lease_id) || `plan-step-lease:${safeId(stepId)}:${safeId(now)}`,
|
||||
state: "active",
|
||||
owner_device_id: getDeviceId(),
|
||||
acquired_at: now,
|
||||
expires_at: new Date(new Date(now).getTime() + ttlMs).toISOString(),
|
||||
owner_device_id: stringValue(input.remoteLease?.owner_device_id) || getDeviceId(),
|
||||
acquired_at: stringValue(input.remoteLease?.acquired_at) || now,
|
||||
expires_at: stringValue(input.remoteLease?.expires_at) || new Date(new Date(now).getTime() + ttlMs).toISOString(),
|
||||
trigger_source: stringValue(input.triggerSource) || null,
|
||||
orchestration_id: stringValue(input.orchestrationId) || null,
|
||||
candidate_count: numberValue(input.candidateCount),
|
||||
source: normalizePlanStepLeaseSource(input.source),
|
||||
remote_lease_id: stringValue(input.remoteLease?.remote_lease_id ?? input.remoteLease?.lease_id) || null,
|
||||
remote_owner_device_id: stringValue(input.remoteLease?.remote_owner_device_id ?? input.remoteLease?.owner_device_id) || null,
|
||||
remote_error: stringValue(input.remoteError ?? input.remoteLease?.remote_error) || null,
|
||||
remote_release_error: null,
|
||||
last_remote_checked_at: stringValue(input.remoteLease?.last_remote_checked_at) || (input.remoteLease || input.remoteError ? now : null),
|
||||
};
|
||||
const payload = { ...previousPayload, dispatchLease: lease };
|
||||
db.prepare(`
|
||||
@@ -1508,6 +1545,8 @@ export function releaseDigitalEmployeePlanStepLease(input: {
|
||||
taskId?: string | null;
|
||||
runId?: string | null;
|
||||
occurredAt?: string | null;
|
||||
remoteReleaseError?: string | null;
|
||||
lastRemoteCheckedAt?: string | null;
|
||||
}): DigitalEmployeePlanStepLeaseResult {
|
||||
const db = getDigitalEmployeeDb();
|
||||
if (!db) return { ok: false, reason: "database_unavailable" };
|
||||
@@ -1538,6 +1577,12 @@ export function releaseDigitalEmployeePlanStepLease(input: {
|
||||
trigger_source: stringValue(currentLease.trigger_source ?? currentLease.triggerSource) || null,
|
||||
orchestration_id: stringValue(currentLease.orchestration_id ?? currentLease.orchestrationId) || null,
|
||||
candidate_count: numberValue(currentLease.candidate_count ?? currentLease.candidateCount),
|
||||
source: normalizePlanStepLeaseSource(currentLease.source),
|
||||
remote_lease_id: stringValue(currentLease.remote_lease_id ?? currentLease.remoteLeaseId) || null,
|
||||
remote_owner_device_id: stringValue(currentLease.remote_owner_device_id ?? currentLease.remoteOwnerDeviceId) || null,
|
||||
remote_error: stringValue(currentLease.remote_error ?? currentLease.remoteError) || null,
|
||||
remote_release_error: stringValue(input.remoteReleaseError ?? currentLease.remote_release_error ?? currentLease.remoteReleaseError) || null,
|
||||
last_remote_checked_at: stringValue(input.lastRemoteCheckedAt ?? currentLease.last_remote_checked_at ?? currentLease.lastRemoteCheckedAt) || null,
|
||||
};
|
||||
const payload = { ...previousPayload, dispatchLease: lease };
|
||||
db.prepare(`
|
||||
@@ -1550,6 +1595,46 @@ export function releaseDigitalEmployeePlanStepLease(input: {
|
||||
return { ok: true, lease };
|
||||
}
|
||||
|
||||
export function recordDigitalEmployeePlanStepLeaseSkipped(input: {
|
||||
stepId: string;
|
||||
planId?: string | null;
|
||||
taskId?: string | null;
|
||||
runId?: string | null;
|
||||
reason?: string | null;
|
||||
occurredAt?: string | null;
|
||||
remoteLease?: DigitalEmployeePlanStepRemoteLeasePatch | null;
|
||||
}): DigitalEmployeePlanStepLeaseResult {
|
||||
const db = getDigitalEmployeeDb();
|
||||
if (!db) return { ok: false, reason: "database_unavailable" };
|
||||
const now = input.occurredAt || new Date().toISOString();
|
||||
const stepId = stringValue(input.stepId);
|
||||
if (!stepId) return { ok: false, reason: "missing_step" };
|
||||
const row = db.prepare(`
|
||||
SELECT id, plan_id, title, status, seq, depends_on, preferred_skill_ids, payload, created_at
|
||||
FROM digital_plan_steps
|
||||
WHERE id = ?
|
||||
`).get(stepId) as Record<string, unknown> | undefined;
|
||||
if (!row) return { ok: false, reason: "missing_step" };
|
||||
const reason = stringValue(input.reason) || "remote_lease_rejected";
|
||||
const ttlMs = PLAN_STEP_LEASE_TTL_MS;
|
||||
const lease: DigitalEmployeePlanStepLease = {
|
||||
lease_id: stringValue(input.remoteLease?.lease_id) || `plan-step-lease:${safeId(stepId)}:${safeId(now)}:skipped`,
|
||||
state: "expired",
|
||||
owner_device_id: stringValue(input.remoteLease?.owner_device_id) || getDeviceId(),
|
||||
acquired_at: stringValue(input.remoteLease?.acquired_at) || now,
|
||||
expires_at: stringValue(input.remoteLease?.expires_at) || new Date(new Date(now).getTime() + ttlMs).toISOString(),
|
||||
release_reason: reason,
|
||||
source: "remote",
|
||||
remote_lease_id: stringValue(input.remoteLease?.remote_lease_id ?? input.remoteLease?.lease_id) || null,
|
||||
remote_owner_device_id: stringValue(input.remoteLease?.remote_owner_device_id ?? input.remoteLease?.owner_device_id) || null,
|
||||
remote_error: stringValue(input.remoteLease?.remote_error) || null,
|
||||
remote_release_error: null,
|
||||
last_remote_checked_at: stringValue(input.remoteLease?.last_remote_checked_at) || now,
|
||||
};
|
||||
recordDigitalEmployeePlanStepLeaseEvent({ row, lease, kind: "plan_step_lease_skipped", reason, now, taskId: input.taskId, runId: input.runId });
|
||||
return { ok: false, lease, reason };
|
||||
}
|
||||
|
||||
export function listDigitalEmployeeReadyPlanStepDispatchCandidates(
|
||||
options: number | { limit?: number; planId?: string | null } = 20,
|
||||
): DigitalEmployeeReadyPlanStepDispatchCandidate[] {
|
||||
@@ -1785,6 +1870,11 @@ function recordDigitalEmployeePlanStepLeaseEvent(input: {
|
||||
lease_state: input.lease.state,
|
||||
lease_owner_device_id: input.lease.owner_device_id,
|
||||
lease_expires_at: input.lease.expires_at,
|
||||
lease_source: input.lease.source || "local",
|
||||
remote_lease_id: input.lease.remote_lease_id || null,
|
||||
remote_lease_error: input.lease.remote_error || null,
|
||||
remote_release_error: input.lease.remote_release_error || null,
|
||||
remote_lease_rejected: stringValue(input.reason) === "remote_lease_rejected",
|
||||
reason: stringValue(input.reason) || input.lease.release_reason || null,
|
||||
};
|
||||
insertEvent(
|
||||
|
||||
@@ -137,6 +137,93 @@ describe("digital employee sync service", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("acquires a remote plan step dispatch lease through the management endpoint", async () => {
|
||||
mockState.fetch.mockResolvedValue(jsonResponse({
|
||||
ok: true,
|
||||
lease: {
|
||||
lease_id: "remote-lease-1",
|
||||
owner_device_id: "device-001",
|
||||
acquired_at: "2026-06-07T08:00:00.000Z",
|
||||
expires_at: "2026-06-07T08:05:00.000Z",
|
||||
},
|
||||
}));
|
||||
|
||||
const { acquireDigitalEmployeePlanStepRemoteLease } = await import("./syncService");
|
||||
const result = await acquireDigitalEmployeePlanStepRemoteLease({
|
||||
planId: "plan-1",
|
||||
stepId: "step-1",
|
||||
taskId: "task-1",
|
||||
triggerSource: "auto",
|
||||
ttlMs: 300_000,
|
||||
occurredAt: "2026-06-07T08:00:00.000Z",
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
ok: true,
|
||||
endpoint: "https://manage.example.com/api/digital-employee/leases/plan-step-dispatch/acquire",
|
||||
lease: {
|
||||
lease_id: "remote-lease-1",
|
||||
remote_lease_id: "remote-lease-1",
|
||||
owner_device_id: "device-001",
|
||||
},
|
||||
});
|
||||
expect(mockState.fetch).toHaveBeenCalledWith(
|
||||
"https://manage.example.com/api/digital-employee/leases/plan-step-dispatch/acquire",
|
||||
expect.objectContaining({
|
||||
method: "POST",
|
||||
headers: expect.objectContaining({
|
||||
Authorization: "Bearer login-token-001",
|
||||
"X-Qiming-Client-Key": "client-key-001",
|
||||
"X-Qiming-Device-Id": "device-001",
|
||||
}),
|
||||
body: expect.stringContaining('"step_id":"step-1"'),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("returns a remote lease rejection without falling back", async () => {
|
||||
mockState.fetch.mockResolvedValue(jsonResponse({ ok: false, reason: "locked_by_other_device" }));
|
||||
|
||||
const { acquireDigitalEmployeePlanStepRemoteLease } = await import("./syncService");
|
||||
const result = await acquireDigitalEmployeePlanStepRemoteLease({ stepId: "step-1", occurredAt: "2026-06-07T08:00:00.000Z" });
|
||||
|
||||
expect(result).toMatchObject({ ok: false, rejected: true, reason: "locked_by_other_device" });
|
||||
});
|
||||
|
||||
it("falls back when remote plan step dispatch lease acquire fails", async () => {
|
||||
mockState.fetch.mockResolvedValue({
|
||||
ok: false,
|
||||
status: 503,
|
||||
text: vi.fn().mockResolvedValue(JSON.stringify({ message: "lease service unavailable" })),
|
||||
} as unknown as Response);
|
||||
|
||||
const { acquireDigitalEmployeePlanStepRemoteLease } = await import("./syncService");
|
||||
const result = await acquireDigitalEmployeePlanStepRemoteLease({ stepId: "step-1", occurredAt: "2026-06-07T08:00:00.000Z" });
|
||||
|
||||
expect(result).toMatchObject({ ok: false, fallback: true, error: "lease service unavailable" });
|
||||
});
|
||||
|
||||
it("reports remote release failures without throwing", async () => {
|
||||
mockState.fetch.mockResolvedValue({
|
||||
ok: false,
|
||||
status: 500,
|
||||
text: vi.fn().mockResolvedValue(JSON.stringify({ message: "release failed" })),
|
||||
} as unknown as Response);
|
||||
|
||||
const { releaseDigitalEmployeePlanStepRemoteLease } = await import("./syncService");
|
||||
const result = await releaseDigitalEmployeePlanStepRemoteLease({
|
||||
planId: "plan-1",
|
||||
stepId: "step-1",
|
||||
taskId: "task-1",
|
||||
leaseId: "remote-lease-1",
|
||||
remoteLeaseId: "remote-lease-1",
|
||||
reason: "completed",
|
||||
occurredAt: "2026-06-07T08:01:00.000Z",
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ ok: false, endpoint: "https://manage.example.com/api/digital-employee/leases/plan-step-dispatch/release", error: "release failed" });
|
||||
});
|
||||
|
||||
it("marks entity-only acknowledgements as synced", async () => {
|
||||
insertPlan("plan-1");
|
||||
insertOutbox("plan:upsert:plan-1", "plan", "plan-1");
|
||||
@@ -659,6 +746,9 @@ describe("digital employee sync service", () => {
|
||||
lease_state: "released",
|
||||
lease_owner_device_id: "device-001",
|
||||
lease_expires_at: "2026-06-07T08:04:00.000Z",
|
||||
lease_source: "remote",
|
||||
remote_lease_id: "remote-lease-step-1",
|
||||
remote_release_error: "release failed",
|
||||
reason: "completed",
|
||||
lease: {
|
||||
lease_id: "lease-step-1",
|
||||
@@ -668,6 +758,9 @@ describe("digital employee sync service", () => {
|
||||
expires_at: "2026-06-07T08:04:00.000Z",
|
||||
released_at: "2026-06-07T08:01:00.000Z",
|
||||
release_reason: "completed",
|
||||
source: "remote",
|
||||
remote_lease_id: "remote-lease-step-1",
|
||||
remote_release_error: "release failed",
|
||||
},
|
||||
});
|
||||
insertEventEntity("event-plan-step-ready", "plan_step_ready", {
|
||||
@@ -937,6 +1030,10 @@ describe("digital employee sync service", () => {
|
||||
lease_expires_at: "2026-06-07T08:04:00.000Z",
|
||||
lease_released_at: "2026-06-07T08:01:00.000Z",
|
||||
lease_release_reason: "completed",
|
||||
lease_source: "remote",
|
||||
remote_lease_id: "remote-lease-step-1",
|
||||
remote_release_error: "release failed",
|
||||
remote_lease_rejected: false,
|
||||
reason: "completed",
|
||||
});
|
||||
expect(businessView("event-plan-step-ready")).toMatchObject({
|
||||
|
||||
@@ -11,6 +11,7 @@ import {
|
||||
|
||||
const DEFAULT_SYNC_PATH = "/api/digital-employee/sync/outbox";
|
||||
const DEFAULT_PLAN_STEP_DISPATCH_POLICY_PATH = "/api/digital-employee/policies/plan-step-dispatch";
|
||||
const DEFAULT_PLAN_STEP_DISPATCH_LEASE_PATH = "/api/digital-employee/leases/plan-step-dispatch";
|
||||
const MANAGEMENT_SYNC_RECORD_PATH =
|
||||
"/system/content/content-digital-employee-sync";
|
||||
const DIGITAL_EMPLOYEE_SYNC_CONTRACT_VERSION = "qimingclaw.digital_employee.sync.v1";
|
||||
@@ -38,6 +39,7 @@ interface DigitalEmployeeSyncConfig {
|
||||
enabled: boolean;
|
||||
endpoint: string | null;
|
||||
policyEndpoint: string | null;
|
||||
leaseEndpoint: string | null;
|
||||
clientKey: string | null;
|
||||
authToken: string | null;
|
||||
batchSize: number;
|
||||
@@ -52,6 +54,48 @@ export interface DigitalEmployeePlanStepDispatchPolicyPullResult {
|
||||
error?: string | null;
|
||||
}
|
||||
|
||||
export interface DigitalEmployeePlanStepRemoteLease {
|
||||
lease_id: string;
|
||||
state: "active" | "released" | "expired";
|
||||
owner_device_id: string;
|
||||
acquired_at: string;
|
||||
expires_at: string;
|
||||
released_at?: string | null;
|
||||
release_reason?: string | null;
|
||||
remote_lease_id?: string | null;
|
||||
remote_owner_device_id?: string | null;
|
||||
}
|
||||
|
||||
export interface DigitalEmployeePlanStepRemoteLeaseResult {
|
||||
ok: boolean;
|
||||
rejected?: boolean;
|
||||
fallback?: boolean;
|
||||
endpoint: string | null;
|
||||
lease?: DigitalEmployeePlanStepRemoteLease | null;
|
||||
reason?: string | null;
|
||||
error?: string | null;
|
||||
}
|
||||
|
||||
export interface DigitalEmployeePlanStepRemoteLeaseAcquireInput {
|
||||
planId?: string | null;
|
||||
stepId: string;
|
||||
taskId?: string | null;
|
||||
triggerSource?: string | null;
|
||||
ttlMs?: number | null;
|
||||
occurredAt?: string | null;
|
||||
}
|
||||
|
||||
export interface DigitalEmployeePlanStepRemoteLeaseReleaseInput {
|
||||
planId?: string | null;
|
||||
stepId: string;
|
||||
taskId?: string | null;
|
||||
runId?: string | null;
|
||||
leaseId?: string | null;
|
||||
remoteLeaseId?: string | null;
|
||||
reason?: string | null;
|
||||
occurredAt?: string | null;
|
||||
}
|
||||
|
||||
interface OutboxRow {
|
||||
id: string;
|
||||
entity_type: string;
|
||||
@@ -245,6 +289,80 @@ export async function pullDigitalEmployeePlanStepDispatchPolicy(
|
||||
}
|
||||
}
|
||||
|
||||
export async function acquireDigitalEmployeePlanStepRemoteLease(
|
||||
input: DigitalEmployeePlanStepRemoteLeaseAcquireInput,
|
||||
): Promise<DigitalEmployeePlanStepRemoteLeaseResult> {
|
||||
const config = resolveSyncConfig();
|
||||
const endpoint = buildLeaseOperationEndpoint(config.leaseEndpoint, "acquire");
|
||||
const missingCredentials = missingSyncCredentialLabels(config);
|
||||
if (!endpoint || missingCredentials.length > 0) {
|
||||
return {
|
||||
ok: false,
|
||||
fallback: true,
|
||||
endpoint,
|
||||
error: !endpoint ? "management lease endpoint unavailable" : `missing credentials: ${missingCredentials.join(", ")}`,
|
||||
};
|
||||
}
|
||||
const occurredAt = input.occurredAt || new Date().toISOString();
|
||||
const ttlMs = Math.max(60_000, Math.min(Number(input.ttlMs) || 5 * 60_000, 30 * 60_000));
|
||||
try {
|
||||
const response = await fetchJsonWithAuth(config, endpoint, {
|
||||
client_key: config.clientKey,
|
||||
device_id: getDeviceId(),
|
||||
plan_id: stringField(input.planId) || null,
|
||||
step_id: input.stepId,
|
||||
task_id: stringField(input.taskId) || null,
|
||||
trigger_source: stringField(input.triggerSource) || null,
|
||||
ttl_ms: ttlMs,
|
||||
occurred_at: occurredAt,
|
||||
});
|
||||
const rejectedReason = readPlanStepLeaseRejectionReason(response);
|
||||
if (rejectedReason) return { ok: false, rejected: true, endpoint, reason: rejectedReason, lease: readRemotePlanStepLease(response, occurredAt, ttlMs) };
|
||||
const responseError = readSyncResponseError(response as SyncResponse);
|
||||
if (responseError) throw new Error(responseError);
|
||||
const lease = readRemotePlanStepLease(response, occurredAt, ttlMs);
|
||||
if (!lease) throw new Error("management lease response missing plan step lease");
|
||||
return { ok: true, endpoint, lease };
|
||||
} catch (error) {
|
||||
return { ok: false, fallback: true, endpoint, error: normalizeError(error) };
|
||||
}
|
||||
}
|
||||
|
||||
export async function releaseDigitalEmployeePlanStepRemoteLease(
|
||||
input: DigitalEmployeePlanStepRemoteLeaseReleaseInput,
|
||||
): Promise<DigitalEmployeePlanStepRemoteLeaseResult> {
|
||||
const config = resolveSyncConfig();
|
||||
const endpoint = buildLeaseOperationEndpoint(config.leaseEndpoint, "release");
|
||||
const missingCredentials = missingSyncCredentialLabels(config);
|
||||
if (!endpoint || missingCredentials.length > 0) {
|
||||
return {
|
||||
ok: false,
|
||||
endpoint,
|
||||
error: !endpoint ? "management lease endpoint unavailable" : `missing credentials: ${missingCredentials.join(", ")}`,
|
||||
};
|
||||
}
|
||||
const occurredAt = input.occurredAt || new Date().toISOString();
|
||||
try {
|
||||
const response = await fetchJsonWithAuth(config, endpoint, {
|
||||
client_key: config.clientKey,
|
||||
device_id: getDeviceId(),
|
||||
plan_id: stringField(input.planId) || null,
|
||||
step_id: input.stepId,
|
||||
task_id: stringField(input.taskId) || null,
|
||||
run_id: stringField(input.runId) || null,
|
||||
lease_id: stringField(input.leaseId) || null,
|
||||
remote_lease_id: stringField(input.remoteLeaseId) || stringField(input.leaseId) || null,
|
||||
reason: stringField(input.reason) || null,
|
||||
occurred_at: occurredAt,
|
||||
});
|
||||
const responseError = readSyncResponseError(response as SyncResponse);
|
||||
if (responseError) throw new Error(responseError);
|
||||
return { ok: true, endpoint, lease: readRemotePlanStepLease(response, occurredAt, 0) };
|
||||
} catch (error) {
|
||||
return { ok: false, endpoint, error: normalizeError(error) };
|
||||
}
|
||||
}
|
||||
|
||||
export async function flushDigitalEmployeeSyncOutbox(
|
||||
options: { force?: boolean } = {},
|
||||
): Promise<DigitalEmployeeSyncStatus> {
|
||||
@@ -375,6 +493,11 @@ function resolveSyncConfig(): DigitalEmployeeSyncConfig {
|
||||
? config.policyEndpoint.trim()
|
||||
: null;
|
||||
const policyEndpoint = policyEndpointOverride || buildEndpoint(serverHost, DEFAULT_PLAN_STEP_DISPATCH_POLICY_PATH);
|
||||
const leaseEndpointOverride =
|
||||
typeof config.planStepDispatchLeaseEndpoint === "string" && config.planStepDispatchLeaseEndpoint.trim()
|
||||
? config.planStepDispatchLeaseEndpoint.trim()
|
||||
: null;
|
||||
const leaseEndpoint = leaseEndpointOverride || buildEndpoint(serverHost, DEFAULT_PLAN_STEP_DISPATCH_LEASE_PATH);
|
||||
const disabled = config.enabled === false;
|
||||
const batchSize =
|
||||
typeof config.batchSize === "number" && config.batchSize > 0
|
||||
@@ -385,12 +508,18 @@ function resolveSyncConfig(): DigitalEmployeeSyncConfig {
|
||||
enabled: !disabled,
|
||||
endpoint,
|
||||
policyEndpoint,
|
||||
leaseEndpoint,
|
||||
clientKey: readClientKey(serverHost),
|
||||
authToken: readAuthToken(serverHost),
|
||||
batchSize,
|
||||
};
|
||||
}
|
||||
|
||||
function buildLeaseOperationEndpoint(baseEndpoint: string | null, operation: "acquire" | "release"): string | null {
|
||||
if (!baseEndpoint) return null;
|
||||
return `${baseEndpoint.replace(/\/+$/, "")}/${operation}`;
|
||||
}
|
||||
|
||||
function buildEndpoint(serverHost: string | null, path = DEFAULT_SYNC_PATH): string | null {
|
||||
if (!serverHost) return null;
|
||||
const normalized = serverHost.startsWith("http")
|
||||
@@ -516,6 +645,60 @@ function readRemotePlanStepDispatchPolicy(response: Record<string, unknown>): Re
|
||||
?? (hasPlanStepDispatchPolicyFields(data) ? data : null);
|
||||
}
|
||||
|
||||
function readPlanStepLeaseRejectionReason(response: Record<string, unknown>): string | null {
|
||||
if (response.ok === false || response.success === false || response.accepted === false || response.granted === false) {
|
||||
return stringField(response.reason ?? response.error ?? response.message) || "remote_lease_rejected";
|
||||
}
|
||||
const data = objectRecord(response.data);
|
||||
if (data && (data.ok === false || data.success === false || data.accepted === false || data.granted === false)) {
|
||||
return stringField(data.reason ?? data.error ?? data.message) || "remote_lease_rejected";
|
||||
}
|
||||
const status = normalizeResponseCode(response.status ?? data?.status);
|
||||
if (status && ["rejected", "denied", "conflict", "locked"].includes(status.toLowerCase())) {
|
||||
return stringField(response.reason ?? response.message ?? data?.reason ?? data?.message) || "remote_lease_rejected";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function readRemotePlanStepLease(
|
||||
response: Record<string, unknown>,
|
||||
occurredAt: string,
|
||||
ttlMs: number,
|
||||
): DigitalEmployeePlanStepRemoteLease | null {
|
||||
const data = objectRecord(response.data) ?? response;
|
||||
const lease = objectRecord(data.lease)
|
||||
?? objectRecord(data.plan_step_dispatch_lease)
|
||||
?? objectRecord(data.planStepDispatchLease)
|
||||
?? (hasPlanStepLeaseFields(data) ? data : null);
|
||||
if (!lease) return null;
|
||||
const remoteLeaseId = stringField(lease.remote_lease_id ?? lease.remoteLeaseId ?? lease.lease_id ?? lease.leaseId);
|
||||
const leaseId = stringField(lease.lease_id ?? lease.leaseId) || remoteLeaseId;
|
||||
const ownerDeviceId = stringField(lease.owner_device_id ?? lease.ownerDeviceId) || getDeviceId();
|
||||
const expiresAt = stringField(lease.expires_at ?? lease.expiresAt)
|
||||
|| (ttlMs > 0 ? new Date(new Date(occurredAt).getTime() + ttlMs).toISOString() : occurredAt);
|
||||
if (!leaseId || !ownerDeviceId || !expiresAt) return null;
|
||||
return {
|
||||
lease_id: leaseId,
|
||||
state: normalizePlanStepRemoteLeaseState(lease.state),
|
||||
owner_device_id: ownerDeviceId,
|
||||
acquired_at: stringField(lease.acquired_at ?? lease.acquiredAt) || occurredAt,
|
||||
expires_at: expiresAt,
|
||||
released_at: stringField(lease.released_at ?? lease.releasedAt) || null,
|
||||
release_reason: stringField(lease.release_reason ?? lease.releaseReason) || null,
|
||||
remote_lease_id: remoteLeaseId || leaseId,
|
||||
remote_owner_device_id: stringField(lease.remote_owner_device_id ?? lease.remoteOwnerDeviceId ?? lease.owner_device_id ?? lease.ownerDeviceId) || null,
|
||||
};
|
||||
}
|
||||
|
||||
function hasPlanStepLeaseFields(record: Record<string, unknown>): boolean {
|
||||
return "lease_id" in record || "leaseId" in record || "remote_lease_id" in record || "remoteLeaseId" in record;
|
||||
}
|
||||
|
||||
function normalizePlanStepRemoteLeaseState(value: unknown): DigitalEmployeePlanStepRemoteLease["state"] {
|
||||
if (value === "released" || value === "expired") return value;
|
||||
return "active";
|
||||
}
|
||||
|
||||
function hasPlanStepDispatchPolicyFields(record: Record<string, unknown>): boolean {
|
||||
return "enabled" in record || "max_per_sweep" in record || "auto_dispatch_ready_steps" in record;
|
||||
}
|
||||
@@ -675,6 +858,11 @@ function planStepBusinessView(
|
||||
lease_expires_at: leaseExpiresAt,
|
||||
lease_released_at: stringField(dispatchLease.released_at ?? dispatchLease.releasedAt) || null,
|
||||
lease_release_reason: stringField(dispatchLease.release_reason ?? dispatchLease.releaseReason) || null,
|
||||
lease_source: stringField(dispatchLease.source) || null,
|
||||
remote_lease_id: stringField(dispatchLease.remote_lease_id ?? dispatchLease.remoteLeaseId) || null,
|
||||
remote_lease_error: stringField(dispatchLease.remote_error ?? dispatchLease.remoteError) || null,
|
||||
remote_release_error: stringField(dispatchLease.remote_release_error ?? dispatchLease.remoteReleaseError) || null,
|
||||
remote_lease_rejected: skipReason === "remote_lease_rejected",
|
||||
lease_active: leaseActive,
|
||||
};
|
||||
}
|
||||
@@ -862,6 +1050,11 @@ function planStepLeaseBusinessView(payload: Record<string, unknown>): Record<str
|
||||
lease_expires_at: stringField(payload.lease_expires_at ?? payload.leaseExpiresAt ?? lease.expires_at ?? lease.expiresAt) || null,
|
||||
lease_released_at: stringField(lease.released_at ?? lease.releasedAt) || null,
|
||||
lease_release_reason: stringField(lease.release_reason ?? lease.releaseReason) || null,
|
||||
lease_source: stringField(payload.lease_source ?? payload.leaseSource ?? lease.source) || null,
|
||||
remote_lease_id: stringField(payload.remote_lease_id ?? payload.remoteLeaseId ?? lease.remote_lease_id ?? lease.remoteLeaseId) || null,
|
||||
remote_lease_error: stringField(payload.remote_lease_error ?? payload.remoteLeaseError ?? lease.remote_error ?? lease.remoteError) || null,
|
||||
remote_release_error: stringField(payload.remote_release_error ?? payload.remoteReleaseError ?? lease.remote_release_error ?? lease.remoteReleaseError) || null,
|
||||
remote_lease_rejected: payload.remote_lease_rejected === true || stringField(payload.reason) === "remote_lease_rejected",
|
||||
reason: stringField(payload.reason) || null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -540,9 +540,9 @@ GET /api/digital-employee/approvals
|
||||
|
||||
1. **PlanStep 与依赖图（依赖图调度已开始）**
|
||||
- 已吸收：Plan / Task / Run 记录、计划模板、`create_plan` 中的 steps/tasks 可物化为任务；PlanStep 已开始作为正式本地实体落入 `digital_plan_steps`。
|
||||
- 当前能力：PlanStep 会进入 `digital_sync_outbox`，entity type 为 `plan_step`；任务中心 flow 和 debug store 优先读取真实 PlanStep 生成步骤图；Task Agent 的 retry / skip / cancel / pause / resume / input / run 控制已可更新真实 Task、PlanStep、Run、Event 和 outbox；前置步骤跳过、取消或重试后会重新评估后继步骤，推进为 ready / blocked / pending 并写入依赖图事件；scheduler sweep 会识别可派发的 ready PlanStep，调用本地 `/computer/chat` 并写入 `plan_step_dispatch_*` 与 `governance_start_run` 事件；派发前会写入 5 分钟本地软租约 `payload.dispatchLease`，已有未过期租约的步骤会以 `lease_active` 跳过，派发完成或失败后释放租约并写入 `plan_step_lease_*` 事件；auto sweep 会按节流从管理端 `GET /api/digital-employee/policies/plan-step-dispatch` 拉取远端派发策略，拉取失败时保留本地策略并记录 `last_pull_error`；embedded 已提供 `/api/digital-employee/plan-steps`、`/api/digital-employee/plan-step-graph`、`/api/digital-employee/plans/:planId/dispatch-ready-steps`、`/api/plan-step/dispatch-policy` 和 `/api/plan-step/dispatch-policy/pull`，ready PlanStep 安全派发、计划级跨任务推进、管理端依赖图视图、本地/远端派发策略、可观察租约状态和首页阻塞依赖处理入口已开始闭环。
|
||||
- 后续缺口：真正由管理端仲裁的跨设备强租约仍待吸收。
|
||||
- 建议：下一轮围绕管理端 lease 仲裁，把本地可治理调度推进到跨端治理闭环。
|
||||
- 当前能力：PlanStep 会进入 `digital_sync_outbox`，entity type 为 `plan_step`；任务中心 flow 和 debug store 优先读取真实 PlanStep 生成步骤图；Task Agent 的 retry / skip / cancel / pause / resume / input / run 控制已可更新真实 Task、PlanStep、Run、Event 和 outbox；前置步骤跳过、取消或重试后会重新评估后继步骤，推进为 ready / blocked / pending 并写入依赖图事件；scheduler sweep 会识别可派发的 ready PlanStep，派发前先通过管理端 `POST /api/digital-employee/leases/plan-step-dispatch/acquire` 做跨设备强租约仲裁，管理端拒绝会以 `remote_lease_rejected` 跳过，管理端异常会降级为 5 分钟本地软租约 `payload.dispatchLease.source = local_fallback` 并记录 `remote_error`；调用本地 `/computer/chat` 后会写入 `plan_step_dispatch_*` 与 `governance_start_run` 事件，派发完成或失败会先尝试 `release` 管理端租约再释放本地 lease 并写入 `plan_step_lease_*` 事件；已有未过期租约的步骤仍会以 `lease_active` 跳过；auto sweep 会按节流从管理端 `GET /api/digital-employee/policies/plan-step-dispatch` 拉取远端派发策略，拉取失败时保留本地策略并记录 `last_pull_error`；embedded 已提供 `/api/digital-employee/plan-steps`、`/api/digital-employee/plan-step-graph`、`/api/digital-employee/plans/:planId/dispatch-ready-steps`、`/api/plan-step/dispatch-policy` 和 `/api/plan-step/dispatch-policy/pull`，ready PlanStep 安全派发、计划级跨任务推进、管理端依赖图视图、本地/远端派发策略、可观察租约状态和首页阻塞依赖处理入口已开始闭环。
|
||||
- 后续缺口：管理端正式路由 UI、风险策略下发和高风险动作审批策略仍待吸收。
|
||||
- 建议：下一轮围绕管理端正式路由 UI 或技能策略远端下发，把本地可治理调度继续推进到跨端治理策略。
|
||||
|
||||
2. **调度 / 定时 / 周期任务**
|
||||
- 已吸收：`digital_schedules`、`digital_schedule_runs` 已进入 qimingclaw 本地 SQLite 与 outbox，entity type 为 `schedule` / `schedule_run`；旧 `/api/cron`、`/api/scheduler/jobs` 和治理命令会优先读取真实本地调度记录。
|
||||
@@ -590,7 +590,7 @@ GET /api/digital-employee/approvals
|
||||
10. **入口路由 / 任务分流（路由业务视图与通知联动已开始）**
|
||||
- 已吸收：管理端通过 `/computer/chat` 进入 qimingclaw 后会生成本地任务事实；embedded `/api/ingress/route`、`/api/route-decisions` 和 `/api/digital-employee/route-decisions` 已由 qimingclaw adapter 接管。
|
||||
- 当前能力：入口请求会按上下文、ready PlanStep、调度、查询和控制动作生成 route decision，并写入 `digital_events.kind = route_decision` 与现有 event outbox；路由时间线优先从 qimingclaw 本地事件读取；低风险 route decision 已开始映射为明确的本地 governance command，并把 `created_command_id` 回填到 route decision；scheduler check 已接入 ready PlanStep 安全派发，首页立即执行后会触发一次派发 sweep 并展示派发结果；路由决策本地业务视图、首页摘要和通知联动已开始闭环。
|
||||
- 后续缺口：管理端正式路由 UI、派发策略下发、风险策略下发和高风险动作审批策略仍待吸收。
|
||||
- 后续缺口：管理端正式路由 UI、风险策略下发和高风险动作审批策略仍待吸收。
|
||||
- 建议：下一轮围绕管理端正式路由 UI 和策略下发，把入口路由从本地可观测推进到跨端治理策略。
|
||||
|
||||
11. **诊断 / 成本 / 审计视图**
|
||||
|
||||
Reference in New Issue
Block a user