Skip to content

Commit 465a020

Browse files
authored
Merge pull request #382 from octogee-ai/codex/continue-restored-backup-purge
Continue verified home backup purges on their provider deadline
2 parents eb8e932 + 090d68a commit 465a020

5 files changed

Lines changed: 62 additions & 16 deletions

File tree

‎fleet-controller/src/__tests__/operation-queue-worker.spec.ts‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1115,11 +1115,14 @@ describe("pending operation watch", () => {
11151115
}
11161116
});
11171117

1118-
it("wakes one fenced archive at its persisted provider deadline after restart", async () => {
1118+
it.each([
1119+
{ action: "archive", createdBy: "glm-controller-return-home-archive", waitMs: 5_000 },
1120+
{ action: "release-home-backup", createdBy: "glm-controller-placement-policy", waitMs: 30_000 },
1121+
])("wakes one $action at its persisted provider deadline after restart", async ({ action, createdBy, waitMs }) => {
11191122
const id = "archive-owner";
11201123
const { db, store, emit } = createOperationsFixture({
1121-
[id]: baseOp({ id, kind: "gateway-vm", origin: "automatic", createdBy: "glm-controller-return-home-archive",
1122-
input: { action: "archive", uid: "alice" }, nextAttemptAt: 6_000, waitingOn: "home-archive-provider",
1124+
[id]: baseOp({ id, kind: "gateway-vm", origin: "automatic", createdBy,
1125+
input: { action, uid: "alice" }, nextAttemptAt: 1_000 + waitMs, waitingOn: "home-archive-provider",
11231126
output: { homeArchiveProviderDeadlineAt: 1_201_000 } }),
11241127
});
11251128
const handler = vi.fn<OperationHandler>().mockResolvedValue({ success: true, steps: [] });
@@ -1131,7 +1134,7 @@ describe("pending operation watch", () => {
11311134
await emit({ kind: "snapshot", records: [pending], cursor: "c1" as never });
11321135
await emit({ kind: "resync", reason: "backend-restart" });
11331136
await emit({ kind: "snapshot", records: [pending], cursor: "c2" as never });
1134-
await vi.advanceTimersByTimeAsync(4_999);
1137+
await vi.advanceTimersByTimeAsync(waitMs - 1);
11351138
expect(handler).not.toHaveBeenCalled();
11361139
await vi.advanceTimersByTimeAsync(1);
11371140
expect(handler).toHaveBeenCalledOnce();

‎fleet-controller/src/__tests__/operation-retry.spec.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import { RELEASE_RESTORED_HOME_BACKUP } from "@octogee/core/isolated-gateway-lifecycle";
12
import type { Operation } from "@octogee/shared/types/glm";
23
import { describe, expect, it } from "vitest";
34
import {
@@ -51,6 +52,21 @@ describe("parked home archive provider continuation", () => {
5152
});
5253
});
5354

55+
describe("restored home backup provider continuation", () => {
56+
it("rechecks accepted provider erasure in 30s without consuming an attempt or losing custody", () => {
57+
const result = { success: false, steps: [], error: "Restored home backup erasure is still converging",
58+
waitingForHomeArchiveProvider: true };
59+
const op = { kind: "gateway-vm", origin: "automatic", createdBy: "glm-controller-placement-policy",
60+
input: { action: RELEASE_RESTORED_HOME_BACKUP }, attempts: MAX_OPERATION_ATTEMPTS };
61+
const pending = buildOperationCompletionUpdate({ op, result, steps: [], now: 7_000 });
62+
expect(pending).toMatchObject({ status: "pending", nextAttemptAt: 37_000, waitingOn: "home-archive-provider" });
63+
expect(pending).not.toHaveProperty("attempts");
64+
expect(pending).not.toHaveProperty("input");
65+
expect(buildOperationCompletionUpdate({ op: { ...op, createdBy: "other-actor" }, result, steps: [], now: 7_000 }))
66+
.toMatchObject({ status: "failed" });
67+
});
68+
});
69+
5470
describe("PostgreSQL source recovery health retries", () => {
5571
const error = "source management health unconfirmed; outcome remains fenced";
5672
it("retains the owned source recovery beyond the attempt cap only for its exact unavailable health result", () => {

‎fleet-controller/src/handlers/home-backup-release.spec.ts‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -81,18 +81,20 @@ describe("release the restored Builder backup", () => {
8181
await expect(h.run()).resolves.toMatchObject({ success: false });
8282
expect(h.provider.eraseHomeSnapshot).not.toHaveBeenCalled();
8383
runtime.vmBootRecovery = ready;
84-
h.provider.eraseHomeSnapshot.mockRejectedValueOnce(new Error("HTTP 403")).mockResolvedValueOnce(false);
84+
h.provider.eraseHomeSnapshot.mockRejectedValueOnce(new Error("HTTP 403"))
85+
.mockResolvedValueOnce(false).mockResolvedValueOnce(false);
86+
await expect(h.run()).resolves.toMatchObject({ success: false, retryAfterMs: expect.any(Number) });
8587
for (let count = 0; count < 2; count += 1) {
86-
await expect(h.run()).resolves.toMatchObject({ success: false, retryAfterMs: expect.any(Number) });
88+
await expect(h.run()).resolves.toMatchObject({ success: false, waitingForHomeArchiveProvider: true });
8789
expect((await h.store.users.records.get(uid))?.home.homeArchive).toEqual({ ...archive, releaseProvenForActivation: activationId });
8890
delete runtime.vmBootRecovery;
8991
}
9092
await expect(h.run()).resolves.toMatchObject({ success: true });
9193
expect((await h.store.users.records.get(uid))?.home).toMatchObject({ homeDisk: restoredDisk });
9294
expect((await h.store.users.records.get(uid))?.home.homeArchive).toBeUndefined();
93-
expect(h.provider.eraseHomeSnapshot).toHaveBeenNthCalledWith(3, archive.snapshot, uid);
95+
expect(h.provider.eraseHomeSnapshot).toHaveBeenNthCalledWith(4, archive.snapshot, uid);
9496
await expect(h.run()).resolves.toMatchObject({ success: true });
95-
expect(h.provider.eraseHomeSnapshot).toHaveBeenCalledTimes(3);
97+
expect(h.provider.eraseHomeSnapshot).toHaveBeenCalledTimes(4);
9698
});
9799
});
98100

‎fleet-controller/src/handlers/home-backup-release.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,10 @@ export function createHomeBackupReleaseHandler(deps: {
124124
await ctx.assertActiveClaim();
125125
try {
126126
if (!await deps.vm.eraseHomeSnapshot(proof.archive.snapshot, id.uid)) {
127-
return retry("Restored home backup erasure is still converging");
127+
return {
128+
steps: [], success: false, waitingForHomeArchiveProvider: true,
129+
error: "Restored home backup erasure is still converging",
130+
};
128131
}
129132
} catch (error) {
130133
if (transientSnapshotError(error)) return retry("Restored home backup erasure temporarily failed; snapshot custody retained");

‎fleet-controller/src/operation-retry.ts‎

Lines changed: 29 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -256,17 +256,24 @@ function recordOutput(value: unknown): boolean {
256256
}
257257

258258
const HOME_ARCHIVE_PROVIDER_WAIT_MS = 5_000;
259+
const HOME_BACKUP_PURGE_WAIT_MS = 30_000;
259260
const HOME_ARCHIVE_PROVIDER_BUDGET_MS = 20 * 60_000;
260261
const WAITING_FOR_HOME_ARCHIVE_PROVIDER = "home-archive-provider";
261262

262263
function isHomeArchiveOperation(op: OperationCompletionInput["op"]): boolean {
263264
return op.kind === "gateway-vm" && op.input?.action === "archive" && isAuthorizedHomeArchiveOrigin(op);
264265
}
265266

267+
function isHomeBackupProviderOperation(op: OperationCompletionInput["op"]): boolean {
268+
return op.kind === "gateway-vm" && op.input?.action === RELEASE_RESTORED_HOME_BACKUP &&
269+
op.origin === "automatic" && op.createdBy === "glm-controller-placement-policy";
270+
}
271+
266272
export function homeArchiveProviderDueAt(op: OperationCompletionInput["op"] & {
267273
status?: string; waitingOn?: string | null; nextAttemptAt?: number | null;
268274
}): number | undefined {
269-
return isHomeArchiveOperation(op) && op.status === "pending" && op.waitingOn === WAITING_FOR_HOME_ARCHIVE_PROVIDER
275+
return (isHomeArchiveOperation(op) || isHomeBackupProviderOperation(op)) &&
276+
op.status === "pending" && op.waitingOn === WAITING_FOR_HOME_ARCHIVE_PROVIDER
270277
? op.nextAttemptAt ?? undefined : undefined;
271278
}
272279

@@ -279,20 +286,36 @@ function homeArchiveProviderCompletion(input: OperationCompletionInput, steps: O
279286
const expired = { ...input, result: { success: false, steps, error: `Home archive provider wait timed out: ${error}` } };
280287
return failedCompletion(expired, steps, attempts, failureCompletionContext(expired, steps, attempts));
281288
}
289+
return pendingHomeProvider(input, steps, Math.min(input.now + HOME_ARCHIVE_PROVIDER_WAIT_MS, deadline), {
290+
homeArchiveProviderDeadlineAt: deadline,
291+
});
292+
}
293+
294+
function pendingHomeProvider(input: OperationCompletionInput, steps: OperationStep[], nextAttemptAt: number,
295+
output: Record<string, unknown>): OperationPatch {
282296
return {
283297
status: "pending",
284298
startedAt: null,
285299
completedAt: null,
286300
expireAt: CLEAR,
287301
steps,
288-
nextAttemptAt: Math.min(input.now + HOME_ARCHIVE_PROVIDER_WAIT_MS, deadline),
302+
nextAttemptAt,
289303
lastProgressAt: input.now,
290304
waitingOn: WAITING_FOR_HOME_ARCHIVE_PROVIDER,
291-
error,
292-
output: { homeArchiveProviderDeadlineAt: deadline },
305+
error: input.result.error ?? "Provider has not completed the archive transition",
306+
output,
293307
};
294308
}
295309

310+
function homeProviderContinuation(input: OperationCompletionInput, steps: OperationStep[], attempts: number): OperationPatch | undefined {
311+
if (input.result.waitingForHomeArchiveProvider !== true) return undefined;
312+
if (isHomeArchiveOperation(input.op)) return homeArchiveProviderCompletion(input, steps, attempts);
313+
if (isHomeBackupProviderOperation(input.op)) {
314+
return pendingHomeProvider(input, steps, input.now + HOME_BACKUP_PURGE_WAIT_MS, { homeBackupReleasePending: true });
315+
}
316+
return undefined;
317+
}
318+
296319
function isHomeBackupReleasePending({ op, result }: OperationCompletionInput): boolean {
297320
return op.kind === "gateway-vm" && op.input?.action === RELEASE_RESTORED_HOME_BACKUP && result.output?.homeBackupReleasePending === true;
298321
}
@@ -521,9 +544,8 @@ export function buildOperationCompletionUpdate(
521544
};
522545
}
523546
if (result.success) return successfulCompletion(input, persistedSteps, attempts);
524-
if (result.waitingForHomeArchiveProvider === true && isHomeArchiveOperation(op)) {
525-
return homeArchiveProviderCompletion(input, persistedSteps, attempts);
526-
}
547+
const continuation = homeProviderContinuation(input, persistedSteps, attempts);
548+
if (continuation) return continuation;
527549

528550
const failure = failureCompletionContext(input, persistedSteps, attempts);
529551
if (result.failedAfterCutover === true) return failedAfterCutoverCompletion(input, persistedSteps, attempts, failure);

0 commit comments

Comments
 (0)