Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions fleet-controller/src/__tests__/operation-queue-worker.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1115,11 +1115,14 @@ describe("pending operation watch", () => {
}
});

it("wakes one fenced archive at its persisted provider deadline after restart", async () => {
it.each([
{ action: "archive", createdBy: "glm-controller-return-home-archive", waitMs: 5_000 },
{ action: "release-home-backup", createdBy: "glm-controller-placement-policy", waitMs: 30_000 },
])("wakes one $action at its persisted provider deadline after restart", async ({ action, createdBy, waitMs }) => {
const id = "archive-owner";
const { db, store, emit } = createOperationsFixture({
[id]: baseOp({ id, kind: "gateway-vm", origin: "automatic", createdBy: "glm-controller-return-home-archive",
input: { action: "archive", uid: "alice" }, nextAttemptAt: 6_000, waitingOn: "home-archive-provider",
[id]: baseOp({ id, kind: "gateway-vm", origin: "automatic", createdBy,
input: { action, uid: "alice" }, nextAttemptAt: 1_000 + waitMs, waitingOn: "home-archive-provider",
output: { homeArchiveProviderDeadlineAt: 1_201_000 } }),
});
const handler = vi.fn<OperationHandler>().mockResolvedValue({ success: true, steps: [] });
Expand All @@ -1131,7 +1134,7 @@ describe("pending operation watch", () => {
await emit({ kind: "snapshot", records: [pending], cursor: "c1" as never });
await emit({ kind: "resync", reason: "backend-restart" });
await emit({ kind: "snapshot", records: [pending], cursor: "c2" as never });
await vi.advanceTimersByTimeAsync(4_999);
await vi.advanceTimersByTimeAsync(waitMs - 1);
expect(handler).not.toHaveBeenCalled();
await vi.advanceTimersByTimeAsync(1);
expect(handler).toHaveBeenCalledOnce();
Expand Down
16 changes: 16 additions & 0 deletions fleet-controller/src/__tests__/operation-retry.spec.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { RELEASE_RESTORED_HOME_BACKUP } from "@octogee/core/isolated-gateway-lifecycle";
import type { Operation } from "@octogee/shared/types/glm";
import { describe, expect, it } from "vitest";
import {
Expand Down Expand Up @@ -51,6 +52,21 @@ describe("parked home archive provider continuation", () => {
});
});

describe("restored home backup provider continuation", () => {
it("rechecks accepted provider erasure in 30s without consuming an attempt or losing custody", () => {
const result = { success: false, steps: [], error: "Restored home backup erasure is still converging",
waitingForHomeArchiveProvider: true };
const op = { kind: "gateway-vm", origin: "automatic", createdBy: "glm-controller-placement-policy",
input: { action: RELEASE_RESTORED_HOME_BACKUP }, attempts: MAX_OPERATION_ATTEMPTS };
const pending = buildOperationCompletionUpdate({ op, result, steps: [], now: 7_000 });
expect(pending).toMatchObject({ status: "pending", nextAttemptAt: 37_000, waitingOn: "home-archive-provider" });
expect(pending).not.toHaveProperty("attempts");
expect(pending).not.toHaveProperty("input");
expect(buildOperationCompletionUpdate({ op: { ...op, createdBy: "other-actor" }, result, steps: [], now: 7_000 }))
.toMatchObject({ status: "failed" });
});
});

describe("PostgreSQL source recovery health retries", () => {
const error = "source management health unconfirmed; outcome remains fenced";
it("retains the owned source recovery beyond the attempt cap only for its exact unavailable health result", () => {
Expand Down
10 changes: 6 additions & 4 deletions fleet-controller/src/handlers/home-backup-release.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -81,18 +81,20 @@ describe("release the restored Builder backup", () => {
await expect(h.run()).resolves.toMatchObject({ success: false });
expect(h.provider.eraseHomeSnapshot).not.toHaveBeenCalled();
runtime.vmBootRecovery = ready;
h.provider.eraseHomeSnapshot.mockRejectedValueOnce(new Error("HTTP 403")).mockResolvedValueOnce(false);
h.provider.eraseHomeSnapshot.mockRejectedValueOnce(new Error("HTTP 403"))
.mockResolvedValueOnce(false).mockResolvedValueOnce(false);
await expect(h.run()).resolves.toMatchObject({ success: false, retryAfterMs: expect.any(Number) });
for (let count = 0; count < 2; count += 1) {
await expect(h.run()).resolves.toMatchObject({ success: false, retryAfterMs: expect.any(Number) });
await expect(h.run()).resolves.toMatchObject({ success: false, waitingForHomeArchiveProvider: true });
expect((await h.store.users.records.get(uid))?.home.homeArchive).toEqual({ ...archive, releaseProvenForActivation: activationId });
delete runtime.vmBootRecovery;
}
await expect(h.run()).resolves.toMatchObject({ success: true });
expect((await h.store.users.records.get(uid))?.home).toMatchObject({ homeDisk: restoredDisk });
expect((await h.store.users.records.get(uid))?.home.homeArchive).toBeUndefined();
expect(h.provider.eraseHomeSnapshot).toHaveBeenNthCalledWith(3, archive.snapshot, uid);
expect(h.provider.eraseHomeSnapshot).toHaveBeenNthCalledWith(4, archive.snapshot, uid);
await expect(h.run()).resolves.toMatchObject({ success: true });
expect(h.provider.eraseHomeSnapshot).toHaveBeenCalledTimes(3);
expect(h.provider.eraseHomeSnapshot).toHaveBeenCalledTimes(4);
});
});

Expand Down
5 changes: 4 additions & 1 deletion fleet-controller/src/handlers/home-backup-release.ts
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,10 @@ export function createHomeBackupReleaseHandler(deps: {
await ctx.assertActiveClaim();
try {
if (!await deps.vm.eraseHomeSnapshot(proof.archive.snapshot, id.uid)) {
return retry("Restored home backup erasure is still converging");
return {
steps: [], success: false, waitingForHomeArchiveProvider: true,
error: "Restored home backup erasure is still converging",
};
}
} catch (error) {
if (transientSnapshotError(error)) return retry("Restored home backup erasure temporarily failed; snapshot custody retained");
Expand Down
36 changes: 29 additions & 7 deletions fleet-controller/src/operation-retry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -256,17 +256,24 @@ function recordOutput(value: unknown): boolean {
}

const HOME_ARCHIVE_PROVIDER_WAIT_MS = 5_000;
const HOME_BACKUP_PURGE_WAIT_MS = 30_000;
const HOME_ARCHIVE_PROVIDER_BUDGET_MS = 20 * 60_000;
const WAITING_FOR_HOME_ARCHIVE_PROVIDER = "home-archive-provider";

function isHomeArchiveOperation(op: OperationCompletionInput["op"]): boolean {
return op.kind === "gateway-vm" && op.input?.action === "archive" && isAuthorizedHomeArchiveOrigin(op);
}

function isHomeBackupProviderOperation(op: OperationCompletionInput["op"]): boolean {
return op.kind === "gateway-vm" && op.input?.action === RELEASE_RESTORED_HOME_BACKUP &&
op.origin === "automatic" && op.createdBy === "glm-controller-placement-policy";
}

export function homeArchiveProviderDueAt(op: OperationCompletionInput["op"] & {
status?: string; waitingOn?: string | null; nextAttemptAt?: number | null;
}): number | undefined {
return isHomeArchiveOperation(op) && op.status === "pending" && op.waitingOn === WAITING_FOR_HOME_ARCHIVE_PROVIDER
return (isHomeArchiveOperation(op) || isHomeBackupProviderOperation(op)) &&
op.status === "pending" && op.waitingOn === WAITING_FOR_HOME_ARCHIVE_PROVIDER
? op.nextAttemptAt ?? undefined : undefined;
}

Expand All @@ -279,20 +286,36 @@ function homeArchiveProviderCompletion(input: OperationCompletionInput, steps: O
const expired = { ...input, result: { success: false, steps, error: `Home archive provider wait timed out: ${error}` } };
return failedCompletion(expired, steps, attempts, failureCompletionContext(expired, steps, attempts));
}
return pendingHomeProvider(input, steps, Math.min(input.now + HOME_ARCHIVE_PROVIDER_WAIT_MS, deadline), {
homeArchiveProviderDeadlineAt: deadline,
});
}

function pendingHomeProvider(input: OperationCompletionInput, steps: OperationStep[], nextAttemptAt: number,
output: Record<string, unknown>): OperationPatch {
return {
status: "pending",
startedAt: null,
completedAt: null,
expireAt: CLEAR,
steps,
nextAttemptAt: Math.min(input.now + HOME_ARCHIVE_PROVIDER_WAIT_MS, deadline),
nextAttemptAt,
lastProgressAt: input.now,
waitingOn: WAITING_FOR_HOME_ARCHIVE_PROVIDER,
error,
output: { homeArchiveProviderDeadlineAt: deadline },
error: input.result.error ?? "Provider has not completed the archive transition",
output,
};
}

function homeProviderContinuation(input: OperationCompletionInput, steps: OperationStep[], attempts: number): OperationPatch | undefined {
if (input.result.waitingForHomeArchiveProvider !== true) return undefined;
if (isHomeArchiveOperation(input.op)) return homeArchiveProviderCompletion(input, steps, attempts);
if (isHomeBackupProviderOperation(input.op)) {
return pendingHomeProvider(input, steps, input.now + HOME_BACKUP_PURGE_WAIT_MS, { homeBackupReleasePending: true });
}
return undefined;
}

function isHomeBackupReleasePending({ op, result }: OperationCompletionInput): boolean {
return op.kind === "gateway-vm" && op.input?.action === RELEASE_RESTORED_HOME_BACKUP && result.output?.homeBackupReleasePending === true;
}
Expand Down Expand Up @@ -521,9 +544,8 @@ export function buildOperationCompletionUpdate(
};
}
if (result.success) return successfulCompletion(input, persistedSteps, attempts);
if (result.waitingForHomeArchiveProvider === true && isHomeArchiveOperation(op)) {
return homeArchiveProviderCompletion(input, persistedSteps, attempts);
}
const continuation = homeProviderContinuation(input, persistedSteps, attempts);
if (continuation) return continuation;

const failure = failureCompletionContext(input, persistedSteps, attempts);
if (result.failedAfterCutover === true) return failedAfterCutoverCompletion(input, persistedSteps, attempts, failure);
Expand Down