Skip to content

Commit b7a54b6

Browse files
iamjr15iamjr15
committed
fix(sandbox): harden snapshot publication
1 parent 3336016 commit b7a54b6

3 files changed

Lines changed: 246 additions & 42 deletions

File tree

‎.github/workflows/build-snapshot.yml‎

Lines changed: 234 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
name: Build Sandbox Snapshot
22

3-
# Snapshot publication is an explicit production-adjacent operation. Each main
4-
# commit maps to one immutable candidate; promotion happens by reviewing the
5-
# DAYTONA_SANDBOX_SNAPSHOT change in the agent-worker configuration.
3+
# Snapshot publication is an explicit production-adjacent operation. Each
4+
# dispatch maps its main commit and workflow run to one immutable candidate;
5+
# promotion happens by reviewing the DAYTONA_SANDBOX_SNAPSHOT change in the
6+
# agent-worker configuration.
67
on:
78
workflow_dispatch:
89
inputs:
@@ -66,7 +67,7 @@ jobs:
6667
- name: Derive immutable snapshot name
6768
id: snapshot
6869
run: |
69-
snapshot_name="cheatcode-sandbox-viewer-bundle-$(git rev-parse --short=12 HEAD)"
70+
snapshot_name="cheatcode-sandbox-viewer-bundle-$(git rev-parse --short=12 HEAD)-${GITHUB_RUN_ID}"
7071
echo "name=$snapshot_name" >> "$GITHUB_OUTPUT"
7172
echo "SNAPSHOT_NAME=$snapshot_name" >> "$GITHUB_ENV"
7273
- name: Install checksum-verified Daytona CLI
@@ -86,21 +87,6 @@ jobs:
8687
test -n "$DAYTONA_API_KEY"
8788
"$RUNNER_TEMP/bin/daytona" version
8889
"$RUNNER_TEMP/bin/daytona" login --api-key "$DAYTONA_API_KEY"
89-
- name: Refuse snapshot-name reuse
90-
run: |
91-
page=1
92-
while true; do
93-
snapshot_list="$(daytona snapshot list --format json --limit 200 --page "$page")"
94-
if jq --exit-status --arg name "$SNAPSHOT_NAME" \
95-
'.[] | select(.name == $name)' <<< "$snapshot_list" > /dev/null; then
96-
echo "Snapshot $SNAPSHOT_NAME already exists; immutable names are never overwritten." >&2
97-
exit 1
98-
fi
99-
if [ "$(jq 'length' <<< "$snapshot_list")" -lt 200 ]; then
100-
break
101-
fi
102-
page=$((page + 1))
103-
done
10490
- name: Build immutable image candidate
10591
run: |
10692
image_tag="cheatcode-sandbox:$GITHUB_SHA"
@@ -145,26 +131,237 @@ jobs:
145131
test -f /opt/cheatcode-browser-driver/server.js
146132
node --check /opt/cheatcode-browser-driver/server.js
147133
'
148-
- name: Publish immutable snapshot candidate
149-
run: |
150-
daytona snapshot push "$IMAGE_TAG" \
151-
--name "$SNAPSHOT_NAME" \
152-
--cpu 2 --memory 4 --disk 10
153-
- name: Verify published candidate
134+
- name: Publish and verify immutable snapshot candidate
154135
run: |
155-
page=1
156-
while true; do
157-
snapshot_list="$(daytona snapshot list --format json --limit 200 --page "$page")"
158-
if jq --exit-status --arg name "$SNAPSHOT_NAME" \
159-
'.[] | select(.name == $name and .state == "active")' \
160-
<<< "$snapshot_list" > /dev/null; then
161-
exit 0
136+
set -Eeuo pipefail
137+
shopt -s inherit_errexit
138+
139+
readonly MAX_PUSH_ATTEMPTS=3
140+
readonly STATE_POLL_ATTEMPTS=36
141+
readonly REMOVE_POLL_ATTEMPTS=30
142+
readonly MAX_LIST_PAGES=100
143+
readonly EXPECTED_CPU=2
144+
readonly EXPECTED_MEMORY=4
145+
readonly EXPECTED_DISK=10
146+
readonly EXPECTED_REGION=us
147+
readonly TIMEOUT_REASON_REGEX='^Job [0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12} timed out after 30000ms$'
148+
readonly UUID_REGEX='^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}$'
149+
readonly ISO_UTC_REGEX='^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?Z$'
150+
151+
snapshot_by_name() {
152+
local accumulated='[]'
153+
local accumulated_length
154+
local page
155+
local page_length
156+
local snapshot_list
157+
158+
for ((page = 1; page <= MAX_LIST_PAGES; page += 1)); do
159+
snapshot_list="$(daytona snapshot list --format json --limit 200 --page "$page")" || return 1
160+
jq --exit-status 'type == "array"' <<< "$snapshot_list" > /dev/null || return 1
161+
accumulated="$(jq --compact-output \
162+
--arg name "$SNAPSHOT_NAME" \
163+
--argjson accumulated "$accumulated" \
164+
'$accumulated + [.[] | select(.name == $name)]' \
165+
<<< "$snapshot_list")" || return 1
166+
page_length="$(jq --exit-status --raw-output 'length' <<< "$snapshot_list")" || return 1
167+
if [ "$page_length" -lt 200 ]; then
168+
accumulated_length="$(jq --exit-status --raw-output 'length' <<< "$accumulated")" || return 1
169+
if [ "$accumulated_length" -gt 1 ]; then
170+
echo "Multiple snapshots share immutable name $SNAPSHOT_NAME." >&2
171+
return 1
172+
fi
173+
jq --compact-output '.[0] // null' <<< "$accumulated" || return 1
174+
return 0
175+
fi
176+
done
177+
178+
echo "Snapshot pagination exceeded $MAX_LIST_PAGES pages." >&2
179+
return 1
180+
}
181+
182+
is_attempt_snapshot() {
183+
local snapshot_json="$1"
184+
jq --exit-status \
185+
--arg name "$SNAPSHOT_NAME" \
186+
--arg created_after "$attempt_started_at" \
187+
--arg ref_suffix "$expected_ref_suffix" \
188+
--arg region "$EXPECTED_REGION" \
189+
--arg uuid_regex "$UUID_REGEX" \
190+
--arg iso_utc_regex "$ISO_UTC_REGEX" \
191+
--argjson cpu "$EXPECTED_CPU" \
192+
--argjson memory "$EXPECTED_MEMORY" \
193+
--argjson disk "$EXPECTED_DISK" \
194+
'try (
195+
type == "object" and
196+
has("id") and ((.id | type) == "string") and (.id | test($uuid_regex)) and
197+
has("name") and ((.name | type) == "string") and .name == $name and
198+
has("createdAt") and ((.createdAt | type) == "string") and
199+
(.createdAt | test($iso_utc_regex)) and
200+
.createdAt >= $created_after and
201+
has("ref") and ((.ref | type) == "string") and (.ref | endswith($ref_suffix)) and
202+
has("cpu") and .cpu == $cpu and
203+
has("mem") and .mem == $memory and
204+
has("disk") and .disk == $disk and
205+
has("regionIds") and ((.regionIds | type) == "array") and
206+
.regionIds == [$region] and
207+
has("state") and ((.state | type) == "string")
208+
) catch false' \
209+
<<< "$snapshot_json" > /dev/null
210+
}
211+
212+
is_retryable_timeout() {
213+
local snapshot_json="$1"
214+
jq --exit-status \
215+
--arg reason "$TIMEOUT_REASON_REGEX" \
216+
'try (
217+
type == "object" and
218+
has("state") and .state == "error" and
219+
has("lastUsedAt") and .lastUsedAt == null and
220+
has("errorReason") and ((.errorReason | type) == "string") and
221+
(.errorReason | test($reason))
222+
) catch false' \
223+
<<< "$snapshot_json" > /dev/null
224+
}
225+
226+
for ((attempt = 1; attempt <= MAX_PUSH_ATTEMPTS; attempt += 1)); do
227+
existing="$(snapshot_by_name)" || {
228+
echo "Could not verify candidate-name availability." >&2
229+
exit 1
230+
}
231+
if [ "$existing" != "null" ]; then
232+
echo "Snapshot $SNAPSHOT_NAME already exists; immutable names are never overwritten." >&2
233+
exit 1
162234
fi
163-
if [ "$(jq 'length' <<< "$snapshot_list")" -lt 200 ]; then
164-
echo "Published snapshot $SNAPSHOT_NAME is not active." >&2
235+
236+
push_status=0
237+
push_log="$RUNNER_TEMP/daytona-push-$attempt.log"
238+
attempt_started_at="$(date --utc '+%Y-%m-%dT%H:%M:%S.000Z')"
239+
daytona snapshot push "$IMAGE_TAG" \
240+
--name "$SNAPSHOT_NAME" \
241+
--cpu "$EXPECTED_CPU" \
242+
--memory "$EXPECTED_MEMORY" \
243+
--disk "$EXPECTED_DISK" \
244+
--region "$EXPECTED_REGION" \
245+
> "$push_log" 2>&1 || push_status=$?
246+
cat "$push_log"
247+
248+
registry_digest="$(sed -nE \
249+
's/.*digest: (sha256:[0-9a-f]{64}) size: [0-9]+.*/\1/p' \
250+
"$push_log" | tail -n 1)"
251+
if [[ ! "$registry_digest" =~ ^sha256:[0-9a-f]{64}$ ]]; then
252+
echo "Daytona push did not expose a verifiable registry digest." >&2
253+
exit 1
254+
fi
255+
expected_ref_suffix="/daytona-${registry_digest#sha256:}:daytona"
256+
257+
retry_candidate=''
258+
for ((poll = 1; poll <= STATE_POLL_ATTEMPTS; poll += 1)); do
259+
snapshot_json="$(snapshot_by_name)" || {
260+
echo "Could not inspect the published snapshot candidate." >&2
261+
exit 1
262+
}
263+
if [ "$snapshot_json" = "null" ]; then
264+
state="absent"
265+
else
266+
state="$(jq --exit-status --raw-output \
267+
'if type == "object" and has("state") and ((.state | type) == "string")
268+
then .state else error("invalid snapshot state") end' \
269+
<<< "$snapshot_json")" || {
270+
echo "Published snapshot returned an invalid state payload." >&2
271+
exit 1
272+
}
273+
fi
274+
275+
case "$state" in
276+
active)
277+
if ! is_attempt_snapshot "$snapshot_json"; then
278+
echo "Active snapshot provenance does not match this publish attempt." >&2
279+
exit 1
280+
fi
281+
echo "Published snapshot $SNAPSHOT_NAME is active."
282+
echo "SNAPSHOT_RETRIES=$((attempt - 1))" >> "$GITHUB_ENV"
283+
exit 0
284+
;;
285+
absent|building|pending|pulling)
286+
;;
287+
error)
288+
if ! is_attempt_snapshot "$snapshot_json"; then
289+
echo "Failed snapshot provenance does not match this publish attempt." >&2
290+
exit 1
291+
fi
292+
if is_retryable_timeout "$snapshot_json"; then
293+
retry_candidate="$snapshot_json"
294+
break
295+
fi
296+
echo "Snapshot publication entered a non-retryable error state." >&2
297+
jq '{id, name, state, errorReason, lastUsedAt}' <<< "$snapshot_json" >&2
298+
exit 1
299+
;;
300+
*)
301+
echo "Snapshot publication entered unexpected state: $state" >&2
302+
exit 1
303+
;;
304+
esac
305+
306+
if [ "$poll" -lt "$STATE_POLL_ATTEMPTS" ]; then
307+
sleep 5
308+
fi
309+
done
310+
311+
if [ -z "$retry_candidate" ]; then
312+
echo "Snapshot $SNAPSHOT_NAME did not become active; push exit status was $push_status." >&2
313+
exit 1
314+
fi
315+
if [ "$attempt" -eq "$MAX_PUSH_ATTEMPTS" ]; then
316+
echo "Snapshot processing timed out without a safe retry remaining." >&2
165317
exit 1
166318
fi
167-
page=$((page + 1))
319+
320+
retry_id="$(jq --exit-status --raw-output '.id' <<< "$retry_candidate")" || exit 1
321+
confirmed="$(snapshot_by_name)" || {
322+
echo "Could not revalidate the retry candidate." >&2
323+
exit 1
324+
}
325+
confirmed_id="$(jq --exit-status --raw-output \
326+
'if type == "object" and has("id") and ((.id | type) == "string")
327+
then .id else error("invalid snapshot id") end' \
328+
<<< "$confirmed")" || exit 1
329+
if [ "$confirmed_id" != "$retry_id" ] || \
330+
! is_attempt_snapshot "$confirmed" || \
331+
! is_retryable_timeout "$confirmed"; then
332+
echo "Retry candidate changed before cleanup; refusing deletion." >&2
333+
exit 1
334+
fi
335+
336+
daytona snapshot delete "$retry_id" || exit 1
337+
removed=false
338+
for ((poll = 1; poll <= REMOVE_POLL_ATTEMPTS; poll += 1)); do
339+
current="$(snapshot_by_name)" || {
340+
echo "Could not verify retry-candidate cleanup." >&2
341+
exit 1
342+
}
343+
if [ "$current" = "null" ]; then
344+
removed=true
345+
break
346+
fi
347+
current_id="$(jq --exit-status --raw-output \
348+
'if type == "object" and has("id") and ((.id | type) == "string")
349+
then .id else error("invalid snapshot id") end' \
350+
<<< "$current")" || exit 1
351+
if [ "$current_id" != "$retry_id" ]; then
352+
echo "Snapshot identity changed during retry cleanup." >&2
353+
exit 1
354+
fi
355+
if [ "$poll" -lt "$REMOVE_POLL_ATTEMPTS" ]; then
356+
sleep 2
357+
fi
358+
done
359+
if [ "$removed" != "true" ]; then
360+
echo "Timed-out snapshot candidate was not removed within the cleanup window." >&2
361+
exit 1
362+
fi
363+
364+
echo "Retrying transient Daytona processing timeout ($attempt/$MAX_PUSH_ATTEMPTS); push exited $push_status."
168365
done
169366
- name: Record promotion instructions
170367
env:
@@ -174,6 +371,7 @@ jobs:
174371
echo "### Daytona snapshot candidate"
175372
echo "- Candidate: \`$SNAPSHOT_NAME\`"
176373
echo "- Source commit: \`$GITHUB_SHA\`"
177-
echo "- No existing snapshot was deleted or replaced."
374+
echo "- Transient provider retries: \`$SNAPSHOT_RETRIES\`"
375+
echo "- Pre-existing, active, or previously used snapshots were not deleted or replaced."
178376
echo "- Promote through review by updating \`DAYTONA_SANDBOX_SNAPSHOT\` in \`apps/agent-worker/wrangler.jsonc\`."
179377
} >> "$GITHUB_STEP_SUMMARY"

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,7 @@ Publish a new immutable Daytona snapshot after changing `infra/containers/sandbo
148148

149149
```bash
150150
docker build --platform=linux/amd64 -t cheatcode-sandbox:<immutable-tag> infra/containers/sandbox
151-
daytona snapshot push cheatcode-sandbox:<immutable-tag> --name <snapshot-name> --cpu 2 --memory 4 --disk 10
151+
daytona snapshot push cheatcode-sandbox:<immutable-tag> --name <unique-snapshot-name> --cpu 2 --memory 4 --disk 10 --region us
152152
```
153153

154154
Cloudflare Secrets Store sync is dry-run by default, never prints secret values, creates missing

‎infra/containers/sandbox/README.md‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,13 @@ runtime user matches `/workspace` + the baked Next/Expo templates under `/home/n
1313

1414
The normal path is the protected **Build Sandbox Snapshot** GitHub workflow. Dispatch
1515
it from `main` and enter `BUILD_SNAPSHOT`. It publishes an immutable candidate named
16-
`cheatcode-sandbox-viewer-bundle-<12-character-commit-sha>`. The workflow never
17-
deletes or replaces an existing snapshot and refuses to reuse a candidate name.
16+
`cheatcode-sandbox-viewer-bundle-<12-character-commit-sha>-<workflow-run-id>` and
17+
refuses any pre-existing or surviving candidate name. If Daytona returns its exact
18+
transient 30-second processing timeout, the workflow may remove only the unused failed
19+
candidate created by that same publish attempt before retrying the run-scoped name.
20+
Candidate ID, creation time, OCI digest, region, resources, error shape, and
21+
`lastUsedAt` are revalidated immediately before deletion. Active, previously used,
22+
pre-existing, or ambiguous snapshots fail closed and are never deleted or replaced.
1823

1924
Promotion is a separate reviewed source change: update the agent-worker
2025
`DAYTONA_SANDBOX_SNAPSHOT` var to the candidate name, then use the protected database
@@ -33,12 +38,13 @@ docker build --platform=linux/amd64 -t cheatcode-sandbox:<immutable-tag> \
3338
# Push the local image straight into Daytona's registry (no external registry needed)
3439
# and register it as a snapshot with baked resources (≤ Tier-2 caps: 4 vCPU / 8 GiB / 10 GiB).
3540
daytona snapshot push cheatcode-sandbox:<immutable-tag> \
36-
--name cheatcode-sandbox-viewer-bundle-<commit-sha> \
37-
--cpu 2 --memory 4 --disk 10
41+
--name cheatcode-sandbox-viewer-bundle-<commit-sha>-<unique-run-id> \
42+
--cpu 2 --memory 4 --disk 10 --region us
3843
```
3944

4045
Then set the agent-worker `DAYTONA_SANDBOX_SNAPSHOT` var to the new snapshot name.
41-
The current default is `cheatcode-sandbox-viewer-bundle-20260714-1220z`.
46+
The authoritative current default is committed in
47+
[`apps/agent-worker/wrangler.jsonc`](../../../apps/agent-worker/wrangler.jsonc).
4248

4349
> Use an **immutable tag**, not `:latest` (rejected) and not a digest (digest pinning is
4450
> currently broken for Daytona pushed-image references). The Dockerfile base image is

0 commit comments

Comments
 (0)