build-snapshot.yml

name: Build Sandbox Snapshot

# Snapshot publication is an explicit production-adjacent operation. Each
# dispatch maps its main commit and workflow run to one immutable candidate;
# promotion happens by reviewing the DAYTONA_SANDBOX_SNAPSHOT change in the
# agent-worker configuration.
on:
  workflow_dispatch:
    inputs:
      confirmation:
        description: Type BUILD_SNAPSHOT to publish an immutable candidate from main.
        required: true
        type: string

permissions:
  actions: read
  contents: read

concurrency:
  group: daytona-snapshot
  cancel-in-progress: false

jobs:
  build-snapshot:
    runs-on: ubuntu-24.04
    timeout-minutes: 60
    environment: Production
    steps:
      - name: Authorize snapshot publication
        env:
          CONFIRMATION: ${{ inputs.confirmation }}
        run: |
          if [ "$GITHUB_REF" != "refs/heads/main" ]; then
            echo "Sandbox snapshots must be dispatched from main." >&2
            exit 1
          fi
          if [ "$CONFIRMATION" != "BUILD_SNAPSHOT" ]; then
            echo "Confirmation must exactly match BUILD_SNAPSHOT." >&2
            exit 1
          fi
      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
        with:
          ref: ${{ github.sha }}
      - uses: aquasecurity/setup-trivy@81e514348e19b6112ce2a7e3ecbafe19c1e1f567 # v0.3.1
        with:
          cache: true
          version: v0.72.0
      - name: Require successful static checks for this commit
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          successful_runs="$(gh api --method GET \
            "repos/${GITHUB_REPOSITORY}/actions/workflows/static-checks.yml/runs" \
            --field branch=main \
            --field event=push \
            --field head_sha="$GITHUB_SHA" \
            --field per_page=1 \
            --field status=success \
            --jq '.total_count')"
          if [ "$successful_runs" -lt 1 ]; then
            echo "Static Checks has not succeeded for $GITHUB_SHA on main." >&2
            exit 1
          fi
      - name: Derive immutable snapshot name
        id: snapshot
        run: |
          source_commit="$(git rev-parse HEAD)"
          if [[ ! "$source_commit" =~ ^[0-9a-f]{40}$ ]] ||
            [[ ! "$GITHUB_RUN_ID" =~ ^[1-9][0-9]*$ ]]; then
            echo "Snapshot provenance did not resolve to canonical commit and workflow-run IDs." >&2
            exit 1
          fi
          snapshot_name="cheatcode-sandbox-viewer-bundle-${source_commit:0:12}-${GITHUB_RUN_ID}"
          echo "name=$snapshot_name" >> "$GITHUB_OUTPUT"
          echo "SNAPSHOT_NAME=$snapshot_name" >> "$GITHUB_ENV"
      - name: Set up checksum-pinned Daytona CLI
        uses: ./.github/actions/setup-daytona
        with:
          api-key: ${{ secrets.DAYTONA_API_KEY }}
      - name: Build immutable image candidate
        run: |
          image_tag="cheatcode-sandbox:$GITHUB_SHA"
          docker build \
            --platform=linux/amd64 \
            --build-context default_skills=skills \
            --tag "$image_tag" \
            infra/containers/sandbox
          echo "IMAGE_TAG=$image_tag" >> "$GITHUB_ENV"
      - name: Scan candidate configuration and image
        run: |
          trivy config \
            --severity MEDIUM,HIGH,CRITICAL \
            --exit-code 1 \
            infra/containers/sandbox
          trivy image \
            --quiet \
            --scanners vuln \
            --ignore-unfixed \
            --ignorefile infra/containers/sandbox/.trivyignore.yaml \
            --severity MEDIUM,HIGH,CRITICAL \
            --table-mode detailed \
            --exit-code 1 \
            "$IMAGE_TAG"
          trivy image \
            --quiet \
            --scanners secret \
            --severity HIGH,CRITICAL \
            --table-mode detailed \
            --exit-code 1 \
            "$IMAGE_TAG"
      - name: Smoke-test candidate runtime
        run: |
          docker run --rm --platform=linux/amd64 --entrypoint /bin/sh "$IMAGE_TAG" -lc '
            test "$(id -un)" = node
            test "$PWD" = /workspace
            test "$(node --version)" = v24.18.0
            libnss_version="$(dpkg-query --show libnss3 | cut --fields=2)"
            dpkg --compare-versions "$libnss_version" ge "2:3.87.1-1+deb12u3"
            code_server_dir="$(dirname "$(dirname "$(readlink -f "$(command -v code-server)")")")"
            test "$(node -p "require(process.argv[1]).version" /opt/cheatcode-doc-runtime/node_modules/brace-expansion/package.json)" = 5.0.9
            test "$(node -p "require(process.argv[1]).version" /opt/cheatcode-browser-driver/node_modules/fast-uri/package.json)" = 3.1.5
            test "$(node -p "require(process.argv[1]).version" /opt/cheatcode-browser-driver/node_modules/hono/package.json)" = 4.12.34
            test "$(node -p "require(process.argv[1]).version" /opt/cheatcode-browser-driver/node_modules/ip-address/package.json)" = 10.3.1
            test "$(node -p "require(process.argv[1]).version" /opt/cheatcode-package-manager/node_modules/npm/node_modules/brace-expansion/package.json)" = 5.0.9
            test "$(node -p "require(process.argv[1]).version" /opt/cheatcode-package-manager/node_modules/npm/node_modules/ip-address/package.json)" = 10.3.1
            test "$(node -p "require(process.argv[1]).version" /opt/cheatcode-package-manager/node_modules/npm/node_modules/undici/package.json)" = 6.28.0
            test "$(node -p "require(process.argv[1]).version" /opt/cheatcode-package-manager/node_modules/pnpm/artifacts/exe/dist/node_modules/undici/package.json)" = 6.28.0
            test "$(node -p "require(process.argv[1]).version" /opt/cheatcode-package-manager/node_modules/pnpm/dist/node_modules/undici/package.json)" = 6.28.0
            test "$(node -p "require(process.argv[1]).version" /opt/cheatcode-package-manager/node_modules/npm/node_modules/tar/package.json)" = 7.5.22
            test "$(node -p "require(process.argv[1]).version" /opt/cheatcode-package-manager/node_modules/pnpm/artifacts/exe/dist/node_modules/tar/package.json)" = 7.5.22
            test "$(node -p "require(process.argv[1]).version" /opt/cheatcode-package-manager/node_modules/pnpm/dist/node_modules/tar/package.json)" = 7.5.22
            test "$(node -p "require(process.argv[1]).version" "$code_server_dir/node_modules/ip-address/package.json")" = 10.3.1
            test "$(node -p "require(process.argv[1]).version" "$code_server_dir/node_modules/js-yaml/package.json")" = 4.3.1
            test "$(node -p "require(process.argv[1]).version" "$code_server_dir/lib/vscode/node_modules/ip-address/package.json")" = 10.3.1
            test "$(node -p "require(process.argv[1]).version" "$code_server_dir/lib/vscode/node_modules/shell-quote/package.json")" = 1.10.0
            test "$(node -p "require(process.argv[1]).version" "$code_server_dir/lib/vscode/node_modules/tar/package.json")" = 7.5.22
            test "$(node -p "require(process.argv[1]).version" "$code_server_dir/lib/vscode/node_modules/undici/package.json")" = 7.29.0
            test "$(node -p "require(process.argv[1]).version" /home/node/.cheatcode/app-runtimes/next/node_modules/.pnpm/nanoid@3.3.17/node_modules/nanoid/package.json)" = 3.3.17
            test "$(node -p "require(process.argv[1]).version" /home/node/.cheatcode/app-runtimes/next/node_modules/.pnpm/postcss@8.5.23/node_modules/postcss/package.json)" = 8.5.23
            test "$(node -p "require(process.argv[1]).version" /home/node/.cheatcode/app-runtimes/expo/node_modules/.pnpm/brace-expansion@5.0.9/node_modules/brace-expansion/package.json)" = 5.0.9
            test "$(node -p "require(process.argv[1]).version" /home/node/.cheatcode/app-runtimes/expo/node_modules/.pnpm/js-yaml@4.3.1/node_modules/js-yaml/package.json)" = 4.3.1
            test "$(node -p "require(process.argv[1]).version" /home/node/.cheatcode/app-runtimes/expo/node_modules/.pnpm/nanoid@3.3.17/node_modules/nanoid/package.json)" = 3.3.17
            test "$(node -p "require(process.argv[1]).version" /home/node/.cheatcode/app-runtimes/expo/node_modules/.pnpm/postcss@8.5.23/node_modules/postcss/package.json)" = 8.5.23
            npm --version
            pnpm --version
            python3 --version
            test -x /opt/cheatcode/project-source-sync.py
            python3 -c "import ast,pathlib;ast.parse(pathlib.Path(\"/opt/cheatcode/project-source-sync.py\").read_text())"
            code-server --version
            "$CHROME_PATH" --version
            libreoffice --version
            test -f /home/node/cheatcode-next-template/pnpm-lock.yaml
            test -f /home/node/cheatcode-expo-template/pnpm-lock.yaml
            test -f /home/node/cheatcode-next-template/next.config.ts
            test ! -e /home/node/cheatcode-next-template/node_modules
            test ! -e /home/node/cheatcode-expo-template/node_modules
            test -x /home/node/.cheatcode/app-runtimes/next/node_modules/.bin/next
            test -x /home/node/.cheatcode/app-runtimes/expo/node_modules/.bin/expo
            NODE_PATH=/home/node/.cheatcode/app-runtimes/next/node_modules node -e "require.resolve(\"next\");require.resolve(\"react\");require.resolve(\"react-dom\")"
            NODE_PATH=/home/node/.cheatcode/app-runtimes/expo/node_modules node -e "require.resolve(\"expo\");require.resolve(\"react-native-web\");require.resolve(\"@expo/metro-runtime\")"
            test -f /home/node/.cheatcode/default-skills/browser-use/SKILL.md
            test -f /home/node/.cheatcode/default-skills/pptx/SKILL.md
            test -f /home/node/.cheatcode/default-skills/pptx/scripts/thumbnail.py
            test -f /home/node/.cheatcode/default-skills/docx/scripts/office/pack.py
            test -f /home/node/.cheatcode/default-skills/docx/scripts/office/schemas/mce/mc.xsd
            test -f /home/node/.cheatcode/default-skills/xlsx/scripts/recalc.py
            test -f /home/node/.cheatcode/default-skills/webapp-testing/scripts/with_server.py
            test -f /home/node/.cheatcode/default-skills/manage-skills/_shared.ts
            test -f /home/node/.cheatcode/default-skills/skill-authoring/SKILL.md
            test -f /home/node/.cheatcode/default-skills/skill-authoring/persist/save.ts
            test -f /home/node/.cheatcode/default-skills/generate-media/SKILL.md
            test -f /home/node/.cheatcode/default-skills/product-self-knowledge/SKILL.md
            test "$(find /home/node/.cheatcode/default-skills -type f -name "*.py" | wc -l)" -eq 56
            test "$(find /home/node/.cheatcode/default-skills -type f -name "*.ts" | wc -l)" -eq 12
            test "$(find /home/node/.cheatcode/default-skills -type f -name "*.xsd" | wc -l)" -eq 117
            test ! -e /home/node/.cheatcode/default-skills/deploy
            test ! -e /home/node/.cheatcode/default-skills/landing-page
            test ! -e /home/node/.cheatcode/default-skills/slide-from-prd
            test ! -e /home/node/.cheatcode/default-skills/social-post-pack
            test ! -e /home/node/.cheatcode/default-skills/competitor-brief
            test -z "$(find /home/node/.cheatcode/default-skills -mindepth 1 -maxdepth 1 -type d -iname "*deploy*" -print -quit)"
            test -f /opt/cheatcode-browser-driver/server.js
            node --check /opt/cheatcode-browser-driver/server.js
            cheatcode-skills --help
            cheatcode-browser --help
            cheatcode-skills manage-skills/manage/list --help
            cheatcode-skills skill-authoring/persist/save --help
            python3 /home/node/.cheatcode/default-skills/pptx/scripts/thumbnail.py --help
            python3 /home/node/.cheatcode/default-skills/docx/scripts/office/validate.py --help
            python3 /home/node/.cheatcode/default-skills/xlsx/scripts/recalc.py --help
            python3 /home/node/.cheatcode/default-skills/webapp-testing/scripts/with_server.py --help
            python3 -c "import defusedxml, lxml, markitdown, pdf2image, pdfplumber, playwright, pypdf, pypdfium2, pytesseract, reportlab"
            python3 -c "import importlib.metadata as m; assert m.version(\"cryptography\") == \"50.0.0\"; assert m.version(\"pypdf\") == \"6.15.0\""
            pandoc --version
            pdfinfo -v
            qpdf --version
            tesseract --version
            test -z "$(grep -R -E "(LEGACY_SKILL_API_KEY|ORCHIDS_API_KEY)" /home/node/.cheatcode/default-skills /opt/cheatcode-skill-runtime || true)"
          '
      - name: Publish and verify immutable snapshot candidate
        run: |
          set -Eeuo pipefail
          shopt -s inherit_errexit

          readonly MAX_PUSH_ATTEMPTS=3
          readonly STATE_POLL_ATTEMPTS=36
          readonly REMOVE_POLL_ATTEMPTS=30
          readonly MAX_LIST_PAGES=100
          readonly EXPECTED_CPU=2
          readonly EXPECTED_MEMORY=4
          readonly EXPECTED_DISK=10
          readonly EXPECTED_REGION=us
          readonly TIMEOUT_REASON_REGEX='^Job [0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12} timed out after 30000ms$'
          readonly UUID_REGEX='^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}$'
          readonly ISO_UTC_REGEX='^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?Z$'

          snapshot_by_name() {
            local accumulated='[]'
            local accumulated_length
            local page
            local page_length
            local snapshot_list

            for ((page = 1; page <= MAX_LIST_PAGES; page += 1)); do
              snapshot_list="$(daytona snapshot list --format json --limit 200 --page "$page")" || return 1
              jq --exit-status 'type == "array"' <<< "$snapshot_list" > /dev/null || return 1
              accumulated="$(jq --compact-output \
                --arg name "$SNAPSHOT_NAME" \
                --argjson accumulated "$accumulated" \
                '$accumulated + [.[] | select(.name == $name)]' \
                <<< "$snapshot_list")" || return 1
              page_length="$(jq --exit-status --raw-output 'length' <<< "$snapshot_list")" || return 1
              if [ "$page_length" -lt 200 ]; then
                accumulated_length="$(jq --exit-status --raw-output 'length' <<< "$accumulated")" || return 1
                if [ "$accumulated_length" -gt 1 ]; then
                  echo "Multiple snapshots share immutable name $SNAPSHOT_NAME." >&2
                  return 1
                fi
                jq --compact-output '.[0] // null' <<< "$accumulated" || return 1
                return 0
              fi
            done

            echo "Snapshot pagination exceeded $MAX_LIST_PAGES pages." >&2
            return 1
          }

          is_attempt_snapshot() {
            local snapshot_json="$1"
            jq --exit-status \
              --arg name "$SNAPSHOT_NAME" \
              --arg created_after "$attempt_started_at" \
              --arg ref_suffix "$expected_ref_suffix" \
              --arg region "$EXPECTED_REGION" \
              --arg uuid_regex "$UUID_REGEX" \
              --arg iso_utc_regex "$ISO_UTC_REGEX" \
              --argjson cpu "$EXPECTED_CPU" \
              --argjson memory "$EXPECTED_MEMORY" \
              --argjson disk "$EXPECTED_DISK" \
              'try (
                type == "object" and
                has("id") and ((.id | type) == "string") and (.id | test($uuid_regex)) and
                has("name") and ((.name | type) == "string") and .name == $name and
                has("createdAt") and ((.createdAt | type) == "string") and
                (.createdAt | test($iso_utc_regex)) and
                .createdAt >= $created_after and
                has("ref") and ((.ref | type) == "string") and (.ref | endswith($ref_suffix)) and
                has("cpu") and .cpu == $cpu and
                has("mem") and .mem == $memory and
                has("disk") and .disk == $disk and
                has("regionIds") and ((.regionIds | type) == "array") and
                .regionIds == [$region] and
                has("state") and ((.state | type) == "string")
              ) catch false' \
              <<< "$snapshot_json" > /dev/null
          }

          is_retryable_timeout() {
            local snapshot_json="$1"
            jq --exit-status \
              --arg reason "$TIMEOUT_REASON_REGEX" \
              'try (
                type == "object" and
                has("state") and .state == "error" and
                has("lastUsedAt") and .lastUsedAt == null and
                has("errorReason") and ((.errorReason | type) == "string") and
                (.errorReason | test($reason))
              ) catch false' \
              <<< "$snapshot_json" > /dev/null
          }

          for ((attempt = 1; attempt <= MAX_PUSH_ATTEMPTS; attempt += 1)); do
            existing="$(snapshot_by_name)" || {
              echo "Could not verify candidate-name availability." >&2
              exit 1
            }
            if [ "$existing" != "null" ]; then
              echo "Snapshot $SNAPSHOT_NAME already exists; immutable names are never overwritten." >&2
              exit 1
            fi

            push_status=0
            push_log="$RUNNER_TEMP/daytona-push-$attempt.log"
            attempt_started_at="$(date --utc '+%Y-%m-%dT%H:%M:%S.000Z')"
            daytona snapshot push "$IMAGE_TAG" \
              --name "$SNAPSHOT_NAME" \
              --cpu "$EXPECTED_CPU" \
              --memory "$EXPECTED_MEMORY" \
              --disk "$EXPECTED_DISK" \
              --region "$EXPECTED_REGION" \
              > "$push_log" 2>&1 || push_status=$?
            cat "$push_log"

            registry_digest="$(sed -nE \
              's/.*digest: (sha256:[0-9a-f]{64}) size: [0-9]+.*/\1/p' \
              "$push_log" | tail -n 1)"
            if [[ ! "$registry_digest" =~ ^sha256:[0-9a-f]{64}$ ]]; then
              echo "Daytona push did not expose a verifiable registry digest." >&2
              exit 1
            fi
            expected_ref_suffix="/daytona-${registry_digest#sha256:}:daytona"

            retry_candidate=''
            for ((poll = 1; poll <= STATE_POLL_ATTEMPTS; poll += 1)); do
              snapshot_json="$(snapshot_by_name)" || {
                echo "Could not inspect the published snapshot candidate." >&2
                exit 1
              }
              if [ "$snapshot_json" = "null" ]; then
                state="absent"
              else
                state="$(jq --exit-status --raw-output \
                  'if type == "object" and has("state") and ((.state | type) == "string")
                    then .state else error("invalid snapshot state") end' \
                  <<< "$snapshot_json")" || {
                    echo "Published snapshot returned an invalid state payload." >&2
                    exit 1
                  }
              fi

              case "$state" in
                active)
                  if ! is_attempt_snapshot "$snapshot_json"; then
                    echo "Active snapshot provenance does not match this publish attempt." >&2
                    exit 1
                  fi
                  echo "Published snapshot $SNAPSHOT_NAME is active."
                  echo "SNAPSHOT_RETRIES=$((attempt - 1))" >> "$GITHUB_ENV"
                  exit 0
                  ;;
                absent|building|pending|pulling)
                  ;;
                error)
                  if ! is_attempt_snapshot "$snapshot_json"; then
                    echo "Failed snapshot provenance does not match this publish attempt." >&2
                    exit 1
                  fi
                  if is_retryable_timeout "$snapshot_json"; then
                    retry_candidate="$snapshot_json"
                    break
                  fi
                  echo "Snapshot publication entered a non-retryable error state." >&2
                  jq '{id, name, state, errorReason, lastUsedAt}' <<< "$snapshot_json" >&2
                  exit 1
                  ;;
                *)
                  echo "Snapshot publication entered unexpected state: $state" >&2
                  exit 1
                  ;;
              esac

              if [ "$poll" -lt "$STATE_POLL_ATTEMPTS" ]; then
                sleep 5
              fi
            done

            if [ -z "$retry_candidate" ]; then
              echo "Snapshot $SNAPSHOT_NAME did not become active; push exit status was $push_status." >&2
              exit 1
            fi
            if [ "$attempt" -eq "$MAX_PUSH_ATTEMPTS" ]; then
              echo "Snapshot processing timed out without a safe retry remaining." >&2
              exit 1
            fi

            retry_id="$(jq --exit-status --raw-output '.id' <<< "$retry_candidate")" || exit 1
            confirmed="$(snapshot_by_name)" || {
              echo "Could not revalidate the retry candidate." >&2
              exit 1
            }
            confirmed_id="$(jq --exit-status --raw-output \
              'if type == "object" and has("id") and ((.id | type) == "string")
                then .id else error("invalid snapshot id") end' \
              <<< "$confirmed")" || exit 1
            if [ "$confirmed_id" != "$retry_id" ] || \
              ! is_attempt_snapshot "$confirmed" || \
              ! is_retryable_timeout "$confirmed"; then
              echo "Retry candidate changed before cleanup; refusing deletion." >&2
              exit 1
            fi

            daytona snapshot delete "$retry_id" || exit 1
            removed=false
            for ((poll = 1; poll <= REMOVE_POLL_ATTEMPTS; poll += 1)); do
              current="$(snapshot_by_name)" || {
                echo "Could not verify retry-candidate cleanup." >&2
                exit 1
              }
              if [ "$current" = "null" ]; then
                removed=true
                break
              fi
              current_id="$(jq --exit-status --raw-output \
                'if type == "object" and has("id") and ((.id | type) == "string")
                  then .id else error("invalid snapshot id") end' \
                <<< "$current")" || exit 1
              if [ "$current_id" != "$retry_id" ]; then
                echo "Snapshot identity changed during retry cleanup." >&2
                exit 1
              fi
              if [ "$poll" -lt "$REMOVE_POLL_ATTEMPTS" ]; then
                sleep 2
              fi
            done
            if [ "$removed" != "true" ]; then
              echo "Timed-out snapshot candidate was not removed within the cleanup window." >&2
              exit 1
            fi

            echo "Retrying transient Daytona processing timeout ($attempt/$MAX_PUSH_ATTEMPTS); push exited $push_status."
          done
      - name: Record promotion instructions
        env:
          SNAPSHOT_NAME: ${{ steps.snapshot.outputs.name }}
        run: |
          {
            echo "### Daytona snapshot candidate"
            echo "- Candidate: \`$SNAPSHOT_NAME\`"
            echo "- Source commit: \`$GITHUB_SHA\`"
            echo "- Transient provider retries: \`$SNAPSHOT_RETRIES\`"
            echo "- Pre-existing, active, or previously used snapshots were not deleted or replaced."
            echo "- Promote through review by updating \`DAYTONA_SANDBOX_SNAPSHOT\` in \`apps/agent-worker/wrangler.jsonc\`."
          } >> "$GITHUB_STEP_SUMMARY"