diff --git a/Bedtime/Bedtime.xcodeproj/project.pbxproj b/Bedtime/Bedtime.xcodeproj/project.pbxproj index 386fa6d..c47e6ed 100644 --- a/Bedtime/Bedtime.xcodeproj/project.pbxproj +++ b/Bedtime/Bedtime.xcodeproj/project.pbxproj @@ -6,8 +6,19 @@ objectVersion = 77; objects = { +/* Begin PBXContainerItemProxy section */ + 373AAF072E91B7E900D8ED84 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 373AAED52E91B7E900D8ED84 /* Project object */; + proxyType = 1; + remoteGlobalIDString = 373AAEDC2E91B7E900D8ED84; + remoteInfo = Bedtime; + }; +/* End PBXContainerItemProxy section */ + /* Begin PBXFileReference section */ 373AAEDD2E91B7E900D8ED84 /* Bedger.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = Bedger.app; sourceTree = BUILT_PRODUCTS_DIR; }; + 373AAF022E91B7E900D8ED84 /* BedtimeUITests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = BedtimeUITests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; /* End PBXFileReference section */ /* Begin PBXFileSystemSynchronizedRootGroup section */ @@ -16,6 +27,11 @@ path = Bedtime; sourceTree = ""; }; + 373AAF012E91B7E900D8ED84 /* BedtimeUITests */ = { + isa = PBXFileSystemSynchronizedRootGroup; + path = BedtimeUITests; + sourceTree = ""; + }; /* End PBXFileSystemSynchronizedRootGroup section */ /* Begin PBXFrameworksBuildPhase section */ @@ -26,6 +42,13 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + 373AAF052E91B7E900D8ED84 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; /* End PBXFrameworksBuildPhase section */ /* Begin PBXGroup section */ @@ -33,6 +56,7 @@ isa = PBXGroup; children = ( 373AAEDF2E91B7E900D8ED84 /* Bedtime */, + 373AAF012E91B7E900D8ED84 /* BedtimeUITests */, 373AAEDE2E91B7E900D8ED84 /* Products */, ); sourceTree = ""; @@ -41,6 +65,7 @@ isa = PBXGroup; children = ( 373AAEDD2E91B7E900D8ED84 /* Bedger.app */, + 373AAF022E91B7E900D8ED84 /* BedtimeUITests.xctest */, ); name = Products; sourceTree = ""; @@ -70,6 +95,29 @@ productReference = 373AAEDD2E91B7E900D8ED84 /* Bedger.app */; productType = "com.apple.product-type.application"; }; + 373AAF032E91B7E900D8ED84 /* BedtimeUITests */ = { + isa = PBXNativeTarget; + buildConfigurationList = 373AAF092E91B7EA00D8ED84 /* Build configuration list for PBXNativeTarget "BedtimeUITests" */; + buildPhases = ( + 373AAF042E91B7E900D8ED84 /* Sources */, + 373AAF052E91B7E900D8ED84 /* Frameworks */, + 373AAF062E91B7E900D8ED84 /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + 373AAF082E91B7E900D8ED84 /* PBXTargetDependency */, + ); + fileSystemSynchronizedGroups = ( + 373AAF012E91B7E900D8ED84 /* BedtimeUITests */, + ); + name = BedtimeUITests; + packageProductDependencies = ( + ); + productName = BedtimeUITests; + productReference = 373AAF022E91B7E900D8ED84 /* BedtimeUITests.xctest */; + productType = "com.apple.product-type.bundle.ui-testing"; + }; /* End PBXNativeTarget section */ /* Begin PBXProject section */ @@ -83,6 +131,10 @@ 373AAEDC2E91B7E900D8ED84 = { CreatedOnToolsVersion = 26.0; }; + 373AAF032E91B7E900D8ED84 = { + CreatedOnToolsVersion = 26.0; + TestTargetID = 373AAEDC2E91B7E900D8ED84; + }; }; }; buildConfigurationList = 373AAED82E91B7E900D8ED84 /* Build configuration list for PBXProject "Bedtime" */; @@ -100,6 +152,7 @@ projectRoot = ""; targets = ( 373AAEDC2E91B7E900D8ED84 /* Bedtime */, + 373AAF032E91B7E900D8ED84 /* BedtimeUITests */, ); }; /* End PBXProject section */ @@ -112,6 +165,13 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + 373AAF062E91B7E900D8ED84 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; /* End PBXResourcesBuildPhase section */ /* Begin PBXSourcesBuildPhase section */ @@ -122,8 +182,23 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + 373AAF042E91B7E900D8ED84 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; /* End PBXSourcesBuildPhase section */ +/* Begin PBXTargetDependency section */ + 373AAF082E91B7E900D8ED84 /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + target = 373AAEDC2E91B7E900D8ED84 /* Bedtime */; + targetProxy = 373AAF072E91B7E900D8ED84 /* PBXContainerItemProxy */; + }; +/* End PBXTargetDependency section */ + /* Begin XCBuildConfiguration section */ 373AAEEC2E91B7EA00D8ED84 /* Debug */ = { isa = XCBuildConfiguration; @@ -322,6 +397,42 @@ }; name = Release; }; + 373AAF0A2E91B7EA00D8ED84 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = 9J44X82D53; + GENERATE_INFOPLIST_FILE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 26.0; + MARKETING_VERSION = 0.9.0; + PRODUCT_BUNDLE_IDENTIFIER = com.burnsides.bedtime.uitests; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_EMIT_LOC_STRINGS = NO; + SWIFT_VERSION = 5.0; + TARGETED_DEVICE_FAMILY = "1,2"; + TEST_TARGET_NAME = Bedtime; + }; + name = Debug; + }; + 373AAF0B2E91B7EA00D8ED84 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = 9J44X82D53; + GENERATE_INFOPLIST_FILE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 26.0; + MARKETING_VERSION = 0.9.0; + PRODUCT_BUNDLE_IDENTIFIER = com.burnsides.bedtime.uitests; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_EMIT_LOC_STRINGS = NO; + SWIFT_VERSION = 5.0; + TARGETED_DEVICE_FAMILY = "1,2"; + TEST_TARGET_NAME = Bedtime; + }; + name = Release; + }; /* End XCBuildConfiguration section */ /* Begin XCConfigurationList section */ @@ -343,6 +454,15 @@ defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; }; + 373AAF092E91B7EA00D8ED84 /* Build configuration list for PBXNativeTarget "BedtimeUITests" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 373AAF0A2E91B7EA00D8ED84 /* Debug */, + 373AAF0B2E91B7EA00D8ED84 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; /* End XCConfigurationList section */ }; rootObject = 373AAED52E91B7E900D8ED84 /* Project object */; diff --git a/Bedtime/Bedtime/ContentView.swift b/Bedtime/Bedtime/ContentView.swift index b1d87ce..70ac7db 100644 --- a/Bedtime/Bedtime/ContentView.swift +++ b/Bedtime/Bedtime/ContentView.swift @@ -93,6 +93,7 @@ struct ContentView: View { .frame(maxWidth: 600) .frame(maxWidth: .infinity) } + .accessibilityIdentifier("home_screen") .background(Color.backgroundBehindCards) .navigationTitle("Bedger") .toolbar { @@ -100,6 +101,7 @@ struct ContentView: View { Button("Settings", systemImage: "gear") { showingSettings.toggle() } + .accessibilityIdentifier("settings_button") } } .refreshable { @@ -125,6 +127,10 @@ struct ContentView: View { } } .task { + #if DEBUG + healthKitManager.prepareForUITestingIfNeeded() + guard !UITestingSupport.isActive else { return } + #endif try? await healthKitManager.fetchSleepData() } } diff --git a/Bedtime/Bedtime/Models/HealthKitManager.swift b/Bedtime/Bedtime/Models/HealthKitManager.swift index 5d5004e..dd9dbcf 100644 --- a/Bedtime/Bedtime/Models/HealthKitManager.swift +++ b/Bedtime/Bedtime/Models/HealthKitManager.swift @@ -158,4 +158,13 @@ class HealthKitManager: ObservableObject { self.sleepSessions = Dictionary(grouping: sessions) { $0.dateForGrouping } } + + #if DEBUG + func prepareForUITestingIfNeeded() { + guard UITestingSupport.isActive else { return } + isAuthorized = true + sleepSessions = UITestingSupport.mockSleepSessions() + errorMessage = nil + } + #endif } diff --git a/Bedtime/Bedtime/Support/UITestingSupport.swift b/Bedtime/Bedtime/Support/UITestingSupport.swift new file mode 100644 index 0000000..f4e59b5 --- /dev/null +++ b/Bedtime/Bedtime/Support/UITestingSupport.swift @@ -0,0 +1,57 @@ +#if DEBUG +import Foundation +import HealthKit + +enum UITestingSupport { + static var isActive: Bool { + ProcessInfo.processInfo.arguments.contains("-ui_testing") + } + + static func mockSleepSessions() -> [Date: [SleepSession]] { + let source = HKSourceRevision(source: HKSource.default(), version: nil) + let calendar = Calendar.current + let today = calendar.startOfDay(for: Date()) + + func session( + dayOffset: Int, + startHour: Int, + startMinute: Int, + endHour: Int, + endMinute: Int + ) -> SleepSession { + let base = calendar.date(byAdding: .day, value: dayOffset, to: today) ?? today + let start = calendar.date( + bySettingHour: startHour, + minute: startMinute, + second: 0, + of: base + ) ?? base + let endDayOffset = endHour < startHour ? 1 : 0 + let endBase = calendar.date(byAdding: .day, value: endDayOffset, to: base) ?? base + let end = calendar.date( + bySettingHour: endHour, + minute: endMinute, + second: 0, + of: endBase + ) ?? endBase + + return SleepSession( + startDate: start, + endDate: end, + sleepType: .asleepCore, + source: source + ) + } + + let lastNight = session(dayOffset: -1, startHour: 23, startMinute: 15, endHour: 7, endMinute: 5) + let twoNightsAgo = session(dayOffset: -2, startHour: 22, startMinute: 45, endHour: 6, endMinute: 30) + let threeNightsAgo = session(dayOffset: -3, startHour: 23, startMinute: 30, endHour: 7, endMinute: 45) + + return [ + lastNight.dateForGrouping: [lastNight], + twoNightsAgo.dateForGrouping: [twoNightsAgo], + threeNightsAgo.dateForGrouping: [threeNightsAgo], + ] + } +} +#endif diff --git a/Bedtime/Bedtime/Views/SettingsView.swift b/Bedtime/Bedtime/Views/SettingsView.swift index 6b53847..23c1a5e 100644 --- a/Bedtime/Bedtime/Views/SettingsView.swift +++ b/Bedtime/Bedtime/Views/SettingsView.swift @@ -154,6 +154,7 @@ struct SettingsView: View { } .navigationTitle("Settings") .navigationBarTitleDisplayMode(.inline) + .accessibilityIdentifier("settings_screen") .toolbar { if horizontalSizeClass == .compact { ToolbarItem(placement: .confirmationAction) { diff --git a/Bedtime/BedtimeUITests/ScreenshotTests.swift b/Bedtime/BedtimeUITests/ScreenshotTests.swift new file mode 100644 index 0000000..f75400d --- /dev/null +++ b/Bedtime/BedtimeUITests/ScreenshotTests.swift @@ -0,0 +1,31 @@ +import XCTest + +/// UI tests that capture screenshots for pull request previews. +/// +/// Screenshots are saved as XCTAttachments with `.keepAlways` so Xcode Cloud +/// includes them in the test result bundle. `ci_post_xcodebuild.sh` can then +/// upload them to Imgur (or another host) and embed the URLs in a PR comment. +final class ScreenshotTests: XCTestCase { + + override func setUpWithError() throws { + continueAfterFailure = false + } + + @MainActor + func testPullRequestScreenshots() throws { + let app = XCUIApplication() + app.launchArguments.append("-ui_testing") + app.launch() + + XCTAssertTrue(app.scrollViews["home_screen"].waitForExistence(timeout: 10)) + attachScreenshot(named: "01-home", in: app) + + app.buttons["settings_button"].tap() + XCTAssertTrue(app.navigationBars["Settings"].waitForExistence(timeout: 5)) + attachScreenshot(named: "02-settings", in: app) + + if app.buttons["Done"].exists { + app.buttons["Done"].tap() + } + } +} diff --git a/Bedtime/BedtimeUITests/XCTestCase+Screenshots.swift b/Bedtime/BedtimeUITests/XCTestCase+Screenshots.swift new file mode 100644 index 0000000..0683e50 --- /dev/null +++ b/Bedtime/BedtimeUITests/XCTestCase+Screenshots.swift @@ -0,0 +1,10 @@ +import XCTest + +extension XCTestCase { + func attachScreenshot(named name: String, in app: XCUIApplication, file: StaticString = #filePath, line: UInt = #line) { + let attachment = XCTAttachment(screenshot: app.screenshot()) + attachment.name = name + attachment.lifetime = .keepAlways + add(attachment) + } +} diff --git a/Bedtime/ci_scripts/ci_post_xcodebuild.sh b/Bedtime/ci_scripts/ci_post_xcodebuild.sh new file mode 100755 index 0000000..e8165ab --- /dev/null +++ b/Bedtime/ci_scripts/ci_post_xcodebuild.sh @@ -0,0 +1,131 @@ +#!/bin/sh +set -eu + +# Runs on the Xcode Cloud Mac after xcodebuild finishes. +# Extracts screenshots, uploads to Imgur, and posts a sticky commit comment. +# Cursor/agents can wait for CI, then read that comment for results. + +OUTPUT_DIR="${CI_DERIVED_DATA_PATH:-/tmp}/xcode-cloud-screenshots" +SCREENSHOTS_DIR="$OUTPUT_DIR/screenshots" +ONLY_FAILURES="${XCODE_CLOUD_SCREENSHOT_ONLY_FAILURES:-false}" +BUILD_ID="${CI_BUILD_ID:-unknown}" +XCODEBUILD_EXIT="${CI_XCODEBUILD_EXIT_CODE:-0}" + +REPO_ROOT="${CI_PRIMARY_REPOSITORY_PATH:-$(CDPATH= cd -- "$(dirname "$0")/../.." && pwd)}" +PYTHON_BIN="${PYTHON_BIN:-python3}" +FETCH_SCRIPT="$REPO_ROOT/scripts/fetch_xcode_cloud_screenshots.py" + +COMMIT_SHA="${CI_COMMIT:-}" +REPO_SLUG="${GITHUB_REPOSITORY:-${CI_PULL_REQUEST_TARGET_REPO:-${CI_PULL_REQUEST_SOURCE_REPO:-}}}" +BASELINE_COMMIT="${CI_PULL_REQUEST_TARGET_COMMIT:-}" + +WHAT_TO_TEST="" +if [ -d "$REPO_ROOT/.git" ]; then + WHAT_TO_TEST="$(git -C "$REPO_ROOT" log -1 --format=%B 2>/dev/null | awk ' + /^What to test:/ { capture=1; next } + capture { print } + ' | sed '/^[[:space:]]*$/d' || true)" +fi + +mkdir -p "$OUTPUT_DIR" +ERRORS_FILE="$OUTPUT_DIR/errors.txt" +: > "$ERRORS_FILE" + +COMMENT_ARGS="--repo \"$REPO_SLUG\" --commit-sha \"$COMMIT_SHA\" --run-id \"$BUILD_ID\"" +if [ -n "$BASELINE_COMMIT" ] && [ "$BASELINE_COMMIT" != "$COMMIT_SHA" ]; then + COMMENT_ARGS="$COMMENT_ARGS --baseline-commit \"$BASELINE_COMMIT\"" +fi +if [ -n "$WHAT_TO_TEST" ]; then + WHAT_TO_TEST_FILE="$OUTPUT_DIR/what-to-test.txt" + printf '%s\n' "$WHAT_TO_TEST" > "$WHAT_TO_TEST_FILE" + COMMENT_ARGS="$COMMENT_ARGS --what-to-test-file \"$WHAT_TO_TEST_FILE\"" +fi + +append_failures_from_bundle() { + if [ -e "${CI_RESULT_BUNDLE_PATH:-}" ]; then + "$PYTHON_BIN" "$FETCH_SCRIPT" extract-failures \ + --bundle-path "$CI_RESULT_BUNDLE_PATH" >> "$ERRORS_FILE" 2>/dev/null || true + fi +} + +publish_report() { + if [ -z "$REPO_SLUG" ] || [ -z "$COMMIT_SHA" ] || [ -z "${GITHUB_TOKEN:-}" ]; then + echo "ci_post_xcodebuild: skipping commit report (need repo, commit, GITHUB_TOKEN)" + return 0 + fi + + echo "ci_post_xcodebuild: publishing build report on commit ${COMMIT_SHA}" + "$PYTHON_BIN" -m pip install --quiet -r "$REPO_ROOT/scripts/requirements.txt" + # shellcheck disable=SC2086 + "$PYTHON_BIN" "$FETCH_SCRIPT" comment-build $COMMENT_ARGS +} + +if [ "$XCODEBUILD_EXIT" != "0" ]; then + echo "ci_post_xcodebuild: xcodebuild failed with exit code ${XCODEBUILD_EXIT}" + printf '%s\n' "xcodebuild failed with exit code ${XCODEBUILD_EXIT}" >> "$ERRORS_FILE" + if [ -n "${CI_XCODEBUILD_ACTION:-}" ]; then + printf '%s\n' "Action: ${CI_XCODEBUILD_ACTION}" >> "$ERRORS_FILE" + fi + if [ -n "${CI_BUILD_URL:-}" ]; then + printf '%s\n' "Build logs: ${CI_BUILD_URL}" >> "$ERRORS_FILE" + fi + append_failures_from_bundle + COMMENT_ARGS="$COMMENT_ARGS --status failed --exit-code \"$XCODEBUILD_EXIT\" --errors-file \"$ERRORS_FILE\"" + publish_report || true + exit 0 +fi + +if [ -z "${CI_RESULT_BUNDLE_PATH:-}" ] || [ ! -e "$CI_RESULT_BUNDLE_PATH" ]; then + echo "ci_post_xcodebuild: no test result bundle available" + printf '%s\n' "No test result bundle was produced." >> "$ERRORS_FILE" + COMMENT_ARGS="$COMMENT_ARGS --status no_screenshots --errors-file \"$ERRORS_FILE\"" + publish_report || true + exit 0 +fi + +EXTRACT_ARGS="extract-local --bundle-path \"$CI_RESULT_BUNDLE_PATH\" --output-dir \"$OUTPUT_DIR\"" +if [ "$ONLY_FAILURES" = "true" ]; then + EXTRACT_ARGS="$EXTRACT_ARGS --only-failures" +fi + +echo "ci_post_xcodebuild: exporting screenshots from $CI_RESULT_BUNDLE_PATH" +set +e +# shellcheck disable=SC2086 +"$PYTHON_BIN" "$FETCH_SCRIPT" $EXTRACT_ARGS +EXTRACT_EXIT=$? +set -eu + +if [ "$EXTRACT_EXIT" -ne 0 ]; then + printf '%s\n' "Screenshot extraction failed with exit code ${EXTRACT_EXIT}" >> "$ERRORS_FILE" + append_failures_from_bundle + COMMENT_ARGS="$COMMENT_ARGS --status failed --exit-code \"$EXTRACT_EXIT\" --errors-file \"$ERRORS_FILE\"" + publish_report || true + exit 0 +fi + +SCREENSHOT_COUNT=0 +if [ -d "$SCREENSHOTS_DIR" ]; then + SCREENSHOT_COUNT="$(find "$SCREENSHOTS_DIR" -name '*.png' -type f | wc -l | tr -d ' ')" +fi + +if [ "$SCREENSHOT_COUNT" = "0" ]; then + echo "ci_post_xcodebuild: no screenshot PNGs extracted" + printf '%s\n' "No screenshot PNG attachments were found in the test result bundle." >> "$ERRORS_FILE" + append_failures_from_bundle + COMMENT_ARGS="$COMMENT_ARGS --status no_screenshots --errors-file \"$ERRORS_FILE\"" + publish_report || true + exit 0 +fi + +if [ -z "${IMGUR_CLIENT_ID:-}" ]; then + echo "ci_post_xcodebuild: IMGUR_CLIENT_ID is not set" + printf '%s\n' "IMGUR_CLIENT_ID is not configured for screenshot upload." >> "$ERRORS_FILE" + COMMENT_ARGS="$COMMENT_ARGS --status failed --errors-file \"$ERRORS_FILE\"" + publish_report || true + exit 0 +fi + +COMMENT_ARGS="$COMMENT_ARGS --status success --screenshots-dir \"$SCREENSHOTS_DIR\"" +publish_report || true + +echo "ci_post_xcodebuild: screenshots available in $SCREENSHOTS_DIR" diff --git a/scripts/.gitignore b/scripts/.gitignore new file mode 100644 index 0000000..0921fae --- /dev/null +++ b/scripts/.gitignore @@ -0,0 +1,4 @@ +xcode-cloud-output/ +__pycache__/ +.pytest_cache/ +*.pyc diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..e34cbfc --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,84 @@ +# Xcode Cloud screenshots + +Every push runs Xcode Cloud. `ci_post_xcodebuild.sh` extracts UI test screenshots, uploads them to Imgur, and posts a sticky **commit comment** with results. + +## Flow + +```text +push to main → tests capture screenshots → commit comment (full set, becomes baseline) + +push to PR → tests capture screenshots → commit comment (diff vs main baseline) +``` + +Cursor (or any agent) can **wait for CI to finish**, then read the commit comment on that push's SHA for screenshots, diffs, or failure details. + +## Xcode Cloud setup + +Add a workflow (or use your existing per-push workflow) with: + +| Setting | Value | +|---------|--------| +| Start condition | **Every push** (branch + pull request) | +| Test action | Scheme `Bedtime`, include `BedtimeUITests` | +| Test plan | Screenshots: **On, and keep all** | + +### Secrets + +```bash +IMGUR_CLIENT_ID=... # public image URLs (success path) +GITHUB_TOKEN=... # commit comments (success + failure) +``` + +`GITHUB_TOKEN` needs permission to create and update **commit comments**. + +## Commit comments + +Reports appear on `github.com/{owner}/{repo}/commit/{sha}` (Commits tab on PRs). + +**Main builds** — upload all screenshots to Imgur; embed URLs in the comment for baseline lookup. + +**PR builds** — download main's baseline from `CI_PULL_REQUEST_TARGET_COMMIT`, pixel-compare, show before/after for changed and new screenshots only. + +**Failed builds** — post exit code, test failures, and build log URL (no Imgur upload). + +Hidden status block for programmatic reads: + +```html + +``` + +## New screenshot scenarios + +Use scenario-specific attachment names (e.g. `01-home-empty-bank`). When adding a new test case, **merge the test to main early** so main captures the baseline before the feature branch gets far ahead. + +## Local / manual CLI + +Used by `ci_post_xcodebuild.sh`; also runnable locally on a Mac with a `.xcresult` bundle: + +```bash +python3 scripts/fetch_xcode_cloud_screenshots.py extract-local \ + --bundle-path /path/to/Result.xcresult \ + --output-dir ./screenshots + +python3 scripts/fetch_xcode_cloud_screenshots.py comment-build \ + --repo owner/repo \ + --commit-sha abc123 \ + --baseline-commit mainsha789 \ + --run-id build-1 \ + --status success \ + --screenshots-dir ./screenshots +``` + +## What generates the screenshots + +**`BedtimeUITests/ScreenshotTests.swift`** — XCUITest attachments with `.keepAlways`. + +```bash +xcodebuild test \ + -project Bedtime/Bedtime.xcodeproj \ + -scheme Bedtime \ + -destination 'platform=iOS Simulator,name=iPhone 16' \ + -only-testing:BedtimeUITests/ScreenshotTests +``` diff --git a/scripts/fetch_xcode_cloud_screenshots.py b/scripts/fetch_xcode_cloud_screenshots.py new file mode 100644 index 0000000..c33be6c --- /dev/null +++ b/scripts/fetch_xcode_cloud_screenshots.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python3 +"""Xcode Cloud screenshot extraction and GitHub commit reporting.""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from scripts.xcode_cloud.build_report import ( + publish_build_failure_report, + publish_no_screenshots_report, + publish_screenshot_commit_report, +) +from scripts.xcode_cloud.extract import extract_screenshots_from_local_bundle +from scripts.xcode_cloud.github_comments import BuildStatus, build_screenshot_comment +from scripts.xcode_cloud.github_commit import upsert_commit_comment +from scripts.xcode_cloud.upload import ( + UploadConfigError, + UploadedScreenshot, + upload_screenshots, + write_manifest, +) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Extract Xcode Cloud UI test screenshots and publish GitHub commit reports." + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + local_parser = subparsers.add_parser( + "extract-local", + help="Extract screenshots from a local .xcresult bundle.", + ) + local_parser.add_argument("--bundle-path", required=True) + local_parser.add_argument( + "--output-dir", + default="./xcode-cloud-output", + help="Directory for extracted screenshots", + ) + local_parser.add_argument("--only-failures", action="store_true") + + failures_parser = subparsers.add_parser( + "extract-failures", + help="Print failing test summaries from a local .xcresult bundle.", + ) + failures_parser.add_argument("--bundle-path", required=True) + + comment_parser = subparsers.add_parser( + "comment-build", + help="Post or update a sticky build report comment on a Git commit.", + ) + comment_parser.add_argument("--repo", required=True, help="owner/repo") + comment_parser.add_argument("--commit-sha", required=True, help="Commit to comment on") + comment_parser.add_argument("--run-id", required=True) + comment_parser.add_argument( + "--status", + choices=[status.value for status in BuildStatus], + help="Terminal build status to publish", + ) + comment_parser.add_argument( + "--baseline-commit", + help="Baseline commit to compare against (success path only)", + ) + comment_parser.add_argument( + "--screenshots-dir", + help="Directory containing extracted .png files (success path)", + ) + comment_parser.add_argument("--exit-code", type=int, help="xcodebuild exit code (failed path)") + comment_parser.add_argument( + "--errors-file", + help="Text file with one error per line (failed/no-screenshots paths)", + ) + comment_parser.add_argument( + "--log-file", + help="Log excerpt to include in a failed build comment", + ) + comment_parser.add_argument( + "--what-to-test-file", + help="Text file with What to test notes for the commit comment", + ) + + comment_commit_parser = subparsers.add_parser( + "comment-commit", + help="Alias for comment-build --status success.", + ) + comment_commit_parser.add_argument("--repo", required=True, help="owner/repo") + comment_commit_parser.add_argument("--commit-sha", required=True, help="Commit to comment on") + comment_commit_parser.add_argument( + "--baseline-commit", + help="Baseline commit to compare against (e.g. main HEAD for PR builds)", + ) + comment_commit_parser.add_argument("--run-id", required=True) + comment_commit_parser.add_argument( + "--screenshots-dir", + help="Directory containing extracted .png files (used when no manifest is provided)", + ) + comment_commit_parser.add_argument( + "--manifest", + help="JSON manifest written by upload-screenshots (legacy, no diff)", + ) + comment_commit_parser.add_argument( + "--what-to-test-file", + help="Text file with What to test notes for the commit comment", + ) + + upload_parser = subparsers.add_parser( + "upload-screenshots", + help="Upload extracted screenshots to Imgur or S3.", + ) + upload_parser.add_argument("--screenshots-dir", required=True) + upload_parser.add_argument("--build-id", required=True) + upload_parser.add_argument( + "--backend", + choices=["auto", "imgur", "s3"], + default="auto", + help="Upload backend (default: auto-detect from env)", + ) + upload_parser.add_argument( + "--manifest", + default="./xcode-cloud-output/screenshots-manifest.json", + help="Where to write the public URL manifest", + ) + return parser + + +def _read_error_lines(path: str | None) -> list[str]: + if not path: + return [] + return [line.strip() for line in Path(path).read_text().splitlines() if line.strip()] + + +def _read_what_to_test(args: argparse.Namespace) -> str | None: + path = getattr(args, "what_to_test_file", None) + if not path: + return None + text = Path(path).read_text().strip() + return text or None + + +def _publish_comment_build(args: argparse.Namespace, parser: argparse.ArgumentParser) -> int: + token = os.environ.get("GITHUB_TOKEN") + if not token: + parser.error("GITHUB_TOKEN is required") + + what_to_test = _read_what_to_test(args) + errors = _read_error_lines(getattr(args, "errors_file", None)) + log_excerpt = None + log_file = getattr(args, "log_file", None) + if log_file: + log_excerpt = Path(log_file).read_text().strip() or None + + status = getattr(args, "status", None) + if status is None and args.screenshots_dir: + status = BuildStatus.SUCCESS.value + if status is None: + parser.error("comment-build requires --status or --screenshots-dir") + + if status == BuildStatus.SUCCESS.value: + if not args.screenshots_dir: + parser.error("comment-build --status success requires --screenshots-dir") + result = publish_screenshot_commit_report( + args.repo, + args.commit_sha, + args.run_id, + Path(args.screenshots_dir), + token=token, + baseline_commit_sha=getattr(args, "baseline_commit", None), + what_to_test=what_to_test, + ) + elif status == BuildStatus.FAILED.value: + result = publish_build_failure_report( + args.repo, + args.commit_sha, + args.run_id, + token=token, + exit_code=getattr(args, "exit_code", None), + errors=errors or None, + log_excerpt=log_excerpt, + ) + else: + result = publish_no_screenshots_report( + args.repo, + args.commit_sha, + args.run_id, + token=token, + errors=errors or None, + ) + + print( + f"Published {result['status']} report on {args.repo}@{args.commit_sha[:7]}: " + f"{result['comment_url']}" + ) + return 0 + + +def main(argv: list[str] | None = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + + if args.command == "extract-local": + screenshots = extract_screenshots_from_local_bundle( + Path(args.bundle_path), + Path(args.output_dir), + only_failures=args.only_failures, + ) + if not screenshots: + print("No screenshots found.") + else: + print(f"Extracted {len(screenshots)} screenshot(s):") + for path in screenshots: + print(path) + return 0 + + if args.command == "extract-failures": + from scripts.xcode_cloud.extract import extract_test_failure_summaries + + for line in extract_test_failure_summaries(Path(args.bundle_path)): + print(line) + return 0 + + if args.command == "comment-build": + return _publish_comment_build(args, parser) + + if args.command == "comment-commit": + token = os.environ.get("GITHUB_TOKEN") + if not token: + parser.error("GITHUB_TOKEN is required for comment-commit") + + what_to_test = _read_what_to_test(args) + + if args.screenshots_dir: + args.status = BuildStatus.SUCCESS.value + return _publish_comment_build(args, parser) + + if args.manifest: + manifest = json.loads(Path(args.manifest).read_text()) + uploaded = [ + UploadedScreenshot(name=item["name"], key=item["key"], url=item["url"]) + for item in manifest.get("screenshots", []) + ] + screenshot_urls = {item.name: item.url for item in uploaded} + body = build_screenshot_comment( + [], + build_run_id=args.run_id, + commit_sha=args.commit_sha, + uploaded=uploaded, + what_to_test=what_to_test, + screenshot_urls=screenshot_urls, + ) + upsert_commit_comment( + args.repo, + args.commit_sha, + body, + token=token, + ) + print(f"Updated commit comment on {args.repo}@{args.commit_sha[:7]}") + return 0 + + parser.error("comment-commit requires --screenshots-dir or --manifest") + + if args.command == "upload-screenshots": + try: + uploads = upload_screenshots( + Path(args.screenshots_dir), + build_id=args.build_id, + backend=args.backend, + ) + except UploadConfigError as error: + parser.error(str(error)) + + manifest_path = Path(args.manifest) + write_manifest(manifest_path, args.build_id, uploads) + print(f"Uploaded {len(uploads)} screenshot(s) via {args.backend}") + for item in uploads: + print(item.url) + print(f"Manifest: {manifest_path}") + return 0 + + parser.error(f"Unknown command: {args.command}") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/requirements.txt b/scripts/requirements.txt new file mode 100644 index 0000000..62804ac --- /dev/null +++ b/scripts/requirements.txt @@ -0,0 +1,2 @@ +httpx>=0.27 +Pillow>=10.0 diff --git a/scripts/xcode_cloud/__init__.py b/scripts/xcode_cloud/__init__.py new file mode 100644 index 0000000..68369d4 --- /dev/null +++ b/scripts/xcode_cloud/__init__.py @@ -0,0 +1 @@ +"""Extract Xcode Cloud UI test screenshots and publish GitHub commit reports.""" diff --git a/scripts/xcode_cloud/build_report.py b/scripts/xcode_cloud/build_report.py new file mode 100644 index 0000000..d6de2a5 --- /dev/null +++ b/scripts/xcode_cloud/build_report.py @@ -0,0 +1,157 @@ +"""Publish sticky commit build reports for agent feedback loops.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import httpx + +from scripts.xcode_cloud.compare import compare_screenshot_sets +from scripts.xcode_cloud.github_comments import ( + BuildStatus, + attach_upload_urls, + build_failure_comment, + build_no_screenshots_comment, + build_screenshot_comment, +) +from scripts.xcode_cloud.github_commit import ( + download_baseline_images, + fetch_screenshot_urls_from_commit, + upsert_commit_comment, +) +from scripts.xcode_cloud.upload import UploadBackend, upload_screenshots + + +def publish_build_failure_report( + repo: str, + commit_sha: str, + build_run_id: str, + *, + token: str, + exit_code: int | None = None, + errors: list[str] | None = None, + log_excerpt: str | None = None, +) -> dict[str, Any]: + body = build_failure_comment( + build_run_id=build_run_id, + commit_sha=commit_sha, + exit_code=exit_code, + errors=errors, + log_excerpt=log_excerpt, + ) + with httpx.Client(timeout=60.0) as client: + comment = upsert_commit_comment( + repo, + commit_sha, + body, + token=token, + client=client, + ) + return { + "status": BuildStatus.FAILED.value, + "comment_id": comment["id"], + "comment_url": comment["html_url"], + } + + +def publish_no_screenshots_report( + repo: str, + commit_sha: str, + build_run_id: str, + *, + token: str, + errors: list[str] | None = None, +) -> dict[str, Any]: + body = build_no_screenshots_comment( + build_run_id=build_run_id, + commit_sha=commit_sha, + errors=errors, + ) + with httpx.Client(timeout=60.0) as client: + comment = upsert_commit_comment( + repo, + commit_sha, + body, + token=token, + client=client, + ) + return { + "status": BuildStatus.NO_SCREENSHOTS.value, + "comment_id": comment["id"], + "comment_url": comment["html_url"], + } + + +def publish_screenshot_commit_report( + repo: str, + commit_sha: str, + build_run_id: str, + screenshots_dir: Path, + *, + token: str, + baseline_commit_sha: str | None = None, + what_to_test: str | None = None, + upload_backend: UploadBackend = "auto", + cache_root: Path | None = None, +) -> dict[str, Any]: + cache_root = cache_root or screenshots_dir.parent / "baseline-cache" + baseline_dir = cache_root / f"baseline-{baseline_commit_sha or 'none'}" + + with httpx.Client(timeout=60.0, follow_redirects=True) as client: + baseline_urls: dict[str, str] = {} + if baseline_commit_sha and baseline_commit_sha != commit_sha: + baseline_urls = fetch_screenshot_urls_from_commit( + repo, + baseline_commit_sha, + token=token, + client=client, + ) + if baseline_urls: + download_baseline_images( + baseline_urls, + baseline_dir, + client=client, + ) + + comparisons = compare_screenshot_sets( + screenshots_dir, + baseline_dir if baseline_urls else None, + baseline_urls=baseline_urls, + ) + + uploads = upload_screenshots( + screenshots_dir, + build_id=build_run_id, + backend=upload_backend, + http_client=client, + ) + comparisons = attach_upload_urls(comparisons, uploads) + screenshot_urls = {item.name: item.url for item in uploads} + + body = build_screenshot_comment( + [], + build_run_id=build_run_id, + commit_sha=commit_sha, + baseline_commit_sha=baseline_commit_sha, + comparisons=comparisons, + what_to_test=what_to_test, + screenshot_urls=screenshot_urls, + ) + comment = upsert_commit_comment( + repo, + commit_sha, + body, + token=token, + client=client, + ) + + return { + "status": BuildStatus.SUCCESS.value, + "comment_id": comment["id"], + "comment_url": comment["html_url"], + "upload_count": len(uploads), + "comparison_count": len(comparisons), + "baseline_commit_sha": baseline_commit_sha, + "screenshot_urls": screenshot_urls, + } diff --git a/scripts/xcode_cloud/compare.py b/scripts/xcode_cloud/compare.py new file mode 100644 index 0000000..44fa6f9 --- /dev/null +++ b/scripts/xcode_cloud/compare.py @@ -0,0 +1,136 @@ +"""Compare UI test screenshots against a previous baseline.""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from enum import Enum +from pathlib import Path + + +class ScreenshotChange(str, Enum): + NEW = "new" + CHANGED = "changed" + UNCHANGED = "unchanged" + REMOVED = "removed" + + +@dataclass(frozen=True) +class ScreenshotComparison: + name: str + change: ScreenshotChange + before_path: Path | None = None + after_path: Path | None = None + before_url: str | None = None + after_url: str | None = None + + +def file_sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _pixel_diff_ratio(left: Path, right: Path) -> float: + try: + from PIL import Image, ImageChops + except ImportError as error: + raise RuntimeError( + "Pillow is required for screenshot comparison. Install with: pip install Pillow" + ) from error + + with Image.open(left) as left_image, Image.open(right) as right_image: + if left_image.size != right_image.size: + right_image = right_image.resize(left_image.size) + + left_rgb = left_image.convert("RGB") + right_rgb = right_image.convert("RGB") + diff = ImageChops.difference(left_rgb, right_rgb) + histogram = diff.histogram() + # RGB histogram: 256 bins per channel + differing_pixels = sum( + histogram[index] + for channel in range(3) + for index in range(1, 256) + ) + total_pixels = left_rgb.size[0] * left_rgb.size[1] + return differing_pixels / (total_pixels * 3) + + +def screenshots_differ( + before: Path, + after: Path, + *, + pixel_threshold: float = 0.001, +) -> bool: + if file_sha256(before) == file_sha256(after): + return False + return _pixel_diff_ratio(before, after) > pixel_threshold + + +def compare_screenshot_sets( + current_dir: Path, + baseline_dir: Path | None, + *, + baseline_urls: dict[str, str] | None = None, + pixel_threshold: float = 0.001, +) -> list[ScreenshotComparison]: + current_paths = { + path.name: path for path in sorted(current_dir.rglob("*.png")) if path.is_file() + } + baseline_paths: dict[str, Path] = {} + if baseline_dir and baseline_dir.exists(): + baseline_paths = { + path.name: path for path in sorted(baseline_dir.rglob("*.png")) if path.is_file() + } + + baseline_urls = baseline_urls or {} + comparisons: list[ScreenshotComparison] = [] + + for name, after_path in current_paths.items(): + before_path = baseline_paths.get(name) + before_url = baseline_urls.get(name) + if before_path is None: + comparisons.append( + ScreenshotComparison( + name=name, + change=ScreenshotChange.NEW, + after_path=after_path, + after_url=None, + ) + ) + continue + + if screenshots_differ(before_path, after_path, pixel_threshold=pixel_threshold): + comparisons.append( + ScreenshotComparison( + name=name, + change=ScreenshotChange.CHANGED, + before_path=before_path, + after_path=after_path, + before_url=before_url, + after_url=None, + ) + ) + else: + comparisons.append( + ScreenshotComparison( + name=name, + change=ScreenshotChange.UNCHANGED, + before_path=before_path, + after_path=after_path, + before_url=before_url, + after_url=before_url, + ) + ) + + for name, before_path in baseline_paths.items(): + if name not in current_paths: + comparisons.append( + ScreenshotComparison( + name=name, + change=ScreenshotChange.REMOVED, + before_path=before_path, + before_url=baseline_urls.get(name), + ) + ) + + return comparisons diff --git a/scripts/xcode_cloud/extract.py b/scripts/xcode_cloud/extract.py new file mode 100644 index 0000000..e6d3cc5 --- /dev/null +++ b/scripts/xcode_cloud/extract.py @@ -0,0 +1,150 @@ +"""Extract screenshot attachments and failure summaries from an .xcresult bundle.""" + +from __future__ import annotations + +import json +import shutil +import subprocess +from pathlib import Path + + +class XcresultToolNotFoundError(RuntimeError): + """Raised when xcresulttool is unavailable on the current machine.""" + + +def xcresulttool_path() -> str | None: + return shutil.which("xcresulttool") + + +def extract_attachments( + bundle_path: Path, + output_dir: Path, + *, + only_failures: bool = False, +) -> list[Path]: + """Export XCTest attachments from a result bundle using xcresulttool.""" + if not bundle_path.exists(): + raise FileNotFoundError(f"Result bundle not found: {bundle_path}") + + tool = xcresulttool_path() + if tool is None: + raise XcresultToolNotFoundError( + "xcresulttool was not found. Run this step on macOS with Xcode installed, " + "or use ci_scripts/ci_post_xcodebuild.sh inside Xcode Cloud." + ) + + output_dir.mkdir(parents=True, exist_ok=True) + command = [ + tool, + "export", + "attachments", + "--path", + str(bundle_path), + "--output-path", + str(output_dir), + ] + if only_failures: + command.append("--only-failures") + + subprocess.run(command, check=True, capture_output=True, text=True) + return sorted(path for path in output_dir.rglob("*.png") if path.is_file()) + + +def extract_test_failure_summaries( + bundle_path: Path, + *, + limit: int = 20, +) -> list[str]: + """Return human-readable failing test summaries when xcresulttool is available.""" + if not bundle_path.exists(): + return [] + + tool = xcresulttool_path() + if tool is None: + return [] + + command = [ + tool, + "get", + "test-results", + "tests", + "--path", + str(bundle_path), + "--format", + "json", + ] + result = subprocess.run(command, capture_output=True, text=True, check=False) + if result.returncode != 0 or not result.stdout.strip(): + return _summarize_test_failures_legacy(tool, bundle_path, limit=limit) + + try: + payload = json.loads(result.stdout) + except json.JSONDecodeError: + return _summarize_test_failures_legacy(tool, bundle_path, limit=limit) + + failures: list[str] = [] + tests = payload.get("tests", payload) + if isinstance(tests, list): + for item in tests: + if not isinstance(item, dict): + continue + status = str(item.get("testStatus", item.get("status", ""))).upper() + if status not in {"FAILURE", "FAILED"}: + continue + name = item.get("name") or item.get("identifier", "unknown test") + message = item.get("failureMessage") or item.get("message") + if message: + failures.append(f"{name}: {message}") + else: + failures.append(str(name)) + if len(failures) >= limit: + break + return failures + + +def _summarize_test_failures_legacy( + tool: str, + bundle_path: Path, + *, + limit: int, +) -> list[str]: + command = [ + tool, + "get", + "test-results", + "summary", + "--path", + str(bundle_path), + "--format", + "json", + ] + result = subprocess.run(command, capture_output=True, text=True, check=False) + if result.returncode != 0 or not result.stdout.strip(): + return [] + + try: + payload = json.loads(result.stdout) + except json.JSONDecodeError: + return [] + + failed_count = payload.get("failedTests", payload.get("totalFailureCount")) + if failed_count: + return [f"{failed_count} test(s) failed"] + return [] + + +def extract_screenshots_from_local_bundle( + bundle_path: Path, + output_dir: Path, + *, + only_failures: bool = False, +) -> tuple[Path, ...]: + """Extract screenshots from a local .xcresult bundle (Xcode Cloud Mac path).""" + screenshots_dir = output_dir / "screenshots" + return tuple( + extract_attachments( + bundle_path, + screenshots_dir, + only_failures=only_failures, + ) + ) diff --git a/scripts/xcode_cloud/github_comments.py b/scripts/xcode_cloud/github_comments.py new file mode 100644 index 0000000..16b48ef --- /dev/null +++ b/scripts/xcode_cloud/github_comments.py @@ -0,0 +1,385 @@ +"""Shared GitHub screenshot comment formatting and URL metadata.""" + +from __future__ import annotations + +import json +import re +from enum import Enum +from pathlib import Path +from typing import Any + +from scripts.xcode_cloud.compare import ScreenshotChange, ScreenshotComparison +from scripts.xcode_cloud.upload import UploadedScreenshot + +COMMENT_MARKER = "" +URLS_MARKER_START = "" + +TERMINAL_BUILD_STATUSES = frozenset({"success", "failed", "no_screenshots"}) + + +class BuildStatus(str, Enum): + SUCCESS = "success" + FAILED = "failed" + NO_SCREENSHOTS = "no_screenshots" + + +def github_headers(token: str) -> dict[str, str]: + return { + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + } + + +def short_sha(commit_sha: str) -> str: + return commit_sha[:7] + + +def _strip_marker_block(body: str, marker_start: str) -> str: + pattern = re.compile( + rf"\n*{re.escape(marker_start)}.*?{re.escape(MARKER_END)}", + flags=re.DOTALL, + ) + return pattern.sub("", body) + + +def _parse_marker_json(body: str, marker_start: str) -> dict[str, Any]: + match = re.search( + rf"{re.escape(marker_start)}\s*(\{{.*?\}})\s*{re.escape(MARKER_END)}", + body, + flags=re.DOTALL, + ) + if not match: + return {} + try: + payload = json.loads(match.group(1)) + except json.JSONDecodeError: + return {} + return payload if isinstance(payload, dict) else {} + + +def _embed_marker_block(body: str, marker_start: str, payload: dict[str, Any]) -> str: + stripped = _strip_marker_block(body, marker_start).rstrip() + encoded = json.dumps(payload, separators=(",", ":"), sort_keys=True) + block = f"{marker_start}\n{encoded}\n{MARKER_END}" + return f"{stripped}\n\n{block}" + + +def strip_metadata_blocks(body: str) -> str: + return _strip_marker_block(_strip_marker_block(body, STATUS_MARKER_START), URLS_MARKER_START) + + +def embed_screenshot_urls(body: str, urls: dict[str, str]) -> str: + return _embed_marker_block(body, URLS_MARKER_START, urls) + + +def strip_screenshot_urls_block(body: str) -> str: + return _strip_marker_block(body, URLS_MARKER_START) + + +def parse_screenshot_urls(body: str) -> dict[str, str]: + urls_payload = _parse_marker_json(body, URLS_MARKER_START) + if urls_payload and "status" not in urls_payload: + return {str(name): str(url) for name, url in urls_payload.items()} + + status_payload = parse_build_status_payload(body) + screenshot_urls = status_payload.get("screenshot_urls", {}) + if isinstance(screenshot_urls, dict): + return {str(name): str(url) for name, url in screenshot_urls.items()} + return {} + + +def embed_build_status(body: str, payload: dict[str, Any]) -> str: + return _embed_marker_block(body, STATUS_MARKER_START, payload) + + +def parse_build_status_payload(body: str) -> dict[str, Any]: + return _parse_marker_json(body, STATUS_MARKER_START) + + +def parse_build_status(body: str) -> BuildStatus | None: + payload = parse_build_status_payload(body) + status = payload.get("status") + if status in TERMINAL_BUILD_STATUSES: + return BuildStatus(status) + return None + + +def build_status_payload( + *, + status: BuildStatus, + build_run_id: str, + commit_sha: str | None = None, + exit_code: int | None = None, + errors: list[str] | None = None, + screenshot_urls: dict[str, str] | None = None, + baseline_commit_sha: str | None = None, +) -> dict[str, Any]: + payload: dict[str, Any] = { + "status": status.value, + "build_id": build_run_id, + } + if commit_sha: + payload["commit_sha"] = commit_sha + if exit_code is not None: + payload["exit_code"] = exit_code + if errors: + payload["errors"] = errors + if screenshot_urls: + payload["screenshot_urls"] = screenshot_urls + if baseline_commit_sha: + payload["baseline_commit_sha"] = baseline_commit_sha + return payload + + +def build_failure_comment( + *, + build_run_id: str, + commit_sha: str, + exit_code: int | None = None, + errors: list[str] | None = None, + log_excerpt: str | None = None, +) -> str: + lines = [ + COMMENT_MARKER, + "### Xcode Cloud: failed", + "", + f"Commit `{short_sha(commit_sha)}`", + f"Build `{build_run_id}`", + "", + ] + if exit_code is not None: + lines.append(f"`xcodebuild` exit code: `{exit_code}`") + lines.append("") + if errors: + lines.extend(["**Errors**", ""]) + lines.extend(f"- {error}" for error in errors) + lines.append("") + if log_excerpt: + lines.extend(["**Log excerpt**", "", "```", log_excerpt.rstrip(), "```", ""]) + lines.append("Fix the issues above and push again to re-run screenshots.") + body = "\n".join(lines).rstrip() + return embed_build_status( + body, + build_status_payload( + status=BuildStatus.FAILED, + build_run_id=build_run_id, + commit_sha=commit_sha, + exit_code=exit_code, + errors=errors or [], + ), + ) + + +def build_no_screenshots_comment( + *, + build_run_id: str, + commit_sha: str, + errors: list[str] | None = None, +) -> str: + lines = [ + COMMENT_MARKER, + "### Xcode Cloud: no screenshots", + "", + f"Commit `{short_sha(commit_sha)}`", + f"Build `{build_run_id}`", + "", + "The build finished but no screenshot PNGs were extracted from the test result bundle.", + "", + ] + if errors: + lines.extend(["**Details**", ""]) + lines.extend(f"- {error}" for error in errors) + lines.append("") + body = "\n".join(lines).rstrip() + return embed_build_status( + body, + build_status_payload( + status=BuildStatus.NO_SCREENSHOTS, + build_run_id=build_run_id, + commit_sha=commit_sha, + errors=errors or [], + ), + ) + + +def finalize_success_comment( + body: str, + *, + build_run_id: str, + commit_sha: str, + screenshot_urls: dict[str, str], + baseline_commit_sha: str | None = None, +) -> str: + body = embed_screenshot_urls(body, screenshot_urls) + return embed_build_status( + body, + build_status_payload( + status=BuildStatus.SUCCESS, + build_run_id=build_run_id, + commit_sha=commit_sha, + screenshot_urls=screenshot_urls, + baseline_commit_sha=baseline_commit_sha, + ), + ) + + +def build_screenshot_comment( + screenshot_paths: list[Path], + *, + build_run_id: str, + commit_sha: str | None = None, + baseline_commit_sha: str | None = None, + title: str = "UI screenshots", + uploaded: list[UploadedScreenshot] | None = None, + what_to_test: str | None = None, + comparisons: list[ScreenshotComparison] | None = None, + screenshot_urls: dict[str, str] | None = None, +) -> str: + if comparisons is not None: + body = _build_diff_comment( + comparisons, + build_run_id=build_run_id, + commit_sha=commit_sha, + baseline_commit_sha=baseline_commit_sha, + title=title, + what_to_test=what_to_test, + ) + elif uploaded: + lines = [ + COMMENT_MARKER, + f"### {title}", + "", + f"Commit `{short_sha(commit_sha)}`" if commit_sha else f"Build run `{build_run_id}`", + "", + ] + if what_to_test: + lines.extend(["**What to test**", "", what_to_test.strip(), ""]) + for item in uploaded: + lines.append(f"**{item.name}**") + lines.append("") + lines.append(f"![{item.name}]({item.url})") + lines.append("") + body = "\n".join(lines).rstrip() + elif not screenshot_paths: + body = ( + f"{COMMENT_MARKER}\n\n" + f"### {title}\n\n" + f"Build run `{build_run_id}` completed, but no screenshot attachments were found." + ) + else: + lines = [ + COMMENT_MARKER, + f"### {title}", + "", + f"Build run `{build_run_id}`", + "", + "Screenshots were extracted but not uploaded. Configure `IMGUR_CLIENT_ID` to embed images.", + "", + "| Screenshot |", + "| --- |", + ] + for path in screenshot_paths: + lines.append(f"| `{path.name}` |") + body = "\n".join(lines) + + if screenshot_urls and commit_sha: + return finalize_success_comment( + body, + build_run_id=build_run_id, + commit_sha=commit_sha, + screenshot_urls=screenshot_urls, + baseline_commit_sha=baseline_commit_sha, + ) + if screenshot_urls: + return embed_screenshot_urls(body, screenshot_urls) + return body + + +def _build_diff_comment( + comparisons: list[ScreenshotComparison], + *, + build_run_id: str, + commit_sha: str | None, + baseline_commit_sha: str | None, + title: str, + what_to_test: str | None, +) -> str: + changed = [item for item in comparisons if item.change == ScreenshotChange.CHANGED] + new = [item for item in comparisons if item.change == ScreenshotChange.NEW] + unchanged = [item for item in comparisons if item.change == ScreenshotChange.UNCHANGED] + removed = [item for item in comparisons if item.change == ScreenshotChange.REMOVED] + + header = f"Commit `{short_sha(commit_sha)}`" if commit_sha else f"Build run `{build_run_id}`" + lines = [ + COMMENT_MARKER, + f"### {title}", + "", + header, + "", + ] + if baseline_commit_sha: + lines.append(f"Compared to `{short_sha(baseline_commit_sha)}`") + lines.append("") + lines.append( + f"{len(changed)} changed, {len(new)} new, {len(unchanged)} unchanged, {len(removed)} removed" + ) + lines.append("") + + if what_to_test: + lines.extend(["**What to test**", "", what_to_test.strip(), ""]) + + if not changed and not new: + if baseline_commit_sha: + lines.append( + f"No screenshot changes compared to `{short_sha(baseline_commit_sha)}`." + ) + else: + lines.append("No screenshot changes detected.") + return "\n".join(lines).rstrip() + + lines.extend(["", "| Screenshot | Before | After |", "| --- | --- | --- |"]) + + for item in [*changed, *new]: + before = _image_cell(item.before_url, item.name, "before") + after = _image_cell(item.after_url, item.name, "after") + label = item.name + if item.change == ScreenshotChange.NEW: + label = f"{item.name} (new)" + lines.append(f"| {label} | {before} | {after} |") + + if removed: + lines.extend(["", "**Removed screenshots**", ""]) + for item in removed: + lines.append(f"- `{item.name}`") + + return "\n".join(lines).rstrip() + + +def _image_cell(url: str | None, name: str, role: str) -> str: + if not url: + return "—" + return f"![{name} {role}]({url})" + + +def attach_upload_urls( + comparisons: list[ScreenshotComparison], + uploads: list[UploadedScreenshot], +) -> list[ScreenshotComparison]: + upload_by_name = {item.name: item.url for item in uploads} + updated: list[ScreenshotComparison] = [] + for item in comparisons: + after_url = upload_by_name.get(item.name, item.after_url) + updated.append( + ScreenshotComparison( + name=item.name, + change=item.change, + before_path=item.before_path, + after_path=item.after_path, + before_url=item.before_url, + after_url=after_url, + ) + ) + return updated diff --git a/scripts/xcode_cloud/github_commit.py b/scripts/xcode_cloud/github_commit.py new file mode 100644 index 0000000..3f7bd79 --- /dev/null +++ b/scripts/xcode_cloud/github_commit.py @@ -0,0 +1,124 @@ +"""GitHub commit comments for screenshot reports.""" + +from __future__ import annotations + +from pathlib import Path + +import httpx + +from scripts.xcode_cloud.github_comments import COMMENT_MARKER, github_headers, parse_screenshot_urls + + +def find_commit_comment_id( + repo: str, + commit_sha: str, + *, + token: str, + client: httpx.Client | None = None, +) -> int | None: + http = client or httpx.Client(timeout=30.0) + close_client = client is None + try: + response = http.get( + f"https://api.github.com/repos/{repo}/commits/{commit_sha}/comments", + headers=github_headers(token), + ) + if response.status_code == 404: + return None + response.raise_for_status() + for comment in response.json(): + if COMMENT_MARKER in comment.get("body", ""): + return comment["id"] + return None + finally: + if close_client: + http.close() + + +def upsert_commit_comment( + repo: str, + commit_sha: str, + body: str, + *, + token: str, + client: httpx.Client | None = None, +) -> dict: + http = client or httpx.Client(timeout=30.0) + close_client = client is None + try: + comment_id = find_commit_comment_id( + repo, + commit_sha, + token=token, + client=http, + ) + headers = github_headers(token) + if comment_id is not None: + response = http.patch( + f"https://api.github.com/repos/{repo}/comments/{comment_id}", + headers=headers, + json={"body": body}, + ) + else: + response = http.post( + f"https://api.github.com/repos/{repo}/commits/{commit_sha}/comments", + headers=headers, + json={"body": body}, + ) + response.raise_for_status() + return response.json() + finally: + if close_client: + http.close() + + +def fetch_screenshot_urls_from_commit( + repo: str, + commit_sha: str, + *, + token: str, + client: httpx.Client | None = None, +) -> dict[str, str]: + http = client or httpx.Client(timeout=30.0) + close_client = client is None + try: + response = http.get( + f"https://api.github.com/repos/{repo}/commits/{commit_sha}/comments", + headers=github_headers(token), + ) + if response.status_code == 404: + return {} + response.raise_for_status() + for comment in response.json(): + if COMMENT_MARKER not in comment.get("body", ""): + continue + urls = parse_screenshot_urls(comment["body"]) + if urls: + return urls + return {} + finally: + if close_client: + http.close() + + +def download_baseline_images( + urls: dict[str, str], + destination: Path, + *, + client: httpx.Client | None = None, +) -> dict[str, str]: + http = client or httpx.Client(timeout=60.0, follow_redirects=True) + close_client = client is None + destination.mkdir(parents=True, exist_ok=True) + downloaded: dict[str, str] = {} + try: + for name, url in urls.items(): + target = destination / name + response = http.get(url) + response.raise_for_status() + target.write_bytes(response.content) + downloaded[name] = url + return downloaded + finally: + if close_client: + http.close() diff --git a/scripts/xcode_cloud/github_pr.py b/scripts/xcode_cloud/github_pr.py new file mode 100644 index 0000000..4fc58a7 --- /dev/null +++ b/scripts/xcode_cloud/github_pr.py @@ -0,0 +1,86 @@ +"""GitHub pull request screenshot comments (legacy manifest path).""" + +from __future__ import annotations + +import httpx + +from scripts.xcode_cloud.github_comments import ( + COMMENT_MARKER, + attach_upload_urls, + build_screenshot_comment, + github_headers, +) + + +def find_pr_comment_id( + repo: str, + pr_number: int, + *, + token: str, + client: httpx.Client | None = None, +) -> int | None: + http = client or httpx.Client(timeout=30.0) + close_client = client is None + try: + page = 1 + while True: + response = http.get( + f"https://api.github.com/repos/{repo}/issues/{pr_number}/comments", + params={"per_page": 100, "page": page}, + headers=github_headers(token), + ) + response.raise_for_status() + comments = response.json() + if not comments: + return None + for comment in comments: + if COMMENT_MARKER in comment.get("body", ""): + return comment["id"] + page += 1 + finally: + if close_client: + http.close() + return None + + +def upsert_pr_comment( + repo: str, + pr_number: int, + body: str, + *, + token: str, + client: httpx.Client | None = None, +) -> dict: + http = client or httpx.Client(timeout=30.0) + close_client = client is None + try: + comment_id = find_pr_comment_id(repo, pr_number, token=token, client=http) + headers = github_headers(token) + if comment_id is not None: + response = http.patch( + f"https://api.github.com/repos/{repo}/issues/comments/{comment_id}", + headers=headers, + json={"body": body}, + ) + else: + response = http.post( + f"https://api.github.com/repos/{repo}/issues/{pr_number}/comments", + headers=headers, + json={"body": body}, + ) + response.raise_for_status() + return response.json() + finally: + if close_client: + http.close() + + +def post_pr_comment( + repo: str, + pr_number: int, + body: str, + *, + token: str, + client: httpx.Client | None = None, +) -> dict: + return upsert_pr_comment(repo, pr_number, body, token=token, client=client) diff --git a/scripts/xcode_cloud/upload.py b/scripts/xcode_cloud/upload.py new file mode 100644 index 0000000..71022d0 --- /dev/null +++ b/scripts/xcode_cloud/upload.py @@ -0,0 +1,197 @@ +"""Upload extracted screenshots to a public image host.""" + +from __future__ import annotations + +import base64 +import json +import mimetypes +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Literal + +import httpx + +UploadBackend = Literal["auto", "imgur", "s3"] + + +class UploadConfigError(ValueError): + """Raised when required upload configuration is missing.""" + + +@dataclass(frozen=True) +class UploadedScreenshot: + name: str + key: str + url: str + + +def upload_backend_from_env() -> UploadBackend: + explicit = os.environ.get("SCREENSHOTS_UPLOAD_BACKEND", "auto").strip().lower() + if explicit in {"imgur", "s3"}: + return explicit # type: ignore[return-value] + if explicit != "auto": + raise UploadConfigError( + "SCREENSHOTS_UPLOAD_BACKEND must be one of: auto, imgur, s3" + ) + if os.environ.get("IMGUR_CLIENT_ID", "").strip(): + return "imgur" + if os.environ.get("SCREENSHOTS_S3_BUCKET", "").strip(): + return "s3" + raise UploadConfigError( + "No upload backend configured. Set IMGUR_CLIENT_ID (simplest) or " + "SCREENSHOTS_S3_BUCKET." + ) + + +def upload_config_from_env() -> dict[str, str]: + bucket = os.environ.get("SCREENSHOTS_S3_BUCKET", "").strip() + if not bucket: + raise UploadConfigError("SCREENSHOTS_S3_BUCKET is required for S3 upload") + + public_base_url = os.environ.get("SCREENSHOTS_PUBLIC_BASE_URL", "").strip() + if not public_base_url: + region = os.environ.get("SCREENSHOTS_S3_REGION", "us-east-1").strip() + public_base_url = f"https://{bucket}.s3.{region}.amazonaws.com" + + return { + "bucket": bucket, + "prefix": os.environ.get("SCREENSHOTS_S3_PREFIX", "xcode-cloud-screenshots").strip("/"), + "public_base_url": public_base_url.rstrip("/"), + "region": os.environ.get("SCREENSHOTS_S3_REGION", "us-east-1").strip(), + "endpoint_url": os.environ.get("SCREENSHOTS_S3_ENDPOINT_URL", "").strip() or None, + } + + +def object_key(prefix: str, build_id: str, filename: str) -> str: + safe_name = Path(filename).name + return "/".join(part for part in (prefix, build_id, safe_name) if part) + + +def public_url_for_key(public_base_url: str, key: str) -> str: + return f"{public_base_url}/{key.lstrip('/')}" + + +def _s3_client(region: str, endpoint_url: str | None): + try: + import boto3 + except ImportError as error: + raise RuntimeError( + "boto3 is required for S3 upload. Install with: python3 -m pip install boto3" + ) from error + + return boto3.client("s3", region_name=region, endpoint_url=endpoint_url) + + +def upload_to_imgur( + screenshot: Path, + *, + client_id: str, + http_client: httpx.Client | None = None, +) -> UploadedScreenshot: + """Upload one image anonymously to Imgur using only a Client-ID.""" + http = http_client or httpx.Client(timeout=60.0) + close_client = http_client is None + try: + response = http.post( + "https://api.imgur.com/3/image", + headers={"Authorization": f"Client-ID {client_id}"}, + data={"image": base64.b64encode(screenshot.read_bytes()).decode("ascii")}, + ) + response.raise_for_status() + payload = response.json() + if not payload.get("success"): + raise RuntimeError(f"Imgur upload failed: {payload}") + data = payload["data"] + return UploadedScreenshot( + name=screenshot.name, + key=data.get("id", screenshot.name), + url=data["link"], + ) + finally: + if close_client: + http.close() + + +def upload_to_s3( + screenshot: Path, + *, + build_id: str, + bucket: str, + prefix: str, + public_base_url: str, + region: str = "us-east-1", + endpoint_url: str | None = None, +) -> UploadedScreenshot: + key = object_key(prefix, build_id, screenshot.name) + content_type = mimetypes.guess_type(screenshot.name)[0] or "image/png" + extra_args = { + "ContentType": content_type, + "CacheControl": "public, max-age=31536000, immutable", + } + if os.environ.get("SCREENSHOTS_S3_USE_ACL", "false").lower() == "true": + extra_args["ACL"] = "public-read" + + client = _s3_client(region, endpoint_url) + client.upload_file(str(screenshot), bucket, key, ExtraArgs=extra_args) + return UploadedScreenshot( + name=screenshot.name, + key=key, + url=public_url_for_key(public_base_url, key), + ) + + +def upload_screenshots( + screenshots_dir: Path, + *, + build_id: str, + backend: UploadBackend = "auto", + http_client: httpx.Client | None = None, +) -> list[UploadedScreenshot]: + """Upload PNG screenshots and return stable public URLs.""" + screenshots = sorted(path for path in screenshots_dir.rglob("*.png") if path.is_file()) + if not screenshots: + return [] + + resolved_backend = backend if backend != "auto" else upload_backend_from_env() + uploads: list[UploadedScreenshot] = [] + + if resolved_backend == "imgur": + client_id = os.environ.get("IMGUR_CLIENT_ID", "").strip() + if not client_id: + raise UploadConfigError("IMGUR_CLIENT_ID is required for Imgur upload") + for screenshot in screenshots: + uploads.append( + upload_to_imgur(screenshot, client_id=client_id, http_client=http_client) + ) + return uploads + + if resolved_backend == "s3": + config = upload_config_from_env() + for screenshot in screenshots: + uploads.append( + upload_to_s3( + screenshot, + build_id=build_id, + bucket=config["bucket"], + prefix=config["prefix"], + public_base_url=config["public_base_url"], + region=config["region"], + endpoint_url=config["endpoint_url"], + ) + ) + return uploads + + raise UploadConfigError(f"Unsupported upload backend: {resolved_backend}") + + +def write_manifest(path: Path, build_id: str, uploads: list[UploadedScreenshot]) -> None: + payload = { + "build_id": build_id, + "screenshots": [ + {"name": item.name, "key": item.key, "url": item.url} + for item in uploads + ], + } + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2))