|
| 1 | +import json |
1 | 2 | import os |
2 | 3 | import re |
3 | 4 | import subprocess |
@@ -77,74 +78,70 @@ async def test_static_crawler_actor_at_apify( |
77 | 78 | output_dir=str(tmp_path), |
78 | 79 | ) |
79 | 80 |
|
| 81 | + project_path = tmp_path / actor_name |
80 | 82 | patch_crawlee_version_in_project( |
81 | | - project_path=tmp_path / actor_name, wheel_path=crawlee_wheel_path, package_manager=package_manager |
| 83 | + project_path=project_path, wheel_path=crawlee_wheel_path, package_manager=package_manager |
82 | 84 | ) |
83 | 85 |
|
| 86 | + # Run the CLI with its own home directory, so the secrets file it reads lives in `tmp_path`. It authenticates |
| 87 | + # through `APIFY_TOKEN` and never touches a stored login. `GIT_CEILING_DIRECTORIES` keeps git from walking up |
| 88 | + # into the surrounding project's .git/ when running under a basetemp inside the repo (see --basetemp in the |
| 89 | + # e2e-templates-tests poe task). |
| 90 | + cli_home = tmp_path / 'apify_cli_home' |
| 91 | + cli_home.mkdir() |
| 92 | + cli_env = { |
| 93 | + **os.environ, |
| 94 | + 'HOME': str(cli_home), |
| 95 | + 'APIFY_TOKEN': os.environ['APIFY_TEST_USER_API_TOKEN'], |
| 96 | + 'GIT_CEILING_DIRECTORIES': str(tmp_path), |
| 97 | + } |
| 98 | + |
| 99 | + def run_cli(*args: str) -> subprocess.CompletedProcess[bytes]: |
| 100 | + return subprocess.run( # noqa: S603 |
| 101 | + ['apify', *args], # noqa: S607 |
| 102 | + capture_output=True, |
| 103 | + check=False, |
| 104 | + cwd=project_path, |
| 105 | + env=cli_env, |
| 106 | + timeout=600, |
| 107 | + ) |
| 108 | + |
84 | 109 | # Print apify version for debugging purposes in rare cases of CLI failures |
85 | 110 | subprocess.run(['apify', '--version'], check=True) # noqa: ASYNC221, S607 |
86 | 111 |
|
87 | | - # Build actor using sequence of cli commands as the user would. |
88 | | - subprocess.run( # noqa: ASYNC221, S603 |
89 | | - ['apify', 'login', '-t', os.environ['APIFY_TEST_USER_API_TOKEN']], # noqa: S607 |
90 | | - capture_output=True, |
91 | | - check=True, |
92 | | - cwd=tmp_path / actor_name, |
93 | | - timeout=600, |
94 | | - ) |
95 | | - subprocess.run( # noqa: ASYNC221, S603 |
96 | | - ['apify', 'init', '-y', actor_name], # noqa: S607 |
97 | | - capture_output=True, |
98 | | - check=True, |
99 | | - cwd=tmp_path / actor_name, |
100 | | - timeout=600, |
101 | | - ) |
| 112 | + init_process = run_cli('init', '-y', actor_name) |
| 113 | + assert init_process.returncode == 0, f'`apify init` failed: {init_process.stderr.decode()}' |
102 | 114 |
|
103 | | - build_process = subprocess.run( # noqa: ASYNC221 |
104 | | - ['apify', 'push'], # noqa: S607 |
105 | | - capture_output=True, |
106 | | - check=False, |
107 | | - cwd=tmp_path / actor_name, |
108 | | - # Prevent git from walking up into the surrounding project's .git/ when running under |
109 | | - # a basetemp inside the repo (see --basetemp in the e2e-templates-tests poe task). |
110 | | - env={**os.environ, 'GIT_CEILING_DIRECTORIES': str(tmp_path)}, |
111 | | - timeout=600, |
112 | | - ) |
113 | | - # Get actor ID from build log |
114 | | - actor_id_regexp = re.compile(r'https:\/\/console\.apify\.com\/actors\/(.*)#\/builds\/\d*\.\d*\.\d*') |
| 115 | + # Stagehand reads the model API key from the environment at run time. `apify push` sets the env vars from |
| 116 | + # `actor.json` on the Actor version before it starts the build, and sends `@`-prefixed values as secrets. |
| 117 | + if crawler_type == 'stagehand': |
| 118 | + secrets_process = run_cli('secrets', 'add', 'openaiApiKey', os.environ['OPENAI_API_KEY']) |
| 119 | + assert secrets_process.returncode == 0, f'`apify secrets add` failed: {secrets_process.stderr.decode()}' |
115 | 120 |
|
116 | | - if match := re.findall(actor_id_regexp, build_process.stderr.decode()): |
117 | | - actor_id = match[0] |
118 | | - else: |
119 | | - raise AssertionError(f'Failed to find actor id in build log: {build_process.stderr.decode()}') |
| 121 | + actor_json_path = project_path / '.actor' / 'actor.json' |
| 122 | + actor_json = json.loads(actor_json_path.read_text(encoding='utf-8')) |
| 123 | + actor_json['environmentVariables'] = {'OPENAI_API_KEY': '@openaiApiKey'} |
| 124 | + actor_json_path.write_text(json.dumps(actor_json, indent=4), encoding='utf-8') |
120 | 125 |
|
121 | | - client = ApifyClientAsync(token=os.getenv('APIFY_TEST_USER_API_TOKEN')) |
122 | | - actor = client.actor(actor_id) |
| 126 | + # With `--json`, `apify push` prints the push result to stdout and the build log to stderr. |
| 127 | + push_process = run_cli('push', '--json') |
| 128 | + try: |
| 129 | + push_result = json.loads(push_process.stdout) |
| 130 | + except json.JSONDecodeError as exc: |
| 131 | + raise AssertionError( |
| 132 | + f'`apify push` did not print a JSON result.\n' |
| 133 | + f'stdout: {push_process.stdout.decode()}\n' |
| 134 | + f'stderr: {push_process.stderr.decode()}' |
| 135 | + ) from exc |
| 136 | + |
| 137 | + client = ApifyClientAsync(token=os.environ['APIFY_TEST_USER_API_TOKEN']) |
| 138 | + actor = client.actor(push_result['actor']['id']) |
123 | 139 |
|
124 | 140 | # Run actor |
125 | 141 | try: |
126 | | - assert build_process.returncode == 0 |
127 | | - |
128 | | - # Stagehand needs the model API key in `os.environ` at run time. Env vars are baked into the |
129 | | - # build at build creation time, so registering the env var on the version after `apify push` |
130 | | - # is not enough — we must trigger a fresh build for the env var to be included. |
131 | | - build_number: str | None = None |
132 | | - if crawler_type == 'stagehand': |
133 | | - env_vars = actor.version('0.0').env_vars() |
134 | | - await env_vars.create(name='OPENAI_API_KEY', value=os.environ['OPENAI_API_KEY'], is_secret=True) |
135 | | - # `ActorClientAsync.build`'s `wait_for_finish` parameter is capped server-side at 60s, |
136 | | - # which is shorter than a stagehand build (playwright + browser deps). Trigger the build, |
137 | | - # then poll client-side via `BuildClientAsync.wait_for_finish` until it reaches a terminal |
138 | | - # status, and assert it succeeded before starting the run. |
139 | | - rebuild = await actor.build(version_number='0.0') |
140 | | - finished_build = await client.build(rebuild['id']).wait_for_finish(wait_secs=900) |
141 | | - assert finished_build is not None, 'Stagehand rebuild did not reach a terminal status within 900s.' |
142 | | - assert finished_build['status'] == 'SUCCEEDED', ( |
143 | | - f'Stagehand rebuild did not succeed: status={finished_build["status"]!r}, build={finished_build}' |
144 | | - ) |
145 | | - build_number = finished_build['buildNumber'] |
146 | | - |
147 | | - started_run_data = await actor.start(memory_mbytes=8192, build=build_number) |
| 142 | + assert push_result['ok'], f'`apify push` failed: {push_result.get("error")}' |
| 143 | + |
| 144 | + started_run_data = await actor.start(memory_mbytes=8192, build=push_result['build']['number']) |
148 | 145 | actor_run = client.run(started_run_data['id']) |
149 | 146 |
|
150 | 147 | finished_run_data = await actor_run.wait_for_finish() |
|
0 commit comments