Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 46 additions & 0 deletions .github/workflows/docker-build-check.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,52 @@ jobs:
tags: airbyte/source-declarative-manifest:pr-${{ github.event.pull_request.number }}
outputs: type=image,name=target,annotation-index.org.opencontainers.image.description=SDM Docker image for PR ${{ github.event.pull_request.number }}

- name: Build Docker image for bytecode check (linux/amd64)
uses: docker/build-push-action@v5
with:
context: .
platforms: linux/amd64
load: true
tags: airbyte/source-declarative-manifest:bytecode-check

- name: Verify every site-packages module has precompiled bytecode
run: |
docker run --rm -i --entrypoint python airbyte/source-declarative-manifest:bytecode-check - <<'EOF'
import importlib.util
import pathlib
import re
import struct
import sys
import sysconfig

SKIP = re.compile(r"/tests?/") # keep in sync with compileall -x in the Dockerfile

root = pathlib.Path(sysconfig.get_paths()["purelib"])
missing, stale = [], []
total = 0
for src in root.rglob("*.py"):
if SKIP.search(str(src)):
continue
total += 1
pyc = pathlib.Path(importlib.util.cache_from_source(str(src)))
if not pyc.is_file():
missing.append(src)
continue
header = pyc.read_bytes()[:16]
flags = int.from_bytes(header[4:8], "little")
if len(header) < 16 or header[:4] != importlib.util.MAGIC_NUMBER or flags != 0:
stale.append(src)
else:
mtime, size = struct.unpack("<II", header[8:16])
st = src.stat()
if (mtime, size) != (int(st.st_mtime) & 0xFFFFFFFF, st.st_size & 0xFFFFFFFF):
stale.append(src)
print(f"{root}: {total} .py files, {len(missing)} without .pyc, {len(stale)} with stale .pyc")
for path in (missing + stale)[:20]:
print(f" {path}")
sys.exit(1 if total == 0 or missing or stale else 0)
EOF

manifest-server-docker-build-check:
name: Manifest Server Docker Image Build
runs-on: ubuntu-24.04
Expand Down
6 changes: 6 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,12 @@ RUN pip install dist/*.whl
# pandas is imported lazily by the CDK, so verify it is importable at build time
RUN python -c "import pandas"

# Precompile bytecode for installed packages, skipping test directories. `poetry install` does not
# compile it, and the runtime `airbyte` user cannot write to the root-owned site-packages, so without
# this every connector process recompiles its dependencies on start.
RUN python -m compileall -q -j 0 -x '/tests?/' /usr/local/lib/python3.13/site-packages \
|| echo "WARNING: some files failed to compile (see errors above); continuing without their bytecode"

# Recreate the original structure
RUN mkdir -p source_declarative_manifest \
&& echo 'from source_declarative_manifest.run import run\n\nif __name__ == "__main__":\n run()' > main.py \
Expand Down
Loading