Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 10 additions & 8 deletions Jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -734,7 +734,9 @@ def get_sonarqube_unresolved_issues(report_task_file){
}
}
def calculateGCOV_PREFIX_STRIP(){
return sh(returnStdout: true, label: 'configuring GCOV_PREFIX_STRIP', script:'echo "$PWD" | awk -F/ \'{c=0; for(i=1;i<=NF;i++) if($i!="") c++; print c+2}\'')
def value = sh(returnStdout: true, label: 'configuring GCOV_PREFIX_STRIP', script:'echo "$PWD" | awk -F/ \'{c=0; for(i=1;i<=NF;i++) if($i!="") c++; print c+2}\'')
echo "Calculated GCOV_PREFIX_STRIP value: ${value}"
return value
}

// *****************************************************************************
Expand Down Expand Up @@ -774,7 +776,7 @@ pipeline {
filename 'ci/docker/linux/jenkins/Dockerfile'
label 'linux && docker && x86'
additionalBuildArgs '--label=purpose=ci --build-arg PIP_EXTRA_INDEX_URL --build-arg CONAN_CENTER_PROXY_V2_URL'
args "--label=purpose=ci --label \"absoluteUrl=${currentBuild.absoluteUrl}\" --label \"JOB_NAME=${env.JOB_NAME}\" --label \"BUILD_NUMBER=${currentBuild.number}\" --mount source=sonar-cache-py3exiv2bind,target=/opt/sonar/.sonar/cache --mount source=python-tmp-py3exiv2bind,target=/tmp --tmpfs /venv:exec -e UV_PROJECT_ENVIRONMENT=/venv --tmpfs /.config:exec"
args "--label=purpose=ci --label \"absoluteUrl=${currentBuild.absoluteUrl}\" --label \"JOB_NAME=${env.JOB_NAME}\" --label \"BUILD_NUMBER=${currentBuild.number}\" --mount source=sonar-cache-py3exiv2bind,target=/opt/sonar/.sonar/cache --mount source=python-tmp-py3exiv2bind,target=/tmp --tmpfs /venv:exec -e UV_PROJECT_ENVIRONMENT=/venv --tmpfs /.config:exec --tmpfs /.tree-sitter:exec "
}
}
environment{
Expand Down Expand Up @@ -922,8 +924,8 @@ pipeline {
}
},
'Audit uv.lock File': {
catchError(buildResult: 'UNSTABLE', message: 'uv-secure found issues', stageResult: 'UNSTABLE') {
sh 'uv run uv-secure --disable-cache uv.lock'
catchError(buildResult: 'UNSTABLE', message: 'uv audit', stageResult: 'UNSTABLE') {
sh 'uv audit'
}
}
])
Expand Down Expand Up @@ -1133,7 +1135,7 @@ pipeline {
// When released, upgrade pysonar and pin pysonar again
sh(
label: 'Running Sonar Scanner',
script: 'uvx pysonar -t $token -Dsonar.projectVersion=$VERSION -Dsonar.buildString="$BUILD_TAG" -Dsonar.cfamily.cache.enabled=false -Dsonar.cfamily.threads=$(grep -c ^processor /proc/cpuinfo) -Dsonar.cfamily.compile-commands=build/build_wrapper_output_directory/compile_commands.json -Dsonar.python.coverage.reportPaths=./reports/coverage/coverage-python.xml -Dsonar.cfamily.cobertura.reportPaths=reports/coverage/coverage_cpp.xml ' + (env.CHANGE_ID ? '-Dsonar.pullrequest.key=$CHANGE_ID -Dsonar.pullrequest.base=$CHANGE_TARGET' : '-Dsonar.branch.name=$BRANCH_NAME')
script: 'uv run pysonar -t $token -Dsonar.projectVersion=$VERSION -Dsonar.buildString="$BUILD_TAG" -Dsonar.cfamily.cache.enabled=false -Dsonar.cfamily.threads=$(grep -c ^processor /proc/cpuinfo) -Dsonar.cfamily.compile-commands=build/build_wrapper_output_directory/compile_commands.json -Dsonar.python.coverage.reportPaths=./reports/coverage/coverage-python.xml -Dsonar.cfamily.cobertura.reportPaths=reports/coverage/coverage_cpp.xml ' + (env.CHANGE_ID ? '-Dsonar.pullrequest.key=$CHANGE_ID -Dsonar.pullrequest.base=$CHANGE_TARGET' : '-Dsonar.branch.name=$BRANCH_NAME')
)
}
}
Expand Down Expand Up @@ -1232,7 +1234,7 @@ pipeline {
'--mount source=python-tmp-py3exiv2bind,target=/tmp --tmpfs /venv:exec -e UV_PROJECT_ENVIRONMENT=/venv --tmpfs /tox:exec -e TOX_WORK_DIR=/tox'
){
sh( label: 'Running Tox',
script: "uv run --only-group=tox-uv --frozen -p ${version} --python-preference only-system tox run -e ${toxEnv} --runner uv-venv-lock-runner -vvv"
script: "uv run --only-group=tox-uv --frozen -p ${version} --python-preference only-system tox run -e ${toxEnv} --runner uv-venv-lock-runner --recreate -vvv"
)
}
}
Expand Down Expand Up @@ -1318,7 +1320,7 @@ pipeline {
try{
bat(label: 'Running Tox',
script: """uv python install cpython-${version}
uv run --only-group=tox-uv --frozen tox run -e ${toxEnv} --runner uv-venv-lock-runner -vv
uv run --only-group=tox-uv --frozen tox run -e ${toxEnv} --runner uv-venv-lock-runner -vv --recreate
rmdir /S /Q %TOX_WORK_DIR%
"""
)
Expand Down Expand Up @@ -1403,7 +1405,7 @@ pipeline {
docker{
image 'ghcr.io/astral-sh/uv:debian'
label 'linux && docker'
args '--mount source=python-tmp-py3exiv2bind,target=/tmp'
args "--label=purpose=ci --label \"JOB_NAME=\$JOB_NAME\" --label \"absoluteUrl=${currentBuild.absoluteUrl}\" --label \"BUILD_NUMBER=${currentBuild.number}\" --tmpfs /.config --tmpfs /.local/share:exec --tmpfs /tmp_data:exec -e UV_PROJECT_ENVIRONMENT=/tmp_data/.venv --mount source=python-tmp-py3exiv2bind,target=/tmp"
}
}
environment{
Expand Down
6 changes: 2 additions & 4 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,7 @@ Download = "https://github.com/UIUCLibrary/pyexiv2bind"
test = [
"pytest",
"pytest-bdd",
"coverage",
]
audit-dependencies = ['uv-secure']
docs = ["sphinx", "setuptools", "toml; python_version < '3.11'"]
tox = ["tox"]
tox-uv = [
Expand All @@ -45,14 +43,14 @@ dev = [
{include-group = "tox"},
{include-group = "test"},
"bandit",
"coverage",
"flake8",
"gcovr",
"importlib-metadata>=0.12; python_version < '3.8'",
"lxml",
"macholib ; sys_platform == 'darwin'",
"mypy",
"pylint",
{include-group = "audit-dependencies"},
"pylint"
]
conan =["conan>=2.0", "cmake<4.0"]
cibuildwheel = ["cibuildwheel"]
Expand Down
91 changes: 16 additions & 75 deletions scripts/build_linux_wheels.sh
Original file line number Diff line number Diff line change
@@ -1,32 +1,13 @@
#!/usr/bin/env bash

set -e
scriptDir=$(dirname -- "$(readlink -f -- "$BASH_SOURCE")")
scriptDir=$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")
PROJECT_ROOT=$(realpath "$scriptDir/..")
DEFAULT_PYTHON_VERSION="3.10"
DOCKERFILE=$(realpath "$scriptDir/resources/package_for_linux/Dockerfile")
DEFAULT_DOCKER_IMAGE_NAME="pyexiv2bind_builder"
OUTPUT_PATH="$PROJECT_ROOT/dist"

SKIP_DIRS_NAMED=(\
'venv' \
'.venv' \
'.tox' \
'.git' \
'.idea' \
'reports' \
'.mypy_cache' \
'__pycache__' \
'wheelhouse' \
'.pytest_cache' \
'py3exiv2bind.egg-info'\
'build' \
)
REMOVE_FILES_FIRST=(\
'CMakeUserPresets.json'
'conan.lock'
)

arch=$(uname -m)

case "$arch" in
Expand Down Expand Up @@ -59,15 +40,17 @@ generate_wheel(){
exit 1
esac

dockerPurposeLabel='build-wheel'
if [[ -v ci ]]; then
local dockerPurposeLabel='build-wheel'
if [ -z "${CI}" ]; then
dockerPurposeLabel='build-wheel'
else
dockerPurposeLabel='ci'
fi

docker build \
--label="purpose=$dockerPurposeLabel" \
-t $docker_image_name_to_use \
--platform=$platform \
-t "$docker_image_name_to_use" \
--platform="$platform" \
-f "$DOCKERFILE" \
--build-arg CONAN_CENTER_PROXY_V2_URL \
--build-arg PIP_EXTRA_INDEX_URL \
Expand All @@ -78,56 +61,14 @@ generate_wheel(){
"$PROJECT_ROOT"

mkdir -p "$OUTPUT_PATH"
echo "Building wheels for Python versions: ${python_versions_to_use[*]}"
CONTAINER_WORKSPACE=/tmp/workspace

COMMAND="echo 'Making a shadow copy to prevent modifying local files' && \
prune_expr=() && \
for name in "${SKIP_DIRS_NAMED[@]}"; do \
prune_expr+=(-name \"\$name\" -type d -prune -o); \
done && \
mkdir -p ${CONTAINER_WORKSPACE} && \
(cd /project/ && \
find . \"\${prune_expr[@]}\" -type d -print | while read -r dir; do \
mkdir -p \"${CONTAINER_WORKSPACE}/\$dir\"
done && \
find . \"\${prune_expr[@]}\" \( -type f -o -type l \) -print | while read -r file; do \
echo \"\$file\"
ln -sf "/project/\$file" \"${CONTAINER_WORKSPACE}/\$file\"
done) && \
for f in "${REMOVE_FILES_FIRST[@]}"; do
OFFENDING_FILE=${CONTAINER_WORKSPACE}/\$f
if [ -f \"\$OFFENDING_FILE\" ]; then
echo \"Removing copy from temporary working path to avoid issues: \$OFFENDING_FILE\";
rm \$OFFENDING_FILE;
fi; \
done && \
echo 'Removing Python cache files' && \
find ${CONTAINER_WORKSPACE} -type d -name '__pycache__' -exec rm -rf {} + && \
find ${CONTAINER_WORKSPACE} -type f -name '*.pyc' -exec rm -f {} + && \
for i in "${python_versions_to_use[@]}"; do
echo \"Creating wheel for Python version: \$i\";
uv build --python=\$i --python-preference=system --wheel --out-dir=/tmp/dist ${CONTAINER_WORKSPACE};
if [ \$? -ne 0 ]; then
echo \"Failed to build wheel for Python \$i\";
exit 1;
fi; \
done && \
echo 'Fixing up wheels' && \
auditwheel -v repair /tmp/dist/*.whl -w /dist/;
for file in /dist/*manylinux*.whl; do
auditwheel show \$file
done && \
echo 'Done'
"

docker run --rm \
--label="purpose=$dockerPurposeLabel" \
--platform=$platform \
--platform="$platform" \
-v "$PROJECT_ROOT":/project:ro \
-v $OUTPUT_PATH:/dist \
--entrypoint="/bin/bash" \
$docker_image_name_to_use \
-c "$COMMAND"
-v "$OUTPUT_PATH":/dist \
"$docker_image_name_to_use" \
build-wheel /project /dist "${python_versions_to_use[@]}"
echo "Built wheel can be found in '$OUTPUT_PATH'"
}
print_usage(){
Expand Down Expand Up @@ -227,7 +168,6 @@ while [[ "$#" -gt 0 ]]; do
echo "Unknown argument: $1"
show_help
exit 1
shift
;;
esac
done
Expand All @@ -238,13 +178,14 @@ fi

# Set default if no versions were specified
if [[ ${#python_versions[@]} -eq 0 ]]; then
python_versions=($DEFAULT_PYTHON_VERSION)
python_versions=("$DEFAULT_PYTHON_VERSION")
fi

if [[ ! -v docker_image_name ]]; then
if [[ -z "${docker_image_name}" ]]; then
docker_image_name=$DEFAULT_DOCKER_IMAGE_NAME
else
echo "Using '$docker_image_name' for the name of the Docker Image generated to build."
fi

check_args
generate_wheel $PLATFORM $docker_image_name ${python_versions[@]}
generate_wheel "$PLATFORM" "$docker_image_name" "${python_versions[@]}"
30 changes: 15 additions & 15 deletions scripts/build_mac_wheel.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,17 +14,17 @@ remove_venv(){
}

generate_venv_with_venv(){
base_python=$1
virtual_env=$2
trap "remove_venv $virtual_env" ERR SIGINT SIGTERM
$base_python -m venv $virtual_env
$virtual_env/bin/pip install --disable-pip-version-check uv
local base_python=$1
local virtual_env=$2
trap 'remove_venv "$virtual_env"' ERR SIGINT SIGTERM
$base_python -m venv "$virtual_env"
"$virtual_env/bin/pip" install --disable-pip-version-check uv
}

generate_wheel(){
uv_exec=$1
project_root=$2
python_version=$3
local uv_exec=$1
local project_root=$2
local python_version=$3

# Get the processor type
processor_type=$(uname -m)
Expand Down Expand Up @@ -53,19 +53,19 @@ generate_wheel(){
out_temp_wheels_dir=$(mktemp -d /tmp/python_wheels.XXXXXX)
output_path="./dist"
echo "Building wheel for Python $python_version on macOS $processor_type"
trap "rm -rf $out_temp_wheels_dir" ERR SIGINT SIGTERM RETURN
trap 'rm -rf "$out_temp_wheels_dir"' ERR SIGINT SIGTERM RETURN
_PYTHON_HOST_PLATFORM=$_PYTHON_HOST_PLATFORM MACOSX_DEPLOYMENT_TARGET=$MACOSX_DEPLOYMENT_TARGET ARCHFLAGS=$ARCHFLAGS $uv_exec build --wheel --out-dir=$out_temp_wheels_dir --python=$python_version $project_root
pattern="$out_temp_wheels_dir/*.whl"
files=( $pattern )
files=( "$pattern" )
undelocate_wheel="${files[0]}"

echo ""
echo "================================================================================"
echo "${undelocate_wheel} is linked to the following:"
$uv_path run --only-group=package --python=$python_version delocate-listdeps --depending "${undelocate_wheel}"
$uv_exec run --frozen --only-group=package --python="$python_version" delocate-listdeps --depending "${undelocate_wheel}"
echo ""
echo "================================================================================"
$uv_path run --only-group=package --python=$python_version delocate-wheel -w $output_path --require-archs $REQUIRED_ARCH --verbose "$undelocate_wheel"
$uv_exec run --frozen --only-group=package --python="$python_version" delocate-wheel -w $output_path --require-archs $REQUIRED_ARCH --verbose "$undelocate_wheel"
}


Expand Down Expand Up @@ -119,7 +119,7 @@ while [[ "$#" -gt 0 ]]; do
;;
esac
done
if [ -z "{uv_path+x}" ]; then
if [ -z "${uv_path+x}" ]; then
uv_path=uv
fi
if [[ ! -f "$uv_path" ]]; then
Expand All @@ -129,7 +129,7 @@ if [[ ! -f "$uv_path" ]]; then
uv_path=/tmp/uv/bin/uv
echo "installed uv: $uv_path"
else
echo "Using existing venv: $build_virtual_env"
echo "Using existing venv: $uv_path"
fi

generate_wheel $uv_path $PROJECT_ROOT $python_version
generate_wheel "$uv_path" "$PROJECT_ROOT" "$python_version"
4 changes: 2 additions & 2 deletions scripts/resources/package_for_linux/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ RUN --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
cp ${CONAN_HOME}/remotes.json /tmp/remotes.json && \
uv run --only-group conan conan remote update conan-center --url ${CONAN_CENTER_PROXY_V2_URL}; \
fi && \
uv run --only-group conan conan install /tmp --build missing && \
uv run --only-group conan conan install /tmp --build missing -c tools.graph:skip_test=True -c tools.build:skip_test=True && \
uv run --only-group conan conan cache clean "*" -b --source --build --temp && \
uv cache prune && \
if [ -f "/tmp/remotes.json" ]; then \
Expand All @@ -87,7 +87,7 @@ RUN --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
# ==============================================================================

FROM base

COPY --chmod=755 scripts/resources/package_for_linux/scripts/build-wheel.sh /usr/bin/build-wheel
ARG CONAN_USER_HOME
ARG CONAN_HOME
COPY --from=conan_builder --chmod=777 ${CONAN_HOME}/ ${CONAN_HOME}/
Expand Down
Loading
Loading