diff --git a/hack/ci/git-org-cleanup/bitbucket-repos-cleanup.sh b/hack/ci/git-org-cleanup/bitbucket-repos-cleanup.sh index 62854526e..aa974e7f8 100755 --- a/hack/ci/git-org-cleanup/bitbucket-repos-cleanup.sh +++ b/hack/ci/git-org-cleanup/bitbucket-repos-cleanup.sh @@ -10,7 +10,7 @@ set -o pipefail # Common configuration DAYS="${DAYS:-14}" -repo_name_regex="${REPO_NAME_REGEX:-^[a-z0-9-]*(python|dotnet-basic|java-quarkus|go|nodejs|java-springboot)[a-z0-9-]*(-gitops)?$}" +repo_name_regex="${REPO_NAME_REGEX:-^[a-zA-Z0-9-]*(python|dotnet-basic|java-quarkus|go|nodejs|java-springboot)[a-zA-Z0-9-]*(-gitops)?\$}" usage() { echo " @@ -85,23 +85,53 @@ bitbucket_cleanup() { echo "Cutoff date: $cutoff_date" echo "" - # Fetch repositories using scoped API token (max pagelen=100 for Bitbucket) - # Sort by updated_on (oldest first) to prioritize older repositories for cleanup - repos=$(curl -s -u "$AUTH_CREDS" -H "Accept: application/json" \ - "https://api.bitbucket.org/2.0/repositories/$BITBUCKET_WORKSPACE?q=project.key=\"$BITBUCKET_PROJECT\"&sort=updated_on&pagelen=100") - - # Check for API errors (only if .error field exists and is not null) - if echo "$repos" | jq -e '.error' >/dev/null 2>&1; then - echo "Error fetching repositories: $(echo "$repos" | jq -r '.error.message')" >&2 - return 1 - fi - - # Check if we got a valid response with values array - if ! echo "$repos" | jq -e '.values' >/dev/null 2>&1; then - echo "Error: Invalid response format from Bitbucket API" >&2 - echo "Response: $repos" - return 1 + # Fetch repositories with pagination (max pagelen=100 for Bitbucket). + # Sort by updated_on (oldest first) to prioritize older repositories for cleanup. + next_url="https://api.bitbucket.org/2.0/repositories/$BITBUCKET_WORKSPACE?q=project.key=\"$BITBUCKET_PROJECT\"&sort=updated_on&pagelen=100" + all_repos=() + while [[ -n "$next_url" ]]; do + tmp_file=$(mktemp) + http_status=$(curl -s -o "$tmp_file" -w "%{http_code}" -u "$AUTH_CREDS" -H "Accept: application/json" "$next_url") + repos_page=$(<"$tmp_file") + rm -f "$tmp_file" + + # Handle HTTP errors first. + if [[ "$http_status" -lt 200 || "$http_status" -ge 300 ]]; then + echo "Error: Bitbucket API returned HTTP $http_status" >&2 + if [[ -n "$repos_page" ]]; then + echo "Response: $repos_page" + fi + return 1 + fi + + # Check for API errors (only if .error field exists and is not null). + if echo "$repos_page" | jq -e '.error' >/dev/null 2>&1; then + echo "Error fetching repositories: $(echo "$repos_page" | jq -r '.error.message')" >&2 + return 1 + fi + + # Check if we got a valid response with values array. + if ! echo "$repos_page" | jq -e '.values' >/dev/null 2>&1; then + echo "Error: Invalid response format from Bitbucket API" >&2 + echo "Response: $repos_page" + return 1 + fi + + while IFS= read -r repo; do + all_repos+=("$repo") + done < <(echo "$repos_page" | jq -c '.values[]') + + next_url=$(echo "$repos_page" | jq -r '.next // empty') + done + + if [[ ${#all_repos[@]} -gt 0 ]]; then + repos=$(printf '%s\n' "${all_repos[@]}" | jq -s '{values: .}') + else + repos='{"values":[]}' fi + + repo_count=$(echo "$repos" | jq '.values | length') + echo "Found $repo_count repositories" # Process repositories (using process substitution to avoid subshell) while read -r repo; do diff --git a/hack/ci/git-org-cleanup/github-repos-cleanup.sh b/hack/ci/git-org-cleanup/github-repos-cleanup.sh index eea53c76e..7c29acc29 100755 --- a/hack/ci/git-org-cleanup/github-repos-cleanup.sh +++ b/hack/ci/git-org-cleanup/github-repos-cleanup.sh @@ -10,7 +10,7 @@ set -o pipefail # Common configuration DAYS="${DAYS:-14}" -repo_name_regex="${REPO_NAME_REGEX:-^[a-z0-9-]*(python|dotnet-basic|java-quarkus|go|nodejs|java-springboot)[a-z0-9-]*(-gitops)?$}" +repo_name_regex="${REPO_NAME_REGEX:-^[a-zA-Z0-9-]*(python|dotnet-basic|java-quarkus|go|nodejs|java-springboot)[a-zA-Z0-9-]*(-gitops)?\$}" usage() { echo " @@ -60,12 +60,60 @@ github_cleanup() { echo "Checking GitHub organization: $GITHUB_ORG" echo "Cutoff date: $(date -d "@$cutoff_time")" - # Fetch repositories (sorted by creation date, oldest first) - repos=$(curl -s -X GET -H "$AUTH_HEADER" "https://api.github.com/orgs/$GITHUB_ORG/repos?per_page=200&sort=created&direction=asc") - if echo "$repos" | jq -e '.status' >/dev/null 2>&1; then - echo "Error fetching repositories: $repos" >&2 - return 1 + # Fetch repositories (sorted by creation date, oldest first) with pagination. + next_url="https://api.github.com/orgs/$GITHUB_ORG/repos?per_page=100&sort=created&direction=asc" + all_repos=() + while [[ -n "$next_url" ]]; do + tmp_body=$(mktemp) + tmp_headers=$(mktemp) + http_status=$(curl -s -D "$tmp_headers" -o "$tmp_body" -w "%{http_code}" -X GET -H "$AUTH_HEADER" "$next_url") + repos_page=$(<"$tmp_body") + rm -f "$tmp_body" + + # Handle HTTP errors first. + if [[ "$http_status" -lt 200 || "$http_status" -ge 300 ]]; then + echo "Error: GitHub API returned HTTP $http_status" >&2 + if [[ -n "$repos_page" ]]; then + echo "Response: $repos_page" + fi + rm -f "$tmp_headers" + return 1 + fi + + if echo "$repos_page" | jq -e '.status' >/dev/null 2>&1; then + echo "Error fetching repositories: $repos_page" >&2 + rm -f "$tmp_headers" + return 1 + fi + + if ! echo "$repos_page" | jq -e 'type == "array"' >/dev/null 2>&1; then + echo "Error: Invalid response format from GitHub API" >&2 + echo "Response: ${repos_page:-}" + rm -f "$tmp_headers" + return 1 + fi + + while IFS= read -r repo; do + all_repos+=("$repo") + done < <(echo "$repos_page" | jq -c '.[]') + + link_header=$(tr -d '\r' < "$tmp_headers" | awk -F': ' 'tolower($1)=="link"{print $2}') + rm -f "$tmp_headers" + if [[ "$link_header" =~ \<([^>]*)\>\;\ rel=\"next\" ]]; then + next_url="${BASH_REMATCH[1]}" + else + next_url="" + fi + done + + if [[ ${#all_repos[@]} -gt 0 ]]; then + repos=$(printf '%s\n' "${all_repos[@]}" | jq -s '.') + else + repos='[]' fi + + repo_count=$(echo "$repos" | jq 'length') + echo "Found $repo_count repositories" # Process repositories while read -r repo; do diff --git a/hack/ci/git-org-cleanup/gitlab-repos-cleanup.sh b/hack/ci/git-org-cleanup/gitlab-repos-cleanup.sh index 058b60749..2a571ddba 100755 --- a/hack/ci/git-org-cleanup/gitlab-repos-cleanup.sh +++ b/hack/ci/git-org-cleanup/gitlab-repos-cleanup.sh @@ -11,7 +11,7 @@ set -o pipefail # Common configuration DAYS="${DAYS:-14}" -repo_name_regex="${REPO_NAME_REGEX:-^[a-z0-9-]*(python|dotnet-basic|java-quarkus|go|nodejs|java-springboot)[a-z0-9-]*(-gitops)?$}" +repo_name_regex="${REPO_NAME_REGEX:-^[a-zA-Z0-9-]*(python|dotnet-basic|java-quarkus|go|nodejs|java-springboot)[a-zA-Z0-9-]*(-gitops)?\$}" usage() { echo " @@ -65,15 +65,57 @@ gitlab_cleanup() { echo "Cutoff date: $cutoff_date" echo "" - # Fetch projects from group (sorted by creation date, oldest first) - projects=$(curl -s -H "$AUTH_HEADER" \ - "$GITLAB_URL/api/v4/groups/$GITLAB_GROUP/projects?per_page=200&order_by=created_at&sort=asc") - - if echo "$projects" | jq -e '.message or .error' 2>/dev/null ; then - echo "Error fetching projects: $projects" >&2 - return 1 + # Fetch all projects from group with pagination. + next_page=1 + all_projects=() + while [[ -n "$next_page" ]]; do + api_url="$GITLAB_URL/api/v4/groups/$GITLAB_GROUP/projects?per_page=100&order_by=name&sort=asc&page=$next_page" + tmp_body=$(mktemp) + tmp_headers=$(mktemp) + http_status=$(curl -s -D "$tmp_headers" -o "$tmp_body" -w "%{http_code}" -H "$AUTH_HEADER" "$api_url") + projects_page=$(<"$tmp_body") + rm -f "$tmp_body" + + # Handle HTTP errors first. + if [[ "$http_status" -lt 200 || "$http_status" -ge 300 ]]; then + echo "Error: GitLab API returned HTTP $http_status" >&2 + if [[ -n "$projects_page" ]]; then + echo "Response: $projects_page" + fi + rm -f "$tmp_headers" + return 1 + fi + + if echo "$projects_page" | jq -e '.message or .error' >/dev/null 2>&1; then + echo "Error fetching projects: $projects_page" >&2 + rm -f "$tmp_headers" + return 1 + fi + + if ! echo "$projects_page" | jq -e 'type == "array"' >/dev/null 2>&1; then + echo "Error: Invalid response format from GitLab API" >&2 + echo "Response: ${projects_page:-}" + rm -f "$tmp_headers" + return 1 + fi + + while IFS= read -r project; do + all_projects+=("$project") + done < <(echo "$projects_page" | jq -c '.[]') + + next_page=$(tr -d '\r' < "$tmp_headers" | awk -F': ' 'tolower($1)=="x-next-page"{print $2}') + rm -f "$tmp_headers" + done + + if [[ ${#all_projects[@]} -gt 0 ]]; then + projects=$(printf '%s\n' "${all_projects[@]}" | jq -s '.') + else + projects='[]' fi - + + project_count=$(echo "$projects" | jq 'length') + echo "Found $project_count repositories" + # Process projects (using process substitution to avoid subshell) while read -r project; do project_name=$(echo "$project" | jq -r '.name' 2>/dev/null) diff --git a/hack/ci/image-org-cleanup/artifactory-image-cleanup.sh b/hack/ci/image-org-cleanup/artifactory-image-cleanup.sh new file mode 100755 index 000000000..defd0d4b6 --- /dev/null +++ b/hack/ci/image-org-cleanup/artifactory-image-cleanup.sh @@ -0,0 +1,218 @@ +#!/bin/bash + +# JFrog Artifactory image cleanup script +# Run with --help for getting usage: +# ./artifactory-image-cleanup.sh --help + +set -o errexit +set -o nounset +set -o pipefail + +# Common configuration +DAYS="${DAYS:-14}" +repo_name_regex="${REPO_NAME_REGEX:-^[a-zA-Z0-9-]*(python|dotnet-basic|java-quarkus|go|nodejs|java-springboot)[a-zA-Z0-9-]*(-gitops)?\$}" + +usage() { + echo " +Usage: + ${0##*/} [options] + +Required environment variables: + ARTIFACTORY_URL - Artifactory server URL + ARTIFACTORY_REPOSITORY - Artifactory repository name + ARTIFACTORY_API_TOKEN - Artifactory API token + +Optional environment variables: + DAYS - Number of days old images must be to qualify for cleanup (default: 14) + REPO_NAME_REGEX - Regex pattern to match image folder names for cleanup (default matches names created by tssc-test) + +Optional arguments: + -d, --dry-run Enable dry run mode - no actual deletions + -h, --help Show this help message + +Examples: + # Dry run for Artifactory cleanup (uses default URL and repository) + ARTIFACTORY_API_TOKEN=xxx ${0##*/} --dry-run + + # Actually delete images + ARTIFACTORY_API_TOKEN=xxx ${0##*/} +" +} + +artifactory_cleanup() { + export ARTIFACTORY_URL="${ARTIFACTORY_URL:-$(cat /usr/local/rhtap-cli-install/artifactory-url 2>/dev/null || echo "")}" + export ARTIFACTORY_REPOSITORY="${ARTIFACTORY_REPOSITORY:-rhtap}" + export ARTIFACTORY_API_TOKEN="${ARTIFACTORY_API_TOKEN:-$(cat /usr/local/rhtap-cli-install/artifactory-token 2>/dev/null || echo "")}" + + # Validate required environment variables + if [[ -z "$ARTIFACTORY_API_TOKEN" || -z "$ARTIFACTORY_URL" ]]; then + echo "Error: Required environment variables are not set" >&2 + echo "Please set:" + echo " - ARTIFACTORY_API_TOKEN (Artifactory API token)" + echo " - ARTIFACTORY_URL (Artifactory server URL)" + exit 1 + fi + + AUTH_HEADER="Authorization: Bearer $ARTIFACTORY_API_TOKEN" + + # Calculate cutoff time + now=$(date +%s) + cutoff_time=$((now - DAYS * 24 * 60 * 60)) + cutoff_date=$(date -d "@$cutoff_time" --iso-8601) + + echo "Checking Artifactory repository: $ARTIFACTORY_REPOSITORY" + echo "Artifactory URL: $ARTIFACTORY_URL" + echo "Cutoff date: $cutoff_date" + echo "Image name regex: $repo_name_regex" + echo "" + + # Use AQL (Artifactory Query Language) with pagination to fetch all results. + api_url="$ARTIFACTORY_URL/artifactory/api/search/aql" + page_size="${ARTIFACTORY_PAGE_SIZE:-200}" + offset=0 + all_items=() + + while :; do + aql_query='items.find({"repo":"'"$ARTIFACTORY_REPOSITORY"'","type":"folder","depth":1}).include("name","created","modified").offset('"$offset"').limit('"$page_size"')' + + # Use a temp file to capture body, and get status code separately. + tmp_file=$(mktemp) + + http_status=$(curl -s -o "$tmp_file" -w "%{http_code}" \ + -H "$AUTH_HEADER" \ + -H "Content-Type: text/plain" \ + -X POST \ + -d "$aql_query" \ + "$api_url") + + items_page=$(<"$tmp_file") + rm -f "$tmp_file" + + # Handle HTTP errors first. + if [[ "$http_status" -eq 401 ]]; then + echo "Error: Authentication Failed (401). Please check your Artifactory credentials." >&2 + return 1 + elif [[ "$http_status" -eq 403 ]]; then + echo "Error: Forbidden (403). Your credentials don't have access to this repository." >&2 + return 1 + elif [[ "$http_status" -eq 404 ]]; then + echo "Error: Repository or API endpoint not found (404)." >&2 + return 1 + elif [[ "$http_status" -lt 200 || "$http_status" -ge 300 ]]; then + echo "Error: API returned HTTP $http_status" >&2 + if [[ -n "$items_page" ]]; then + echo "Response: $items_page" + fi + return 1 + fi + + # Check if we got a valid response with results array. + if ! echo "$items_page" | jq -e '.results' >/dev/null 2>&1; then + echo "Error: Invalid response format from Artifactory API" >&2 + echo "Response: ${items_page:-}" + return 1 + fi + + while IFS= read -r item; do + all_items+=("$item") + done < <(echo "$items_page" | jq -c '.results[]') + + page_count=$(echo "$items_page" | jq '.results | length') + if [[ "$page_count" -lt "$page_size" ]]; then + break + fi + + offset=$((offset + page_size)) + done + + if [[ ${#all_items[@]} -gt 0 ]]; then + items=$(printf '%s\n' "${all_items[@]}" | jq -s '{results: .}') + else + items='{"results":[]}' + fi + + item_count=$(echo "$items" | jq '.results | length') + echo "Found $item_count items" + + # Process items + while read -r item; do + item_name=$(echo "$item" | jq -r '.name') + + if [[ ! "$item_name" =~ $repo_name_regex ]]; then + continue + fi + + item_modified=$(echo "$item" | jq -r '.modified // .created // empty') + + # Skip if no modified date + if [[ -z "$item_modified" || "$item_modified" == "null" ]]; then + continue + fi + + # Convert modified time to Unix timestamp (Artifactory uses ISO 8601 format) + item_modified_time=$(date -d "$item_modified" +%s 2>/dev/null || echo "0") + + # Image name matched regex and is old enough + if [[ $item_modified_time -lt $cutoff_time ]]; then + if [[ "$dry_run" == "true" ]]; then + echo "[DRY RUN] Would delete item '$item_name'. Last modified: $item_modified" + else + echo "Deleting item '$item_name'. Last modified: $item_modified" + delete_url="$ARTIFACTORY_URL/artifactory/$ARTIFACTORY_REPOSITORY/$item_name" + + http_code=$(curl -s -o /dev/null -w "%{http_code}" -X DELETE \ + -H "$AUTH_HEADER" \ + "$delete_url") + + if [[ "$http_code" -eq 204 || "$http_code" -eq 200 ]]; then + echo "Item '$item_name' deleted successfully." + else + echo "Failed to delete item '$item_name': HTTP $http_code" + fi + fi + fi + done <<< "$(echo "$items" | jq -c '.results[]')" +} + +parse_args() { + # Default configuration + dry_run="false" + + # Parse command line arguments + while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) + usage + exit 0 + ;; + -d|--dry-run) + dry_run="true" + shift + ;; + *) + echo "Error: Invalid command syntax" >&2 + usage + exit 1 + ;; + esac + done +} + +main() { + parse_args "$@" + + echo "===================================" + echo "Artifactory Repository Cleanup Script" + echo "===================================" + echo "Dry run: $dry_run" + echo "Days threshold: $DAYS" + echo "" + + artifactory_cleanup + + echo "Artifactory cleanup completed!" +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + main "$@" +fi diff --git a/hack/ci/image-org-cleanup/nexus-image-cleanup.sh b/hack/ci/image-org-cleanup/nexus-image-cleanup.sh new file mode 100755 index 000000000..9aa2d5b31 --- /dev/null +++ b/hack/ci/image-org-cleanup/nexus-image-cleanup.sh @@ -0,0 +1,213 @@ +#!/bin/bash + +# Nexus image cleanup script +# Run with --help for getting usage: +# ./nexus-image-cleanup.sh --help + +set -o errexit +set -o nounset +set -o pipefail + +# Common configuration +DAYS="${DAYS:-14}" +# Allow / in names (e.g. rhtap/go-rtbcsyes); same stacks as other cleanup scripts. +repo_name_regex="${REPO_NAME_REGEX:-^[a-zA-Z0-9/-]*(python|dotnet-basic|java-quarkus|go|nodejs|java-springboot)[a-zA-Z0-9/-]*(-gitops)?\$}" + +usage() { + echo " +Usage: + ${0##*/} [options] + +Required environment variables: + NEXUS_USERNAME - Nexus username (default: admin) + NEXUS_PASSWORD - Nexus password + NEXUS_URL - Nexus server URL (default: https://nexus-ui-nexus.apps.rosa.rhtap-services.xmdt.p3.openshiftapps.com) + NEXUS_REPOSITORY - Nexus repository name (default: rhtap) + +Optional environment variables: + DAYS - Number of days old images must be to qualify for cleanup (default: 14) + REPO_NAME_REGEX - Regex pattern to match component names for cleanup (default matches names created by tssc-test) + +Optional arguments: + -d, --dry-run Enable dry run mode - no actual deletions + -h, --help Show this help message + +Examples: + # Dry run for Nexus cleanup (uses defaults for URL and repository) + NEXUS_PASSWORD=xxx ${0##*/} --dry-run + + # Actually delete images + NEXUS_PASSWORD=xxx ${0##*/} +" +} + +nexus_cleanup() { + export NEXUS_USERNAME="${NEXUS_USERNAME:-admin}" + export NEXUS_PASSWORD="${NEXUS_PASSWORD:-$(cat /usr/local/rhtap-cli-install/nexus-password 2>/dev/null || echo "")}" + export NEXUS_URL="${NEXUS_URL:-$(cat /usr/local/rhtap-cli-install/nexus-ui-url 2>/dev/null || echo "")}" + export NEXUS_REPOSITORY="${NEXUS_REPOSITORY:-rhtap}" + + # Validate required environment variables + if [[ -z "$NEXUS_USERNAME" || -z "$NEXUS_PASSWORD" || -z "$NEXUS_URL" ]]; then + echo "Error: Required environment variables are not set" >&2 + echo "Please set:" + echo " - NEXUS_USERNAME (Nexus username)" + echo " - NEXUS_PASSWORD (Nexus password)" + echo " - NEXUS_URL (Nexus server URL)" + exit 1 + fi + + AUTH_CREDS="$NEXUS_USERNAME:$NEXUS_PASSWORD" + + # Calculate cutoff time + now=$(date +%s) + cutoff_time=$((now - DAYS * 24 * 60 * 60)) + cutoff_date=$(date -d "@$cutoff_time" --iso-8601) + + echo "Checking Nexus repository: $NEXUS_REPOSITORY" + echo "Nexus URL: $NEXUS_URL" + echo "Cutoff date: $cutoff_date" + echo "" + + # Fetch components from repository using Nexus REST API with pagination. + next_token="" + all_components=() + + while :; do + api_url="$NEXUS_URL/service/rest/v1/components?repository=$NEXUS_REPOSITORY" + if [[ -n "$next_token" ]]; then + api_url="$api_url&continuationToken=$next_token" + fi + + # Use a temp file to capture body, and get status code separately. + tmp_file=$(mktemp) + http_status=$(curl -s -o "$tmp_file" -w "%{http_code}" -u "$AUTH_CREDS" "$api_url") + components_page=$(<"$tmp_file") + rm -f "$tmp_file" + + # Handle HTTP errors first. + if [[ "$http_status" -eq 401 ]]; then + echo "Error: Authentication Failed (401). Please check your Nexus credentials." >&2 + return 1 + elif [[ "$http_status" -eq 403 ]]; then + echo "Error: Forbidden (403). Your credentials don't have access to this repository." >&2 + return 1 + elif [[ "$http_status" -eq 404 ]]; then + echo "Error: Repository not found (404). Please check NEXUS_REPOSITORY name." >&2 + return 1 + elif [[ "$http_status" -lt 200 || "$http_status" -ge 300 ]]; then + echo "Error: API returned HTTP $http_status" >&2 + if [[ -n "$components_page" ]]; then + echo "Response: $components_page" + fi + return 1 + fi + + # Check if we got a valid response with items array. + if ! echo "$components_page" | jq -e '.items' >/dev/null 2>&1; then + echo "Error: Invalid response format from Nexus API" >&2 + echo "Response: ${components_page:-}" + return 1 + fi + + while IFS= read -r component; do + all_components+=("$component") + done < <(echo "$components_page" | jq -c '.items[]') + + next_token=$(echo "$components_page" | jq -r '.continuationToken // empty') + if [[ -z "$next_token" ]]; then + break + fi + done + + if [[ ${#all_components[@]} -gt 0 ]]; then + components=$(printf '%s\n' "${all_components[@]}" | jq -s '{items: .}') + else + components='{"items":[]}' + fi + + component_count=$(echo "$components" | jq '.items | length') + echo "Found $component_count components" + + # Process components + while read -r component; do + component_name=$(echo "$component" | jq -r '.name') + component_id=$(echo "$component" | jq -r '.id') + + if [[ ! "$component_name" =~ $repo_name_regex ]]; then + continue + fi + + last_modified=$(echo "$component" | jq -r '[.assets[].lastModified // empty] | map(select(. != null and . != "")) | sort | last // empty') + + # Skip if no last_modified date + if [[ -z "$last_modified" || "$last_modified" == "null" ]]; then + continue + fi + + # Convert last_modified to Unix timestamp (Nexus uses ISO 8601 format) + last_modified_time=$(date -d "$last_modified" +%s 2>/dev/null || echo "0") + + # Component name matched regex and is old enough + if [[ $last_modified_time -lt $cutoff_time ]]; then + if [[ "$dry_run" == "true" ]]; then + echo "[DRY RUN] Would delete component '$component_name'. Last modified: $last_modified" + else + echo "Deleting component '$component_name'. Last modified: $last_modified" + delete_url="$NEXUS_URL/service/rest/v1/components/$component_id" + http_code=$(curl -s -o /dev/null -w "%{http_code}" -X DELETE \ + -u "$AUTH_CREDS" \ + "$delete_url") + + if [[ "$http_code" -eq 204 || "$http_code" -eq 200 ]]; then + echo "Component '$component_name' deleted successfully." + else + echo "Failed to delete component '$component_name': HTTP $http_code" + fi + fi + fi + done <<< "$(echo "$components" | jq -c '.items[]')" +} + +parse_args() { + # Default configuration + dry_run="false" + + # Parse command line arguments + while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) + usage + exit 0 + ;; + -d|--dry-run) + dry_run="true" + shift + ;; + *) + echo "Error: Invalid command syntax" >&2 + usage + exit 1 + ;; + esac + done +} + +main() { + parse_args "$@" + + echo "===================================" + echo "Nexus Repository Cleanup Script" + echo "===================================" + echo "Dry run: $dry_run" + echo "Days threshold: $DAYS" + echo "" + + nexus_cleanup + + echo "Nexus cleanup completed!" +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + main "$@" +fi diff --git a/hack/ci/image-org-cleanup/quay-image-cleanup.sh b/hack/ci/image-org-cleanup/quay-image-cleanup.sh new file mode 100755 index 000000000..056c3616c --- /dev/null +++ b/hack/ci/image-org-cleanup/quay-image-cleanup.sh @@ -0,0 +1,226 @@ +#!/bin/bash + +# Quay image cleanup script +# Run with --help for getting usage: +# ./quay-image-cleanup.sh --help + +set -o errexit +set -o nounset +set -o pipefail + +# Common configuration +DAYS="${DAYS:-14}" +repo_name_regex="${REPO_NAME_REGEX:-^[a-zA-Z0-9-]*(python|dotnet-basic|java-quarkus|go|nodejs|java-springboot)[a-zA-Z0-9-]*(-gitops)?\$}" + +usage() { + echo " +Usage: + ${0##*/} [options] + +Required environment variables: + QUAY_API_TOKEN - Quay API token (OAuth access token) + QUAY_ORGANIZATION - Quay organization name + +Optional environment variables: + QUAY_URL - Quay server URL (default: https://quay.io) + DAYS - Number of days old repositories must be to qualify for cleanup (default: 14) + REPO_NAME_REGEX - Regex pattern to match repository names for cleanup (default matches names created by tssc-test) + +Optional arguments: + -d, --dry-run Enable dry run mode - no actual deletions + -h, --help Show this help message + +Examples: + # Dry run for Quay cleanup + QUAY_API_TOKEN=xxx QUAY_ORGANIZATION=myorg ${0##*/} --dry-run + + # Actually delete repositories (default behavior) + QUAY_API_TOKEN=xxx QUAY_ORGANIZATION=myorg ${0##*/} +" +} + +quay_cleanup() { + export QUAY_API_TOKEN="${QUAY_API_TOKEN:-$(cat /usr/local/rhtap-cli-install/quay-api-token 2>/dev/null || echo "")}" + export QUAY_ORGANIZATION="${QUAY_ORGANIZATION:-rhtap_qe}" + export QUAY_URL="${QUAY_URL:-https://quay.io}" + + # Validate required environment variables + if [[ -z "$QUAY_API_TOKEN" || -z "$QUAY_ORGANIZATION" ]]; then + echo "Error: Required environment variables are not set" >&2 + echo "Please set:" + echo " - QUAY_API_TOKEN (Quay API token)" + echo " - QUAY_ORGANIZATION (Quay organization name)" + exit 1 + fi + + AUTH_HEADER="Authorization: Bearer $QUAY_API_TOKEN" + + # Calculate cutoff time + now=$(date +%s) + cutoff_time=$((now - DAYS * 24 * 60 * 60)) + cutoff_date=$(date -d "@$cutoff_time" --iso-8601) + + echo "Checking Quay organization: $QUAY_ORGANIZATION" + echo "Quay URL: $QUAY_URL" + echo "Cutoff date: $cutoff_date" + echo "Repository name regex: $repo_name_regex" + echo "" + + # Fetch repositories from organization with pagination support. + next_page="" + all_repos=() + + while :; do + api_url="$QUAY_URL/api/v1/repository?namespace=$QUAY_ORGANIZATION" + if [[ -n "$next_page" ]]; then + api_url="$api_url&next_page=$next_page" + fi + + # Use a temp file to capture body, and get status code separately. + tmp_file=$(mktemp) + http_status=$(curl -s -o "$tmp_file" -w "%{http_code}" -H "$AUTH_HEADER" "$api_url") + repos=$(<"$tmp_file") + rm -f "$tmp_file" + + # Handle HTTP errors first. + if [[ "$http_status" -eq 401 ]]; then + echo "Error: Authentication Failed (401). Please check your Quay token." >&2 + return 1 + elif [[ "$http_status" -eq 403 ]]; then + echo "Error: Forbidden (403). Your token doesn't have access to this organization." >&2 + return 1 + elif [[ "$http_status" -eq 404 ]]; then + echo "Error: Organization not found (404). Please check QUAY_ORGANIZATION name." >&2 + return 1 + elif [[ "$http_status" -lt 200 || "$http_status" -ge 300 ]]; then + echo "Error: API returned HTTP $http_status" >&2 + if [[ -n "$repos" ]]; then + echo "Response: $repos" + fi + return 1 + fi + + # Check for API errors. + if echo "$repos" | jq -e '.error_message' >/dev/null 2>&1; then + echo "Error fetching repositories: $(echo "$repos" | jq -r '.error_message')" >&2 + return 1 + fi + + # Check if we got a valid response with repositories array. + if ! echo "$repos" | jq -e '.repositories' >/dev/null 2>&1; then + echo "Error: Invalid response format from Quay API" >&2 + echo "Response: ${repos:-}" + return 1 + fi + + # Append current page repositories into the accumulator. + while IFS= read -r repo; do + all_repos+=("$repo") + done < <(echo "$repos" | jq -c '.repositories[]') + + next_page=$(echo "$repos" | jq -r '.next_page // empty') + if [[ -z "$next_page" ]]; then + break + fi + done + + if [[ ${#all_repos[@]} -gt 0 ]]; then + repos=$(printf '%s\n' "${all_repos[@]}" | jq -s '{repositories: .}') + else + repos='{"repositories":[]}' + fi + + repo_count=$(echo "$repos" | jq '.repositories | length') + echo "Found $repo_count repositories" + + # Process repositories + while read -r repo; do + repo_name=$(echo "$repo" | jq -r '.name') + + if [[ ! "$repo_name" =~ $repo_name_regex ]]; then + continue + fi + + # Fetch repository details to get tags (list API doesn't include last_modified) + repo_detail_url="$QUAY_URL/api/v1/repository/$QUAY_ORGANIZATION/$repo_name" + repo_detail=$(curl -s -H "$AUTH_HEADER" "$repo_detail_url") + + # Get the most recent tag's last_modified date + last_modified=$(echo "$repo_detail" | jq -r '[.tags[].last_modified // empty] | map(select(. != null and . != "")) | sort | last // empty') + + # Skip if no last_modified date found in any tag + if [[ -z "$last_modified" || "$last_modified" == "null" ]]; then + continue + fi + + # Convert RFC 2822 date format to Unix timestamp + # Format: "Mon, 09 Jun 2025 11:17:19 -0000" + last_modified_time=$(date -d "$last_modified" +%s 2>/dev/null || echo "0") + last_modified_date=$(date -d "$last_modified" --iso-8601 2>/dev/null || echo "$last_modified") + + # Check if old enough + if [[ $last_modified_time -ge $cutoff_time ]]; then + continue + fi + + # Repository name matched regex and is old enough + if [[ "$dry_run" == "true" ]]; then + echo "[DRY RUN] Would delete repository '$repo_name'. Last modified: $last_modified_date" + else + echo "Deleting repository '$repo_name'. Last modified: $last_modified_date" + delete_url="$QUAY_URL/api/v1/repository/$QUAY_ORGANIZATION/$repo_name" + http_code=$(curl -s -o /dev/null -w "%{http_code}" -X DELETE \ + -H "$AUTH_HEADER" \ + "$delete_url") + + if [[ "$http_code" -eq 204 || "$http_code" -eq 200 ]]; then + echo "Repository '$repo_name' deleted successfully." + else + echo "Failed to delete repository '$repo_name': HTTP $http_code" + fi + fi + done <<< "$(echo "$repos" | jq -c '.repositories[]')" +} + +parse_args() { + # Default configuration + dry_run="false" + + # Parse command line arguments + while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) + usage + exit 0 + ;; + -d|--dry-run) + dry_run="true" + shift + ;; + *) + echo "Error: Invalid command syntax" >&2 + usage + exit 1 + ;; + esac + done +} + +main() { + parse_args "$@" + + echo "===================================" + echo "Quay Repository Cleanup Script" + echo "===================================" + echo "Dry run: $dry_run" + echo "Days threshold: $DAYS" + echo "" + + quay_cleanup + + echo "Quay cleanup completed!" +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + main "$@" +fi diff --git a/hack/ci/service-jenkins-cleanup/jenkins-cleanup.sh b/hack/ci/service-jenkins-cleanup/jenkins-cleanup.sh new file mode 100755 index 000000000..ca8aafb63 --- /dev/null +++ b/hack/ci/service-jenkins-cleanup/jenkins-cleanup.sh @@ -0,0 +1,302 @@ +#!/bin/bash + +# Jenkins cleanup script +# Run with --help for getting usage: +# ./jenkins-cleanup.sh --help + +set -o errexit +set -o nounset +set -o pipefail + +# Common configuration +DAYS="${DAYS:-14}" + +usage() { + echo " +Usage: + ${0##*/} [options] + +Required environment variables: + JENKINS_API_TOKEN - Jenkins API token + JENKINS_URL - Jenkins server URL + JENKINS_USERNAME - Jenkins username + +Optional environment variables: + DAYS - Number of days old jobs must be to qualify for cleanup (default: 14) + +Optional arguments: + -d, --dry-run Enable dry run mode - no actual deletions + -e, --empty-folders Delete empty folders + -n, --no-builds Delete jobs with no builds + -v, --verbose Enable verbose output + -h, --help Show this help message + +Examples: + # Dry run for Jenkins cleanup + JENKINS_API_TOKEN=xxx JENKINS_URL=https://jenkins.example.com JENKINS_USERNAME=admin ${0##*/} --dry-run + + # Actually delete jobs + JENKINS_API_TOKEN=xxx JENKINS_URL=https://jenkins.example.com JENKINS_USERNAME=admin ${0##*/} + + # Delete empty folders and jobs with no builds + JENKINS_API_TOKEN=xxx JENKINS_URL=https://jenkins.example.com JENKINS_USERNAME=admin ${0##*/} --empty-folders --no-builds +" +} + +# Global variables for folder processing +indent="" +delete_folder="true" +builds="false" +folder="false" +last_mod=0 +check_date=0 + +item_cleanup() { + local name="$1" + local url="$2" + + # If NO_builds set, Force deletion if no builds by setting builds to true + if [[ "${folder}" != "true" ]] && [[ "${no_builds}" == "true" ]] && [[ "${builds}" == "false" ]]; then + builds="true" + if [[ "$verbose" == "true" ]]; then + echo "${indent}No Builds - Flag set deleting" + fi + fi + + # Clean up item according to settings + if [[ "$delete_folder" == "true" ]] && { [[ "$builds" == "true" ]] || [[ "$folder" == "true" ]]; }; then + local mod_date="" + if [[ "$last_mod" != "0" ]]; then + mod_date=$(TZ=UTC date -d @"$((last_mod/1000))") + fi + if [[ "$dry_run" == "false" ]]; then + printf "%-10s %-60s %-40s\n" "Deleting" "${name}" "${mod_date}" + curl -s -X POST -u "$JENKINS_USERNAME:$JENKINS_API_TOKEN" "${url}doDelete" + else + printf "%-10s %-60s %-40s\n" "[DRY RUN]" "${name}" "${mod_date}" + fi + else + if [[ "$verbose" == "true" ]]; then + echo "${indent}Skipping $name" + fi + fi +} + +process_workflow() { + local workflow_url="$1" + indent="${indent} " + + # Get list of builds and their timestamp + local build_list + build_list=$(curl -s -X POST -L -u "$JENKINS_USERNAME:$JENKINS_API_TOKEN" "${workflow_url}/api/json?tree=builds[number,timestamp]" --globoff | jq -r '.builds') + + local items + items=$(echo "$build_list" | jq length) + + # If no builds skip directory - Do not delete + if [[ "${items}" == "0" ]] && [[ "${builds}" != "true" ]]; then + builds="false" + if [[ "$verbose" == "true" ]]; then + echo "${indent}No Builds" + fi + return + fi + + # Loop through builds and if no recent builds mark for deletion + builds="true" + while read -r build_item; do + local timestamp + timestamp=$(echo "$build_item" | jq -r '.timestamp' 2>/dev/null) + + if (( timestamp > check_date )); then + delete_folder="false" + if [[ "$verbose" == "true" ]]; then + echo "${indent}Skipping - Build has recently run" + fi + break + fi + + if (( timestamp > last_mod )); then + last_mod=$timestamp + fi + done < <(echo "$build_list" | jq -c '.[]') +} + +process_folder() { + local folder_url="$1" + indent="${indent} " + + # Get list of entries in directory and process + local sub_dirs + sub_dirs=$(curl -s --globoff -X POST -L -u "$JENKINS_USERNAME:$JENKINS_API_TOKEN" "${folder_url}api/json?tree=jobs[name,url]" | jq -r '.jobs') + + local items + items=$(echo "$sub_dirs" | jq length) + + # If empty dir/folder - handle based on empty_folders flag + if [[ "${items}" == "0" ]]; then + if [[ "$empty_folders" == "true" ]]; then + folder="true" + if [[ "$verbose" == "true" ]]; then + echo "${indent}Deleting - Empty folder" + fi + else + delete_folder="false" + if [[ "$verbose" == "true" ]]; then + echo "${indent}Skipping - Empty folder" + fi + fi + return + fi + process_list "${sub_dirs}" +} + +process_list() { + local list="$1" + local items + items=$(echo "$list" | jq length) + + if [[ "$verbose" == "true" ]]; then + echo "${indent}items: $items" + fi + + local i=0 + while read -r item; do + i=$((i+1)) + + local class name url + class=$(echo "$item" | jq -r '._class' 2>/dev/null) + name=$(echo "$item" | jq -r '.name' 2>/dev/null) + url=$(echo "$item" | jq -r '.url' 2>/dev/null) + + if [[ "${indent}" == "" ]]; then + delete_folder="true" + builds="false" + folder="false" + last_mod=0 + fi + + if [[ "$class" == "com.cloudbees.hudson.plugins.folder.Folder" ]]; then + # Process folders + if [[ "$verbose" == "true" ]]; then + echo "${indent}--- Processing item $i Folder $name ---" + fi + process_folder "${url}" + if [[ "${indent}" == " " ]]; then + item_cleanup "$name" "$url" + fi + elif [[ "$class" == "org.jenkinsci.plugins.workflow.job.WorkflowJob" ]] || \ + [[ "$class" == "hudson.model.FreeStyleProject" ]]; then + # Process workflow or freestyle Job + if [[ "$verbose" == "true" ]]; then + echo "${indent}--- Processing item $i Job $name" + fi + process_workflow "${url}" + if [[ "${indent}" == " " ]]; then + item_cleanup "$name" "$url" + fi + else + echo "${indent}WARNING: Unhandled class ${class}" + fi + + indent=${indent::-6} + done < <(echo "$list" | jq -c '.[]') +} + +jenkins_cleanup() { + export JENKINS_API_TOKEN="${JENKINS_API_TOKEN:-$(cat /usr/local/rhtap-cli-install/jenkins-api-token 2>/dev/null || echo "")}" + export JENKINS_URL="${JENKINS_URL:-$(cat /usr/local/rhtap-cli-install/jenkins-url 2>/dev/null || echo "")}" + export JENKINS_USERNAME="${JENKINS_USERNAME:-$(cat /usr/local/rhtap-cli-install/jenkins-username 2>/dev/null || echo "")}" + + # Validate required environment variables + if [[ -z "$JENKINS_API_TOKEN" || -z "$JENKINS_URL" || -z "$JENKINS_USERNAME" ]]; then + echo "Error: Required environment variables are not set" >&2 + echo "Please set:" + echo " - JENKINS_API_TOKEN (Jenkins API token)" + echo " - JENKINS_URL (Jenkins server URL)" + echo " - JENKINS_USERNAME (Jenkins username)" + exit 1 + fi + + # Calculate cutoff time + check_date=$(date -d "-${DAYS} days" +%s%3N) + + echo "Checking Jenkins server: $JENKINS_URL" + echo "Jenkins username: $JENKINS_USERNAME" + echo "Cutoff date: $(date -d "-${DAYS} days")" + echo "Empty folders: $empty_folders" + echo "No builds: $no_builds" + echo "Verbose: $verbose" + echo "" + + # Get list of top level entries in directory + local dir_list + dir_list=$(curl -s --globoff -X POST -L -u "$JENKINS_USERNAME:$JENKINS_API_TOKEN" "${JENKINS_URL}/api/json?tree=jobs[name,url]" | jq -r '.jobs') + + if [[ -z "$dir_list" || "$dir_list" == "null" ]]; then + echo "Error: Failed to fetch Jenkins jobs or no jobs found" >&2 + return 1 + fi + + # Process entries in list searching for directories that do not have builds that have run in X number of days + process_list "${dir_list}" +} + +parse_args() { + # Default configuration + dry_run="false" + empty_folders="false" + no_builds="false" + verbose="false" + + # Parse command line arguments + while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) + usage + exit 0 + ;; + -d|--dry-run) + dry_run="true" + shift + ;; + -e|--empty-folders) + empty_folders="true" + shift + ;; + -n|--no-builds) + no_builds="true" + shift + ;; + -v|--verbose) + verbose="true" + shift + ;; + *) + echo "Error: Invalid command syntax" >&2 + usage + exit 1 + ;; + esac + done +} + +main() { + parse_args "$@" + + echo "===================================" + echo "Jenkins Cleanup Script" + echo "===================================" + echo "Dry run: $dry_run" + echo "Days threshold: $DAYS" + echo "" + + jenkins_cleanup + + echo "Jenkins cleanup completed!" +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + main "$@" +fi + diff --git a/integration-tests/pipelines/periodic-cleanup-pipeline.yaml b/integration-tests/pipelines/periodic-cleanup-pipeline.yaml index a44840461..7437f3a16 100644 --- a/integration-tests/pipelines/periodic-cleanup-pipeline.yaml +++ b/integration-tests/pipelines/periodic-cleanup-pipeline.yaml @@ -25,3 +25,24 @@ spec: - hack/ci/git-org-cleanup/github-repos-cleanup.sh - hack/ci/git-org-cleanup/gitlab-repos-cleanup.sh - hack/ci/git-org-cleanup/bitbucket-repos-cleanup.sh + - hack/ci/image-org-cleanup/artifactory-image-cleanup.sh + - hack/ci/image-org-cleanup/nexus-image-cleanup.sh + - hack/ci/image-org-cleanup/quay-image-cleanup.sh + - hack/ci/service-jenkins-cleanup/jenkins-cleanup.sh + - name: cleanup-rosa-cluster + taskRef: + resolver: git + params: + - name: url + value: https://github.com/redhat-appstudio/tssc-cli.git + - name: revision + value: main + - name: pathInRepo + value: integration-tests/tasks/download-and-execute-cleanup.yaml + params: + - name: script-path + value: scripts/rosa/delete-rosa-clusters.sh + - name: source-git-repo + value: konflux-ci/tekton-integration-catalog + - name: source-git-revision + value: main diff --git a/integration-tests/tasks/download-and-execute-cleanup.yaml b/integration-tests/tasks/download-and-execute-cleanup.yaml index 32a4c74f4..08b110cd5 100644 --- a/integration-tests/tasks/download-and-execute-cleanup.yaml +++ b/integration-tests/tasks/download-and-execute-cleanup.yaml @@ -22,12 +22,19 @@ spec: - name: tssc-cli-volume secret: secretName: rhtap-cli-install + - name: konflux-test-infra-volume + secret: + secretName: konflux-test-infra steps: - name: download-and-execute-cleanup image: quay.io/konflux-ci/tekton-integration-catalog/utils:latest volumeMounts: - name: tssc-cli-volume mountPath: /usr/local/rhtap-cli-install + readOnly: true + - name: konflux-test-infra-volume + mountPath: /usr/local/konflux-test-infra + readOnly: true script: | #!/bin/bash set -euo pipefail @@ -38,6 +45,18 @@ spec: # Build Script URL from parameters SCRIPT_URL="https://raw.githubusercontent.com/$(params.source-git-repo)/$(params.source-git-revision)/$(params.script-path)" + if [[ "$SCRIPT_NAME" == "delete-rosa-clusters.sh" ]]; then + # Read credentials from mounted volume + CREDENTIALS_FILE="/usr/local/konflux-test-infra/rhtap-cloud-credentials-us-east-1" + + # Extract individual values from the JSON credentials + export ROSA_TOKEN=$(jq -r '.aws["rosa-hcp"]["rosa-token"]' "$CREDENTIALS_FILE") + export AWS_ACCESS_KEY_ID=$(jq -r '.aws["access-key-id"]' "$CREDENTIALS_FILE") + export AWS_SECRET_ACCESS_KEY=$(jq -r '.aws["access-key-secret"]' "$CREDENTIALS_FILE") + export AWS_SUBNET_IDS=$(jq -r '.aws["rosa-hcp"]["subnets-ids"]' "$CREDENTIALS_FILE") + export AWS_DEFAULT_REGION=$(jq -r '.aws["region"]' "$CREDENTIALS_FILE") + fi + # Function to prefix logs log_prefix() { while read -r line; do