Skip to content

Add image-org-cleanup to periodic pipeline (also move rosa & jenkins cleanup) #1774

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
chetna14manku wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Add image-org-cleanup to periodic pipeline (also move rosa & jenkins cleanup) #1774

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 47 additions & 17 deletions hack/ci/git-org-cleanup/bitbucket-repos-cleanup.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ set -o pipefail

# Common configuration
DAYS="${DAYS:-14}"
repo_name_regex="${REPO_NAME_REGEX:-^[a-z0-9-]*(python|dotnet-basic|java-quarkus|go|nodejs|java-springboot)[a-z0-9-]*(-gitops)?$}"
repo_name_regex="${REPO_NAME_REGEX:-^[a-zA-Z0-9-]*(python|dotnet-basic|java-quarkus|go|nodejs|java-springboot)[a-zA-Z0-9-]*(-gitops)?\$}"

usage() {
echo "
Expand Down Expand Up @@ -85,23 +85,53 @@ bitbucket_cleanup() {
echo "Cutoff date: $cutoff_date"
echo ""

# Fetch repositories using scoped API token (max pagelen=100 for Bitbucket)
# Sort by updated_on (oldest first) to prioritize older repositories for cleanup
repos=$(curl -s -u "$AUTH_CREDS" -H "Accept: application/json" \
"https://api.bitbucket.org/2.0/repositories/$BITBUCKET_WORKSPACE?q=project.key=\"$BITBUCKET_PROJECT\"&sort=updated_on&pagelen=100")

# Check for API errors (only if .error field exists and is not null)
if echo "$repos" | jq -e '.error' >/dev/null 2>&1; then
echo "Error fetching repositories: $(echo "$repos" | jq -r '.error.message')" >&2
return 1
fi

# Check if we got a valid response with values array
if ! echo "$repos" | jq -e '.values' >/dev/null 2>&1; then
echo "Error: Invalid response format from Bitbucket API" >&2
echo "Response: $repos"
return 1
# Fetch repositories with pagination (max pagelen=100 for Bitbucket).
# Sort by updated_on (oldest first) to prioritize older repositories for cleanup.
next_url="https://api.bitbucket.org/2.0/repositories/$BITBUCKET_WORKSPACE?q=project.key=\"$BITBUCKET_PROJECT\"&sort=updated_on&pagelen=100"
all_repos=()
while [[ -n "$next_url" ]]; do
tmp_file=$(mktemp)
http_status=$(curl -s -o "$tmp_file" -w "%{http_code}" -u "$AUTH_CREDS" -H "Accept: application/json" "$next_url")
repos_page=$(<"$tmp_file")
rm -f "$tmp_file"

# Handle HTTP errors first.
if [[ "$http_status" -lt 200 || "$http_status" -ge 300 ]]; then
echo "Error: Bitbucket API returned HTTP $http_status" >&2
if [[ -n "$repos_page" ]]; then
echo "Response: $repos_page"
fi
return 1
fi

# Check for API errors (only if .error field exists and is not null).
if echo "$repos_page" | jq -e '.error' >/dev/null 2>&1; then
echo "Error fetching repositories: $(echo "$repos_page" | jq -r '.error.message')" >&2
return 1
fi

# Check if we got a valid response with values array.
if ! echo "$repos_page" | jq -e '.values' >/dev/null 2>&1; then
echo "Error: Invalid response format from Bitbucket API" >&2
echo "Response: $repos_page"
return 1
fi

while IFS= read -r repo; do
all_repos+=("$repo")
done < <(echo "$repos_page" | jq -c '.values[]')

next_url=$(echo "$repos_page" | jq -r '.next // empty')
done

if [[ ${#all_repos[@]} -gt 0 ]]; then
repos=$(printf '%s\n' "${all_repos[@]}" | jq -s '{values: .}')
else
repos='{"values":[]}'
fi

repo_count=$(echo "$repos" | jq '.values | length')
echo "Found $repo_count repositories"

# Process repositories (using process substitution to avoid subshell)
while read -r repo; do
Expand Down
60 changes: 54 additions & 6 deletions hack/ci/git-org-cleanup/github-repos-cleanup.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ set -o pipefail

# Common configuration
DAYS="${DAYS:-14}"
repo_name_regex="${REPO_NAME_REGEX:-^[a-z0-9-]*(python|dotnet-basic|java-quarkus|go|nodejs|java-springboot)[a-z0-9-]*(-gitops)?$}"
repo_name_regex="${REPO_NAME_REGEX:-^[a-zA-Z0-9-]*(python|dotnet-basic|java-quarkus|go|nodejs|java-springboot)[a-zA-Z0-9-]*(-gitops)?\$}"

usage() {
echo "
Expand Down Expand Up @@ -60,12 +60,60 @@ github_cleanup() {
echo "Checking GitHub organization: $GITHUB_ORG"
echo "Cutoff date: $(date -d "@$cutoff_time")"

# Fetch repositories (sorted by creation date, oldest first)
repos=$(curl -s -X GET -H "$AUTH_HEADER" "https://api.github.com/orgs/$GITHUB_ORG/repos?per_page=200&sort=created&direction=asc")
if echo "$repos" | jq -e '.status' >/dev/null 2>&1; then
echo "Error fetching repositories: $repos" >&2
return 1
# Fetch repositories (sorted by creation date, oldest first) with pagination.
next_url="https://api.github.com/orgs/$GITHUB_ORG/repos?per_page=100&sort=created&direction=asc"
all_repos=()
while [[ -n "$next_url" ]]; do
tmp_body=$(mktemp)
tmp_headers=$(mktemp)
http_status=$(curl -s -D "$tmp_headers" -o "$tmp_body" -w "%{http_code}" -X GET -H "$AUTH_HEADER" "$next_url")
repos_page=$(<"$tmp_body")
rm -f "$tmp_body"

# Handle HTTP errors first.
if [[ "$http_status" -lt 200 || "$http_status" -ge 300 ]]; then
echo "Error: GitHub API returned HTTP $http_status" >&2
if [[ -n "$repos_page" ]]; then
echo "Response: $repos_page"
fi
rm -f "$tmp_headers"
return 1
fi

if echo "$repos_page" | jq -e '.status' >/dev/null 2>&1; then
echo "Error fetching repositories: $repos_page" >&2
rm -f "$tmp_headers"
return 1
fi

if ! echo "$repos_page" | jq -e 'type == "array"' >/dev/null 2>&1; then
echo "Error: Invalid response format from GitHub API" >&2
echo "Response: ${repos_page:-<empty>}"
rm -f "$tmp_headers"
return 1
fi

while IFS= read -r repo; do
all_repos+=("$repo")
done < <(echo "$repos_page" | jq -c '.[]')

link_header=$(tr -d '\r' < "$tmp_headers" | awk -F': ' 'tolower($1)=="link"{print $2}')
rm -f "$tmp_headers"
if [[ "$link_header" =~ \<([^>]*)\>\;\ rel=\"next\" ]]; then
next_url="${BASH_REMATCH[1]}"
else
next_url=""
fi
done

if [[ ${#all_repos[@]} -gt 0 ]]; then
repos=$(printf '%s\n' "${all_repos[@]}" | jq -s '.')
else
repos='[]'
fi

repo_count=$(echo "$repos" | jq 'length')
echo "Found $repo_count repositories"

# Process repositories
while read -r repo; do
Expand Down
60 changes: 51 additions & 9 deletions hack/ci/git-org-cleanup/gitlab-repos-cleanup.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ set -o pipefail

# Common configuration
DAYS="${DAYS:-14}"
repo_name_regex="${REPO_NAME_REGEX:-^[a-z0-9-]*(python|dotnet-basic|java-quarkus|go|nodejs|java-springboot)[a-z0-9-]*(-gitops)?$}"
repo_name_regex="${REPO_NAME_REGEX:-^[a-zA-Z0-9-]*(python|dotnet-basic|java-quarkus|go|nodejs|java-springboot)[a-zA-Z0-9-]*(-gitops)?\$}"

usage() {
echo "
Expand Down Expand Up @@ -65,15 +65,57 @@ gitlab_cleanup() {
echo "Cutoff date: $cutoff_date"
echo ""

# Fetch projects from group (sorted by creation date, oldest first)
projects=$(curl -s -H "$AUTH_HEADER" \
"$GITLAB_URL/api/v4/groups/$GITLAB_GROUP/projects?per_page=200&order_by=created_at&sort=asc")

if echo "$projects" | jq -e '.message or .error' 2>/dev/null ; then
echo "Error fetching projects: $projects" >&2
return 1
# Fetch all projects from group with pagination.
next_page=1
all_projects=()
while [[ -n "$next_page" ]]; do
api_url="$GITLAB_URL/api/v4/groups/$GITLAB_GROUP/projects?per_page=100&order_by=name&sort=asc&page=$next_page"
tmp_body=$(mktemp)
tmp_headers=$(mktemp)
http_status=$(curl -s -D "$tmp_headers" -o "$tmp_body" -w "%{http_code}" -H "$AUTH_HEADER" "$api_url")
projects_page=$(<"$tmp_body")
rm -f "$tmp_body"

# Handle HTTP errors first.
if [[ "$http_status" -lt 200 || "$http_status" -ge 300 ]]; then
echo "Error: GitLab API returned HTTP $http_status" >&2
if [[ -n "$projects_page" ]]; then
echo "Response: $projects_page"
fi
rm -f "$tmp_headers"
return 1
fi

if echo "$projects_page" | jq -e '.message or .error' >/dev/null 2>&1; then
echo "Error fetching projects: $projects_page" >&2
rm -f "$tmp_headers"
return 1
fi

if ! echo "$projects_page" | jq -e 'type == "array"' >/dev/null 2>&1; then
echo "Error: Invalid response format from GitLab API" >&2
echo "Response: ${projects_page:-<empty>}"
rm -f "$tmp_headers"
return 1
fi

while IFS= read -r project; do
all_projects+=("$project")
done < <(echo "$projects_page" | jq -c '.[]')

next_page=$(tr -d '\r' < "$tmp_headers" | awk -F': ' 'tolower($1)=="x-next-page"{print $2}')
rm -f "$tmp_headers"
done

if [[ ${#all_projects[@]} -gt 0 ]]; then
projects=$(printf '%s\n' "${all_projects[@]}" | jq -s '.')
else
projects='[]'
fi


project_count=$(echo "$projects" | jq 'length')
echo "Found $project_count repositories"

# Process projects (using process substitution to avoid subshell)
while read -r project; do
project_name=$(echo "$project" | jq -r '.name' 2>/dev/null)
Expand Down
Loading