Replace Portage-Stable with Gentoo

.github/workflows/image_changes.sh

@@ -25,8 +25,6 @@ function github_ricj_callback() {
     show_changes_env+=(
         # Override the default locations of repositories.
         "SCRIPTS_REPO=."
-        "COREOS_OVERLAY_REPO=../coreos-overlay"
-        "PORTAGE_STABLE_REPO=../portage-stable"
     )
     show_changes_params+=(
         # We may not have a tag handy, so we tell show-changes

.github/workflows/update-portage-stable-packages-from-list.yaml

@@ -1,92 +1,53 @@
-name: Keep portage-stable packages updated
+name: Keep Gentoo submodule updated
 on:
   schedule:
-    - cron:  '0 7 * * 1'
+    - cron: '0 4 * * 0-4'
   workflow_dispatch:
 
 jobs:
-  keep-packages-updated:
+  keep-gentoo-updated:
     runs-on: ubuntu-latest
     steps:
       - name: Check out scripts
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
         with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          path: ./scripts
-      - name: Check out Gentoo
-        uses: actions/checkout@v4
-        with:
-          repository: gentoo/gentoo
-          path: gentoo
-          # Gentoo is quite a large repo, so limit ourselves to last
-          # quarter milion of commits. It is about two years worth of changes.
-          # Is is needed by the sync script to find out the hash of the last commit
-          # that made the changes to the package.
-          fetch-depth: 250000
-          ref: master
-      - name: Check out build scripts
-        uses: actions/checkout@v4
-        with:
-          repository: flatcar/flatcar-build-scripts
-          path: flatcar-build-scripts
-      - name: Update listed packages
-        id: update-listed-packages
+          token: ${{ secrets.BOT_PR_TOKEN }}
+          path: scripts
+          sparse-checkout: |
+            .gitmodules
+            .github/workflows/common.sh
+          sparse-checkout-cone-mode: false
+      - name: Update Gentoo submodule
+        id: update-gentoo
+        env:
+          WORK_SCRIPTS_DIR: "${{ github.workspace }}/scripts"
         run: |
-          git config --global user.name "Flatcar Buildbot"
-          git config --global user.email "buildbot@flatcar-linux.org"
-          old_head=$(git -C scripts rev-parse HEAD)
-          packages_list=$(realpath scripts/.github/workflows/portage-stable-packages-list)
-          gentoo_repo=$(realpath gentoo)
-          build_scripts=$(realpath flatcar-build-scripts)
-          pushd scripts/sdk_container/src/third_party/portage-stable
-          while read -r package; do
-              if [[ ! -e "${package}" ]]; then
-                  # If this happens, it means that the package was moved to overlay
-                  # or dropped, the list ought to be updated.
-                  echo "::warning title=${package}::Nonexistent package"
-                  continue
-              fi
-              if [[ ! -e "${gentoo_repo}/${package}" ]]; then
-                  # If this happens, it means that the package was obsoleted or moved
-                  # in Gentoo. The obsoletion needs to be handled in the case-by-case
-                  # manner, while move should be handled by doing the same move
-                  # in portage-stable. The build should not break because of the move,
-                  # because most likely it's already reflected in the profiles/updates
-                  # directory.
-                  echo "::warning title=${package}::Obsolete or moved package"
-                  continue
-              fi
-              GENTOO_REPO="${gentoo_repo}" "${build_scripts}/sync-with-gentoo" "${package}"
-          done < <(grep '^[^#]' "${packages_list}")
-          popd
-          new_head=$(git -C scripts rev-parse HEAD)
-          updated=0
-          if [[ "${new_head}" != "${old_head}" ]]; then
-              updated=1
+          set -euo pipefail
+          source scripts/.github/workflows/common.sh
+          prepare_git_repo
+          submodule=sdk_container/src/third_party/gentoo
+          url=$(git -C "${WORK_SCRIPTS_DIR}" config --file .gitmodules --get "submodule.${submodule}.url")
+          branch=$(git -C "${WORK_SCRIPTS_DIR}" config --file .gitmodules --get "submodule.${submodule}.branch")
+          read -r latest_commit _ < <(git ls-remote --exit-code "${url}" "refs/heads/${branch}")
+          short_commit=${latest_commit:0:12}
+          git -C "${WORK_SCRIPTS_DIR}" update-index --cacheinfo "160000,${latest_commit},${submodule}"
+          if git -C "${WORK_SCRIPTS_DIR}" diff --cached --quiet -- "${submodule}"; then
+              echo 'UPDATE_NEEDED=0' >>"${GITHUB_OUTPUT}"
+              exit 0
           fi
-          todaydate=$(date +%Y-%m-%d)
-          echo "UPDATED=${updated}" >>"${GITHUB_OUTPUT}"
-          echo "TODAYDATE=${todaydate}" >>"${GITHUB_OUTPUT}"
-      - name: Create pull request for main branch
-        uses: peter-evans/create-pull-request@v6
-        if: steps.update-listed-packages.outputs.UPDATED == 1
+          git -C "${WORK_SCRIPTS_DIR}" commit --signoff -m "gentoo: Update to ${short_commit}"
+          cleanup_repo
+          echo 'UPDATE_NEEDED=1' >>"${GITHUB_OUTPUT}"
+          echo "LATEST_COMMIT=${latest_commit}" >>"${GITHUB_OUTPUT}"
+          echo "SHORT_COMMIT=${short_commit}" >>"${GITHUB_OUTPUT}"
+      - name: Create pull request
+        uses: peter-evans/create-pull-request@v8
+        if: steps.update-gentoo.outputs.UPDATE_NEEDED == 1
         with:
-          token: ${{ secrets.GITHUB_TOKEN }}
+          token: ${{ secrets.BOT_PR_TOKEN }}
           path: scripts
-          branch: buildbot/weekly-portage-stable-package-updates-${{steps.update-listed-packages.outputs.TODAYDATE }}
+          branch: buildbot/update-gentoo-submodule
           delete-branch: true
-          base: main
-          title: Weekly portage-stable package updates ${{steps.update-listed-packages.outputs.TODAYDATE }}
-          body: |
-            CI: TODO
-
-            --
-
-            TODO: Changes.
-
-            --
-
-            - [ ] changelog
-            - [ ] image diff
-          labels: main
-          draft: true
+          title: Update gentoo submodule to ${{ steps.update-gentoo.outputs.SHORT_COMMIT }}
+          body: Update `sdk_container/src/third_party/gentoo` to flatcar-hub/gentoo@${{ steps.update-gentoo.outputs.LATEST_COMMIT }}.
+          labels: auto-ci,jenkins-ci,main

.gitmodules

@@ -0,0 +1,5 @@
+[submodule "sdk_container/src/third_party/gentoo"]
+	path = sdk_container/src/third_party/gentoo
+	url = https://github.com/flatcar-hub/gentoo.git
+	branch = master
+	shallow = true

PREFIX.md

@@ -1,3 +1,11 @@
+# !!! OUTDATED. This needs to be ported to match the changes after the Gentoo move !!!
+
+If anything, working with prefixes should be a lot easier now.
+But the below is outdated and will not work.
+`prefix-overlay` has been removed too, as it duplicated ebuilds now available via `gentoo`.
+
+# !!! OUTDATED. This needs to be ported to match the changes after the Gentoo move !!!
+
 # Prefix - build portable, distro-independent apps
 
 **!!! NOTE: Prefix support in the Flatcar SDK is EXPERIMENTAL at this time !!!**

README.md

@@ -28,18 +28,17 @@ The SDK can be used to
 # Using the scripts repository
 
 The repository is meant to be the entry point for Flatcar builds and development.
-Ebuilds for all packages reside in one of 2 subdirectories - [coreos-overlay](sdk_container/src/third_party/coreos-overlay) and [portage-stable](sdk_container/src/third_party/portage-stable/):
+Ebuilds for all packages reside in one of 2 subdirectories - [coreos-overlay](sdk_container/src/third_party/coreos-overlay) and [gentoo](sdk_container/src/third_party/gentoo/):
 ```
 scripts
    +--sdk_container
           +---------src
                      +--third_party
                              +------coreos-overlay
-                             +------portage-stable
+                             +------gentoo
 ```
 
-`portage-stable` is kept in alignment with upstream Gentoo and should not contain any modifications (with only minor, well-justified exceptions).
-Consider it a small sub-set of Gentoo.
+`gentoo` is a git submodule, pointing to Flatcar's fork of the main upstream Gentoo package repository. That fork is generally aligned with Gentoo and only occasionally diverges into alternative branches for brief periods to apply fixes shortly before upstream does. It includes far more packages than Flatcar actually uses.
 
 `coreos-overlay` contains significantly modified or even entirely self-written ebuilds.
 
@@ -66,7 +65,7 @@ While work on a native ARM64 native SDK is ongoing, it's unfortunately not ready
 
 The container can be run in one of two ways - "standalone", or integrated with the [scripts](https://github.com/flatcar/scripts) repo:
 * Standalone mode will use no host volumes and will allow you to play with the SDK in a sandboxed throw-away environment. In standalone mode, you interface with Docker directly to use the SDK container.
-* Integrated mode will closely integrate with the scripts repo directory and bind-mount it as well as the portage-stable and coreos-overlay directories into the container. Integrated mode uses wrapper scripts to interact with the SDK container. This is the recommended way for developing patches for Flatcar.
+* Integrated mode will closely integrate with the scripts repo directory and bind-mount it as well as the gentoo and coreos-overlay directories into the container. Integrated mode uses wrapper scripts to interact with the SDK container. This is the recommended way for developing patches for Flatcar.
 
 ## Standalone mode
 
@@ -96,7 +95,7 @@ To start a container in privileged mode with `/dev` available use:
 
 This is the preferred mode of working with the SDK.
 Interaction with the container happens via wrapper scripts from the scripts repository.
-Both the host's scripts repo as well as the ebuild paths (portage-stable and coreos-overlay) are made available in the container, allowing for work on these directly.
+Both the host's scripts repo as well as the ebuild directories (gentoo and coreos-overlay) are made available in the container, allowing for work on these directly.
 The wrapper scripts will re-use existing containers instead of creating new ones to preserve your work in the container, enabling consistency.
 
 To clone the scripts repo and pick a version:

bootstrap_sdk_container

@@ -9,9 +9,10 @@ set -eu
 cd $(dirname "$0")
 source sdk_lib/sdk_container_common.sh
 
+unset seed_tarball
+bootstrap_args=()
 seed_version=""
 target_version=""
-logdir=''
 
 declare -a cleanup
 
@@ -28,10 +29,10 @@ usage() {
     echo "                             The SDK container will be pulled and the tarball"
     echo "                             downloaded if necessary."
     echo "       <new-sdk-vernum>   - SDK version number (e.g. '3027.0.0') of the new SDK."
+    echo "      --seed_tarball <x>  - Optional path or URL to override the default seed tarball."
     echo "      -x <cleanup-script> - For each resource generated during build (container etc.)"
     echo "                             add a cleanup line to <script> which, when run, will free"
     echo "                             the resource. Useful for CI."
-    echo "      -l <directory>      - Gather build logs here."
     echo "      -h                  - Print this help."
     echo
 }
@@ -40,8 +41,8 @@ usage() {
 while [ 0 -lt $# ] ; do
     case "$1" in
     -h) usage; exit 0;;
-    -l) logdir=${2}; shift 2;;
-    -x) cleanup=("-x" "$2"); shift; shift;;
+    -x) cleanup=("-x" "$2"); shift 2;;
+    --seed_tarball) seed_tarball=$2; shift 2 ;;
     *)  if [ -z "$seed_version" ] ; then
             seed_version="$1"
         elif [ -z "$target_version" ] ; then
@@ -74,10 +75,12 @@ if $official; then
     export COREOS_OFFICIAL=1
 fi
 
+[[ -n ${seed_tarball-} ]] && bootstrap_args+=( --seed_tarball "$seed_tarball" )
+
 # bootstrap_sdk needs FLATCAR_SDK_VERSION set to the seed version
 failed=''
 ./run_sdk_container "${cleanup[@]}" -V "$seed_version" -v "$target_version" \
-          sudo -E ./bootstrap_sdk || failed=x
+          sudo -E ./bootstrap_sdk "${bootstrap_args[@]}" || failed=x
 
 # Update versionfile to the actual SDK version
 create_versionfile "${target_version}"

build_library/build_image_util.sh

@@ -294,7 +294,7 @@ get_metadata() {
             local mirror="$(echo "${v}" | grep mirror:// | cut -d '/' -f 3)"
             if [ -n "${mirror}" ]; then
                 # Take only first mirror, those not working should be removed
-                local location="$(grep "^${mirror}"$'\t' /mnt/host/source/src/third_party/portage-stable/profiles/thirdpartymirrors | cut -d $'\t' -f 2- | cut -d ' ' -f 1 | tr -d $'\t')"
+                local location="$(grep "^${mirror}"$'\t' /mnt/host/source/src/third_party/gentoo/profiles/thirdpartymirrors | cut -d $'\t' -f 2- | cut -d ' ' -f 1 | tr -d $'\t')"
                 v="$(echo "${v}" | sed "s#mirror://${mirror}/#${location}#g")"
             fi
             new_val+="${v} "
@@ -419,7 +419,7 @@ EOF
     license_list="$(jq -r '.[] | "\(.licenses | .[])"' "${json_input}" | sort | uniq)"
     local license_dirs=(
         "/mnt/host/source/src/third_party/coreos-overlay/licenses/"
-        "/mnt/host/source/src/third_party/portage-stable/licenses/"
+        "/mnt/host/source/src/third_party/gentoo/licenses/"
         "none"
     )
     for license_file in ${license_list}; do