Fix/pykube ng kopf runtime

.github/workflows/image.yml

@@ -48,7 +48,8 @@ jobs:
       - name: Assign Custom Image Name
         if:  ${{ github.repository_owner != 'apache'}}
         run: |
-          echo "MY_OPERATOR_IMAGE=${{ vars.MY_OPERATOR_IMAGE }}" >> "$GITHUB_ENV"
+          echo "MY_OPERATOR_IMAGE=ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}" >> "$GITHUB_ENV"
+          echo "IMAGE_REGISTRY=ghcr.io" >> "$GITHUB_ENV"
       - name: Set up Python 3.12
         uses: actions/setup-python@v4
         with:
@@ -85,9 +86,9 @@ jobs:
       - name: Registry login
         uses: docker/login-action@v3
         with:
-          registry: ${{ vars.IMAGE_REGISTRY || 'registry.hub.docker.com' }}
-          username: ${{ secrets.DOCKERHUB_USER || github.actor }}
-          password: ${{ secrets.DOCKERHUB_TOKEN || secrets.GITHUB_TOKEN }}
+          registry: ${{ env.IMAGE_REGISTRY || 'registry.hub.docker.com' }}
+          username: ${{ env.IMAGE_REGISTRY == 'ghcr.io' && github.repository_owner || secrets.DOCKERHUB_USER }}
+          password: ${{ env.IMAGE_REGISTRY == 'ghcr.io' && secrets.GITHUB_TOKEN || secrets.DOCKERHUB_TOKEN }}
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
         with:

.github/workflows/trigger-testing.yaml

@@ -16,75 +16,79 @@
 # under the License.
 #
 name: Trigger Testing
-run-name: Dispatch operator PR test for ${{ github.event.issue.number || github.event.inputs.pr_number }}
+run-name: Dispatch operator PR test for PR #${{ github.event.pull_request.number }}
 
 on:
-  issue_comment:
-    types: [created]
-  workflow_dispatch:
-    inputs:
-      pr_number:
-        description: "PR number to test"
-        required: true
-        type: string
-      platform:
-        description: "Platform to test on (e.g. k3s-amd, eks-amd)"
-        required: true
-        type: string
+  pull_request_target:
+    types: [labeled, synchronize, reopened]
 
 jobs:
   dispatch:
     name: Dispatch operator-pr-test
     runs-on: ubuntu-22.04
-    # Run on /testing comments from authorized users, or on manual dispatch
-    if: >-
-      github.event_name == 'workflow_dispatch' ||
-      (
-        github.event.issue.pull_request &&
-        startsWith(github.event.comment.body, '/testing ') &&
-        contains(fromJSON('["MEMBER","OWNER","COLLABORATOR"]'), github.event.comment.author_association)
-      )
     steps:
-      - name: Parse comment
+      - name: Parse testing tag
         id: parse
+        env:
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          PR_REF: ${{ github.event.pull_request.head.ref }}
+          PR_SHA: ${{ github.event.pull_request.head.sha }}
+          PR_REPO: ${{ github.event.pull_request.head.repo.full_name }}
         run: |
-          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
-            echo "platform=${{ github.event.inputs.platform }}" >> "$GITHUB_OUTPUT"
-            echo "pr_number=${{ github.event.inputs.pr_number }}" >> "$GITHUB_OUTPUT"
-          else
-            COMMENT="${{ github.event.comment.body }}"
-            PLATFORM="${COMMENT#/testing }"
-            echo "platform=${PLATFORM}" >> "$GITHUB_OUTPUT"
-            echo "pr_number=${{ github.event.issue.number }}" >> "$GITHUB_OUTPUT"
+          VALID_TEST_RE='^(kind|k3s|k3sarm|k8s|k8sarm|mk8s|mk8sarm|eks|eksarm|aks|aksarm|gke|gkearm|osh|osharm)-([0-9a-f]{7,40})$'
+          MATCHING_TAGS=()
+
+          while IFS= read -r label; do
+            [[ -n "$label" ]] || continue
+            if [[ "$label" =~ $VALID_TEST_RE ]]; then
+              label_hash="${BASH_REMATCH[2]}"
+              if [[ "$PR_SHA" == "$label_hash"* ]]; then
+                MATCHING_TAGS+=("$label")
+              fi
+            fi
+          done < <(jq -r '.pull_request.labels[]?.name // empty' "$GITHUB_EVENT_PATH")
+
+          echo "pr_number=$PR_NUMBER" >> "$GITHUB_OUTPUT"
+          echo "pr_ref=$PR_REF" >> "$GITHUB_OUTPUT"
+          echo "pr_sha=$PR_SHA" >> "$GITHUB_OUTPUT"
+          echo "repo=$PR_REPO" >> "$GITHUB_OUTPUT"
+
+          if [[ "${#MATCHING_TAGS[@]}" -gt 1 ]]; then
+            echo "Found multiple matching <test>-<hash> tags for PR SHA $PR_SHA:" >&2
+            printf ' - %s\n' "${MATCHING_TAGS[@]}" >&2
+            exit 1
           fi
 
-      - name: Get PR details
-        id: pr
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          PR_JSON=$(gh api repos/${{ github.repository }}/pulls/${{ steps.parse.outputs.pr_number }})
-          echo "ref=$(echo "$PR_JSON" | jq -r '.head.ref')" >> "$GITHUB_OUTPUT"
-          echo "repo=$(echo "$PR_JSON" | jq -r '.head.repo.full_name')" >> "$GITHUB_OUTPUT"
-          echo "sha=$(echo "$PR_JSON" | jq -r '.head.sha')" >> "$GITHUB_OUTPUT"
+          if [[ "${#MATCHING_TAGS[@]}" -eq 1 ]]; then
+            TEST_TAG="${MATCHING_TAGS[0]}"
+            TEST_NAME="${TEST_TAG%-*}"
+            TEST_HASH="${TEST_TAG##*-}"
+            echo "enabled=true" >> "$GITHUB_OUTPUT"
+            echo "test_tag=$TEST_TAG" >> "$GITHUB_OUTPUT"
+            echo "test_name=$TEST_NAME" >> "$GITHUB_OUTPUT"
+            echo "test_hash=$TEST_HASH" >> "$GITHUB_OUTPUT"
+            echo "reason=Matched testing tag $TEST_TAG for PR SHA $PR_SHA" >> "$GITHUB_OUTPUT"
+          else
+            echo "enabled=false" >> "$GITHUB_OUTPUT"
+            echo "reason=No <test>-<hash> tag found on PR labels for current SHA $PR_SHA" >> "$GITHUB_OUTPUT"
+          fi
 
-      - name: Add reaction to comment
-        if: github.event_name != 'workflow_dispatch'
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          gh api repos/${{ github.repository }}/issues/comments/${{ github.event.comment.id }}/reactions \
-            -f content=rocket
+      - name: Skip when no matching testing tag is present
+        if: steps.parse.outputs.enabled != 'true'
+        run: echo "${{ steps.parse.outputs.reason }}"
 
       - name: Dispatch to testing repo
+        if: steps.parse.outputs.enabled == 'true'
         env:
           GH_TOKEN: ${{ secrets.OPENSERVERLESS_TESTING_PAT }}
         run: |
           gh api repos/${{ github.repository_owner }}/openserverless-testing/dispatches \
             -X POST \
             -f event_type=operator-pr-test \
             -f 'client_payload[pr_number]=${{ steps.parse.outputs.pr_number }}' \
-            -f 'client_payload[pr_ref]=${{ steps.pr.outputs.ref }}' \
-            -f 'client_payload[pr_sha]=${{ steps.pr.outputs.sha }}' \
-            -f 'client_payload[operator_repo]=${{ steps.pr.outputs.repo }}' \
-            -f 'client_payload[platform]=${{ steps.parse.outputs.platform }}'
+            -f 'client_payload[pr_ref]=${{ steps.parse.outputs.pr_ref }}' \
+            -f 'client_payload[pr_sha]=${{ steps.parse.outputs.pr_sha }}' \
+            -f 'client_payload[operator_repo]=${{ steps.parse.outputs.repo }}' \
+            -f 'client_payload[test_tag]=${{ steps.parse.outputs.test_tag }}' \
+            -f 'client_payload[test_name]=${{ steps.parse.outputs.test_name }}' \
+            -f 'client_payload[test_hash]=${{ steps.parse.outputs.test_hash }}'

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "olaris"]
 	path = olaris
-	url = https://github.com/apache/openserverless-task.git
+	url = https://github.com/nuvolaris/openserverless-task.git

README.md

@@ -26,6 +26,8 @@ We describe how to build and test the operator in our development environment
 
 Please refer to the [website](https://openserverless.apache.org) for user information.
 
+For PR-driven infrastructure tests on GitHub, use a PR label in the form `<test>-<commit-hash>`, for example `k3s-abcdef1`.
+
 ## How to build and use an operator image
 
 Ensure you have satisfied the prerequisites below. Most notably, you need to use our development virtual machine and you
@@ -90,4 +92,3 @@ task all
 
 The operator instance will be configured applying the `test/k3s/whisk.yaml` template.
 All the components are activated except TLS and MONITORING.
-

deploy/ferretdb/ferretdb-sts.yaml

@@ -36,9 +36,12 @@ spec:
         app: nuvolaris-mongodb
         name: nuvolaris-mongodb
     spec:
+      securityContext:
+        fsGroup: 1001
+        fsGroupChangePolicy: Always
       containers:
       - image: ghcr.io/nuvolaris/ferretdb:1.6.0
         name: ferretdb
         env:
         - name: FERRETDB_POSTGRESQL_URL
-          value: postgresql://nuvolaris:s0meP%40ass3@nuvolaris-postgres.nuvolaris.svc.cluster.local:5432/nuvolaris
+          value: postgresql://nuvolaris:s0meP%40ass3@nuvolaris-postgres.nuvolaris.svc.cluster.local:5432/nuvolaris

deploy/nuvolaris-permissions/operator-roles.yaml

@@ -83,7 +83,7 @@ rules:
   verbs: ["get","patch","list","update","watch","create","delete"]
 
 # required for traefik middlewares
-- apiGroups: ["traefik.containo.us"]
+- apiGroups: ["traefik.io"]
   resources: ["middlewares"]
   verbs: ["get","patch","list","update","watch","create","delete"]
 

nuvolaris/endpoint.py

@@ -240,11 +240,11 @@ def delete(owner=None):
             return res
 
         if(ingress_class == 'traefik'):            
-            res = kube.kubectl("delete", "middleware.traefik.containo.us",api_middleware_ingress_name(namespace,"apihost"))
-            res += kube.kubectl("delete", "middleware.traefik.containo.us",api_middleware_ingress_name(namespace,"apihost-my"))
-            res += kube.kubectl("delete", "middleware.traefik.containo.us",api_middleware_ingress_name(namespace,"apihost-info"))
+            res = kube.kubectl("delete", "middleware.traefik.io",api_middleware_ingress_name(namespace,"apihost"))
+            res += kube.kubectl("delete", "middleware.traefik.io",api_middleware_ingress_name(namespace,"apihost-my"))
+            res += kube.kubectl("delete", "middleware.traefik.io",api_middleware_ingress_name(namespace,"apihost-info"))
             if should_delete_www:
-                res += kube.kubectl("delete", "middleware.traefik.containo.us",api_middleware_ingress_name(namespace,"apihost-www-my"))          
+                res += kube.kubectl("delete", "middleware.traefik.io",api_middleware_ingress_name(namespace,"apihost-www-my"))
 
         res += kube.kubectl("delete", "ingress",api_ingress_name(namespace,"apihost"))
         res += kube.kubectl("delete", "ingress",api_ingress_name(namespace,"apihost-my"))
@@ -321,8 +321,8 @@ def delete_ow_api_endpoint(ucfg):
             return res
 
         if(ingress_class == 'traefik'):                        
-            res += kube.kubectl("delete", "middleware.traefik.containo.us",api_middleware_ingress_name(namespace,"apihost"))
-            res += kube.kubectl("delete", "middleware.traefik.containo.us",api_middleware_ingress_name(namespace,"apihost-my"))             
+            res += kube.kubectl("delete", "middleware.traefik.io",api_middleware_ingress_name(namespace,"apihost"))
+            res += kube.kubectl("delete", "middleware.traefik.io",api_middleware_ingress_name(namespace,"apihost-my"))
 
         res += kube.kubectl("delete", "ingress",api_ingress_name(namespace,"apihost"))
         res += kube.kubectl("delete", "ingress",api_ingress_name(namespace,"apihost-my"))

nuvolaris/main.py

@@ -50,13 +50,26 @@ def configure(settings: kopf.OperatorSettings, **_):
 
 # tested by an integration test
 @kopf.on.login()
-def login(**kwargs):
+def login(logger, **kwargs):
     token = '/var/run/secrets/kubernetes.io/serviceaccount/token'
     if os.path.isfile(token):
-        logging.debug("found serviceaccount token: login via pykube in kubernetes")
-        return kopf.login_via_pykube(**kwargs)
-    logging.debug("login via client")
-    return kopf.login_via_client(**kwargs)
+        for method, handler in [
+            ("service-account", kopf.login_with_service_account),
+            ("client", kopf.login_via_client),
+            ("pykube", kopf.login_via_pykube),
+        ]:
+            try:
+                credentials = handler(logger=logger, **kwargs)
+            except Exception:
+                logger.exception("login via %s failed", method)
+                continue
+            if credentials is not None:
+                logger.info("authenticated in-cluster via %s", method)
+                return credentials
+            logger.warning("login via %s returned no credentials", method)
+        raise kopf.LoginError("No in-cluster credentials were retrieved from service account, client, or pykube.")
+    logger.debug("login via client")
+    return kopf.login_via_client(logger=logger, **kwargs)
 
 # tested by an integration test
 @kopf.on.create('nuvolaris.org', 'v1', 'whisks')
@@ -421,4 +434,4 @@ def runtimes_cm_event_watcher(event, **kwargs):
     patcher.patch_preloader(owner)
 
     if cfg.get('components.openwhisk'):
-        patcher.restart_whisk(owner)
+        patcher.restart_whisk(owner)

nuvolaris/minio_ingress.py

@@ -203,7 +203,7 @@ def delete_minio_ingress(runtime, namespace, ingress_class, type, owner=None):
         res += kube.kubectl("delete", "ingress",endpoint.api_ingress_name(namespace,type))    
 
         if(ingress_class == 'traefik'):            
-            res = kube.kubectl("delete", "middleware.traefik.containo.us",endpoint.api_middleware_ingress_name(namespace,type))         
+            res = kube.kubectl("delete", "middleware.traefik.io",endpoint.api_middleware_ingress_name(namespace,type))
 
         return res
     except Exception as e:

nuvolaris/registry_deploy.py

@@ -221,7 +221,7 @@ def delete_registry_ingress(owner=None, namespace="nuvolaris"):
         res += kube.kubectl("delete", "ingress",endpoint.api_ingress_name(namespace,"registry"))    
 
         if(ingress_class == 'traefik'):            
-            res = kube.kubectl("delete", "middleware.traefik.containo.us",endpoint.api_middleware_ingress_name(namespace,"registry"))         
+            res = kube.kubectl("delete", "middleware.traefik.io",endpoint.api_middleware_ingress_name(namespace,"registry"))
 
         return res
     except Exception as e:

nuvolaris/seaweedfs_ingress.py

@@ -151,7 +151,7 @@ def delete_seaweedfs_ingress(runtime, namespace, ingress_class, type, owner=None
         res += kube.kubectl("delete", "ingress",endpoint.api_ingress_name(namespace,type))    
 
         if(ingress_class == 'traefik'):            
-            res = kube.kubectl("delete", "middleware.traefik.containo.us",endpoint.api_middleware_ingress_name(namespace,type))         
+            res = kube.kubectl("delete", "middleware.traefik.io",endpoint.api_middleware_ingress_name(namespace,type))
 
         return res
     except Exception as e:

nuvolaris/storage_static.py

@@ -177,7 +177,7 @@ def delete_ow_static_endpoint(ucfg):
 
         if(ingress_class == 'traefik'):            
             middleware_name = static_middleware_ingress_name(namespace)
-            res += kube.kubectl("delete", "middleware.traefik.containo.us",middleware_name)            
+            res += kube.kubectl("delete", "middleware.traefik.io",middleware_name)
 
         ingress_name = static_ingress_name(namespace)
         res += kube.kubectl("delete", "ingress",ingress_name)
@@ -221,11 +221,11 @@ def delete_nuv_ingresses():
 
         if(ingress_class == 'traefik'):            
             middleware_name = static_middleware_ingress_name("nuvolaris")
-            res += kube.kubectl("delete", "middleware.traefik.containo.us",middleware_name)
+            res += kube.kubectl("delete", "middleware.traefik.io",middleware_name)
 
             if should_delete_www:
                 middleware_name = static_middleware_ingress_name("www-nuvolaris")
-                res += kube.kubectl("delete", "middleware.traefik.containo.us",middleware_name)             
+                res += kube.kubectl("delete", "middleware.traefik.io",middleware_name)
 
         ingress_name = static_ingress_name("nuvolaris")
         res += kube.kubectl("delete", "ingress",ingress_name)

nuvolaris/templates/ferretdb-sts.yaml

@@ -36,14 +36,12 @@ spec:
         app: nuvolaris-mongodb
         name: nuvolaris-mongodb         
     spec:
-      {% if applypodsecurity %}
       securityContext:
-        fsGroup: 65534
-        runAsUser: 65534      
-      {% endif %}  
+        fsGroup: 1001
+        fsGroupChangePolicy: Always
       containers:
       - image: ghcr.io/nuvolaris/ferretdb:1.6.0
         name: ferretdb
         env:
         - name: FERRETDB_POSTGRESQL_URL
-          value: {{ferretdb_postgres_url}}          
+          value: {{ferretdb_postgres_url}}

nuvolaris/templates/traefik-middleware-tpl.yaml

@@ -16,7 +16,7 @@
 # under the License.
 #
 ---
-apiVersion: traefik.containo.us/v1alpha1
+apiVersion: traefik.io/v1alpha1
 kind: Middleware
 metadata:
   namespace: {{namespace}}

nuvolaris/util.py

@@ -158,17 +158,35 @@ def get_ingress_class(runtime):
         logging.warn(f"skipping ingress class auto detection and returning {ingress_class}")
         return ingress_class
 
-    # ingress class default to nginx
+    # Default according to the historical runtime assumptions.
     ingress_class = "nginx"
-
-    # On microk8s ingress class must be public
     if runtime == "microk8s":
         ingress_class = "public"
-
-    # On k3s ingress class must be traefik
-    if runtime == "k3s":
+    elif runtime == "k3s":
         ingress_class = "traefik"
 
+    # Prefer the ingress class that actually exists in the current cluster.
+    try:
+        detected = kube.kubectl(
+            "get",
+            "ingressclass",
+            namespace=None,
+            jsonpath=r"{.items[*].metadata.name}",
+            debugresult=False,
+        )
+        detected = [item for item in detected if item]
+        if detected:
+            logging.info(f"auto-detected ingress classes: {detected}")
+            if "nginx" in detected:
+                return "nginx"
+            if runtime == "microk8s" and "public" in detected:
+                return "public"
+            if runtime == "k3s" and "traefik" in detected:
+                return "traefik"
+            return detected[0]
+    except Exception as e:
+        logging.warning(f"failed to auto-detect ingress classes, using default {ingress_class}: {e}")
+
     return ingress_class
 
 # determine the ingress-nginx flavour
@@ -890,4 +908,3 @@ def get_seaweedds_filer_host():
      seaweedfs_filer_port   = cfg.get("seaweedfs.port", "SEAWEEDFS_API_PORT", "9090")
      return f"http://{seaweedfs_filer_host}:{seaweedfs_filer_port}"
 
-