AshleyDumaine · Oct 26, 2026 · Oct 23, 2026 · Oct 26, 2026 · Oct 26, 2026 · Oct 26, 2026
@@ -4,6 +4,7 @@ kind: Configuration
 metadata:
   name: configuration
 spec:
+  parallel: 6
   timeouts:
     assert: 5m0s
     cleanup: 5m0s

@@ -235,7 +235,7 @@ e2e-test:
 	LINODE_TOKEN=$(LINODE_TOKEN) \
 	LINODE_URL=$(LINODE_URL) \
 	E2E_RESERVED_IP_TAG=$(E2E_RESERVED_IP_TAG) \
-	chainsaw test e2e/test --parallel 2 --selector all $(E2E_FLAGS)
+	chainsaw test e2e/test --selector all $(E2E_FLAGS)
 
 .PHONY: e2e-test-ipv6-backends
 e2e-test-ipv6-backends:

@@ -26,8 +26,10 @@ spec:
             shell: bash
             shellArgs:
               - -c
+            timeout: 4m
             content: |
               set -euo pipefail
+              source ../test/scripts/lib.sh
 
               if [ -z "$FIRST_CONFIG" ] || [ -z "$SECOND_CONFIG" ] || [ -z "$LINODE_TOKEN" ]; then
                   echo "Error: FIRST_CONFIG, SECOND_CONFIG, and LINODE_TOKEN environment variables must be set"
@@ -50,44 +52,36 @@ spec:
                 .[0].items[] | [.metadata.name, .spec.podCIDR, .spec.providerID, $foreign] | @tsv
               ')
 
-              # Node Ready does not guarantee that CCM has populated interface routes.
-              # Share one deadline across all nodes, within the suite's 5-minute exec timeout.
-              # SECONDS is Bash's built-in elapsed timer; it advances automatically.
-              deadline=$((SECONDS + 180))
-              while true; do
-                observations=()
+              check_route_state() {
+                local observations=() node pod_cidr provider_id foreign_cidrs interfaces state result
                 while IFS=$'\t' read -r node pod_cidr provider_id foreign_cidrs; do
-                  # Stop starting API requests after the deadline, even partway through a node scan.
-                  (( SECONDS < deadline )) || exit 1
-                  state=$(curl --fail --silent --show-error --max-time 15 \
-                    -H "Authorization: Bearer $LINODE_TOKEN" \
-                    "$LINODE_URL/v4/linode/instances/${provider_id#linode://}/configs" \
-                    | jq -c --arg node "$node" --arg cidr "$pod_cidr" \
-                      --argjson foreign "$foreign_cidrs" '
-                      [.data[0].interfaces[] | select(.purpose == "vpc") | .ip_ranges[]?] as $ranges |
-                      {
-                        node: $node,
-                        own_route_present: ($ranges | index($cidr) != null),
-                        foreign_routes: [$ranges[] | select(. as $range | $foreign | index($range) != null)]
-                      }
-                    ')
-                  # Never retry an isolation violation.
-                  if jq -e '.foreign_routes | length > 0' <<< "$state" > /dev/null; then
-                    # Return JSON successfully so Chainsaw's foreign-route assertion reports the failure.
+                  interfaces=$(linode_cli linodes configs-list "${provider_id#linode://}" --json \
+                    | jq -ce '.[0].interfaces') || return 1
+                  state=$(jq -cn --arg node "$node" --arg cidr "$pod_cidr" \
+                    --argjson foreign "$foreign_cidrs" --argjson interfaces "$interfaces" '
+                    [$interfaces[] | select(.purpose == "vpc") | .ip_ranges[]?] as $ranges |
+                    {
+                      node: $node,
+                      own_route_present: ($ranges | index($cidr) != null),
+                      foreign_routes: [$ranges[] | select(. as $range | $foreign | index($range) != null)]
+                    }
+                  ')
+                  if jq -e '.foreign_routes | length > 0' <<< "$state" >/dev/null; then
                     printf '[%s]\n' "$state"
-                    exit 0
+                    return 0
                   fi
                   observations+=("$state")
                 done <<< "$targets"
-                # Combine node observations into the JSON array consumed by Chainsaw below.
+
                 result=$(printf '%s\n' "${observations[@]}" | jq -s '.')
-                if jq -e 'all(.[]; .own_route_present)' <<< "$result" > /dev/null; then
-                  break
+                if jq -e 'all(.[]; .own_route_present)' <<< "$result" >/dev/null; then
+                  printf '%s\n' "$result"
+                  return 0
                 fi
-                sleep 5
-                (( SECONDS < deadline )) || break
-              done
-              printf '%s\n' "$result"
+                return 1
+              }
+
+              retry 36 5 check_route_state
             check:
               ($error == null): true
               (length(json_parse($stdout)[?own_route_present == `false`])): 0

@@ -21,6 +21,7 @@ spec:
             value: ($fwname)
         content: |
           set -uo pipefail
+          source ../scripts/lib.sh
           echo "Test failed. Fetching CCM logs..."
           kubectl logs -n kube-system daemonsets/ccm-linode | grep "fw-use-specified-nb" | tail -100 || true
 
@@ -29,16 +30,13 @@ spec:
           # deletes it, the firewall would leak. Look it up by its (deterministic)
           # label and delete it here (best-effort, idempotent).
           echo "Cleaning up firewall (if any)..."
-          fwid=$(curl -s \
-            -H "Authorization: Bearer $LINODE_TOKEN" \
-            -H "Content-Type: application/json" \
-            -H "X-Filter: {\"label\": \"$FWLABEL\"}" \
-            "$LINODE_URL/v4/networking/firewalls" | jq -r '.data[0].id // empty' 2>/dev/null || true)
+          fwid=$(linode_firewall_id_by_label "$FWLABEL" 2>/dev/null || true)
           if [[ "$fwid" =~ ^[0-9]+$ ]]; then
             echo "Deleting firewall $fwid"
-            curl -s -o /dev/null -X DELETE \
-              -H "Authorization: Bearer $LINODE_TOKEN" \
-              "$LINODE_URL/v4/networking/firewalls/$fwid" || true
+            if ! linode_cli firewalls delete "$fwid" --json >/dev/null 2>&1 \
+              && ! linode_firewall_absent "$fwid"; then
+              echo "Unable to delete firewall $fwid during test cleanup" >&2
+            fi
           fi
   steps:
     - name: Create firewall, Create pods and services
@@ -54,36 +52,24 @@ spec:
               set -euo pipefail
               source ../scripts/lib.sh
 
-              create_fw=$(curl -s --write-out "%{http_code}\n" --output /dev/null --request POST \
-                -H "Authorization: Bearer $LINODE_TOKEN" \
-                -H "Content-Type: application/json" --fail-early --retry 3 \
-                -H "accept: application/json" \
-                "$LINODE_URL/v4/networking/firewalls" \
-                --data "
-              {
-                \"label\": \"$FWLABEL\",
-                \"rules\": {
-                  \"inbound\": [{
-                    \"action\": \"ACCEPT\",
-                    \"label\": \"inbound-rule123\",
-                    \"description\": \"inbound rule123\",
-                    \"ports\": \"4321\",
-                    \"protocol\": \"TCP\",
-                    \"addresses\": {
-                      \"ipv4\": [\"0.0.0.0/0\"]
-                    }
+              firewall_data=$(jq -cn --arg label "$FWLABEL" '{
+                label: $label,
+                rules: {
+                  inbound: [{
+                    action: "ACCEPT",
+                    label: "inbound-rule123",
+                    description: "inbound rule123",
+                    ports: "4321",
+                    protocol: "TCP",
+                    addresses: {ipv4: ["0.0.0.0/0"]}
                   }],
-                  \"inbound_policy\": \"ACCEPT\",
-                  \"outbound_policy\": \"ACCEPT\"
+                  inbound_policy: "ACCEPT",
+                  outbound_policy: "ACCEPT"
                 }
-              }
-              "
-              )
-
-              if [[ $create_fw == "200" ]]; then
-                  echo "fw created"
-              else
-                  fail "Failed creating firewall, got HTTP $create_fw"
+              }')
+              fwid=$(create_firewall "$firewall_data")
+              if [[ ! "$fwid" =~ ^[0-9]+$ ]]; then
+                fail "Failed creating firewall: $fwid"
               fi
             check:
               ($error == null): true
@@ -115,51 +101,32 @@ spec:
             env:
               - name: FWLABEL
                 value: ($fwname)
+            timeout: 3m
             content: |
               set -euo pipefail
               source ../scripts/lib.sh
               re='^[0-9]+$'
 
-              fwid=$(curl -s \
-                    -H "Authorization: Bearer $LINODE_TOKEN" \
-                    -H "Content-Type: application/json" --fail-early --retry 3 \
-                    -H "X-Filter: {\"label\": \"$FWLABEL\"}" \
-                    "$LINODE_URL/v4/networking/firewalls" | jq .data[].id)
+              fwid=$(linode_firewall_id_by_label "$FWLABEL")
 
               if ! [[ $fwid =~ $re ]]; then
                   echo "Firewall id [$fwid] is incorrect, failed to fetch firewall"
                   exit 1
               fi
 
               kubectl annotate svc svc-test -n $NAMESPACE service.beta.kubernetes.io/linode-loadbalancer-firewall-id=$fwid
-              sleep 5
-
-              matched=false
-              for _ in {1..10}; do
-                  nbid=$(KUBECONFIG=$KUBECONFIG NAMESPACE=$NAMESPACE LINODE_TOKEN=$LINODE_TOKEN ../scripts/get-nb-id.sh)
-
-                  fwconfig=$(curl -s \
-                    -H "Authorization: Bearer $LINODE_TOKEN" \
-                    -H "Content-Type: application/json" --fail-early --retry 3 \
-                    "$LINODE_URL/v4/networking/firewalls/$fwid" || true)
-
-                  fw_attached_to_nb=$(echo $fwconfig | jq ".entities[] | select(.id == $nbid) | .id == $nbid")
-
-                  if [[ $fw_attached_to_nb == "true" ]]; then
-                      echo "Conditions met"
-                      matched=true
-                      break
-                  fi
-
-                  sleep 10
-              done
-              if [[ "$matched" != true ]]; then
+              nbid=$(KUBECONFIG=$KUBECONFIG NAMESPACE=$NAMESPACE LINODE_TOKEN=$LINODE_TOKEN ../scripts/get-nb-id.sh)
+              firewall_attached_to_nodebalancer() {
+                  local firewalls
+                  firewalls=$(linode_firewalls_for_nodebalancer "$nbid") || return 1
+                  jq -e --argjson id "$fwid" 'any(.[]; .id == $id)' <<< "$firewalls" >/dev/null
+              }
+              if ! retry 10 10 firewall_attached_to_nodebalancer; then
                   fail "Firewall $fwid was not attached to NodeBalancer $nbid after 10 attempts"
               fi
 
-              curl -s --fail -X DELETE \
-                -H "Authorization: Bearer $LINODE_TOKEN" \
-                -H "Content-Type: application/json" --fail-early --retry 3 \
-                "$LINODE_URL/v4/networking/firewalls/$fwid"
+              if ! linode_cli firewalls delete "$fwid" --json >/dev/null; then
+                  fail "Failed deleting firewall $fwid"
+              fi
             check:
               ($error == null): true
@@ -77,12 +77,22 @@ spec:
               name: svc-test
             status:
               (loadBalancer.ingress[0].ip == null): true
+    - name: assert invalid IP warning event
+      try:
+        - assert:
+            resource:
+              apiVersion: v1
+              kind: Event
+              type: Warning
+              reason: SyncLoadBalancerFailed
+              "(contains(message, 'Error syncing load balancer: failed to ensure load balancer: [400] Invalid IPv4 address'))": true
     - name: get service ip and compare with invalid ip
       try:
         - script: 
             shell: bash
             shellArgs:
               - -c
+            timeout: 3m
             content: |
               set -euo pipefail
               source ../scripts/lib.sh
@@ -91,22 +101,6 @@ spec:
                   fail "No invalid ip found in configmap"
               fi
 
-              warning_found=false
-              for _ in {1..10}; do
-                events=$(kubectl get events -n $NAMESPACE --field-selector reason=SyncLoadBalancerFailed --sort-by='.lastTimestamp' -o json)
-                message=$(echo $events | jq .items[0].message)
-
-                if [[ "$message" == *"Error syncing load balancer: failed to ensure load balancer: [400] Invalid IPv4 address"* ]]; then
-                  echo "Warning event found"
-                  warning_found=true
-                  break
-                fi
-                sleep 10
-              done
-              if [[ "$warning_found" != true ]]; then
-                fail "Warning event not found after 10 attempts: expected invalid IPv4 address event"
-              fi
-
               service_ip=$(kubectl get svc svc-test -n $NAMESPACE -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
               if [[ "$service_ip" != "" ]]; then
                   fail "Service ip found for service svc-test: expected empty, got $service_ip"

@@ -15,6 +15,7 @@ spec:
           - -c
         content: |
           set -uo pipefail
+          source ../scripts/lib.sh
           echo "Test failed. Fetching CCM logs..."
           kubectl logs -n kube-system daemonsets/ccm-linode | grep "lb-created-with-new-nb-id" | tail -100 || true
 
@@ -31,9 +32,7 @@ spec:
           for nbid in $nbids; do
             if [[ "$nbid" =~ ^[0-9]+$ ]]; then
               echo "Deleting NodeBalancer $nbid"
-              curl -s -o /dev/null -X DELETE \
-                -H "Authorization: Bearer $LINODE_TOKEN" \
-                "$LINODE_URL/v4/nodebalancers/$nbid" || true
+              linode_cli nodebalancers delete "$nbid" --json >/dev/null || echo "WARNING: failed to delete NodeBalancer $nbid; it may have leaked" >&2
             fi
           done
   steps:
@@ -74,7 +73,7 @@ spec:
               # Get data to use for new nodebalancer
               data=$(KUBECONFIG=$KUBECONFIG NAMESPACE=$NAMESPACE LINODE_TOKEN=$LINODE_TOKEN LABEL=$LABEL REGION=$REGION ../scripts/get-nb-create-data.sh svc-dummy)
 
-              nbid=$(retry 3 5 linode_api POST /v4beta/nodebalancers --data "$data" | jq -er '.id')
+              nbid=$(create_nodebalancer "$data")
 
               if ! [[ $nbid =~ $re ]]; then
                   echo "Nodebalancer id [$nbid] is incorrect, failed to create nodebalancer"
@@ -133,6 +132,7 @@ spec:
             shell: bash
             shellArgs:
               - -c
+            timeout: 3m
             content: |
               set -euo pipefail
               source ../scripts/lib.sh
@@ -144,23 +144,17 @@ spec:
               LABEL="ccm-$(head /dev/urandom | tr -dc 'a-z0-9' | head -c 5)"
 
               # Get VPC config if it exists
-              vpcconfig=$(curl -s \
-                -H "Authorization: Bearer $LINODE_TOKEN" \
-                -H "Content-Type: application/json" --fail-early --retry 3 \
-                "$LINODE_URL/v4beta/nodebalancers/$old_nbid/vpcs")
+              vpcconfig=$(linode_cli nodebalancers vpcs-list "$old_nbid" --json)
 
               echo "Old Nodebalancer ID: $old_nbid"
               echo "VPC config: $vpcconfig"
 
-              SUBNET_ID=$(echo $vpcconfig | jq -r ".data[] | select(.nodebalancer_id == $old_nbid) | .subnet_id")
+              SUBNET_ID=$(jq -er --argjson id "$old_nbid" '.[] | select(.nodebalancer_id == $id) | .subnet_id' <<< "$vpcconfig" 2>/dev/null || true)
               echo "Subnet ID: $SUBNET_ID"
 
-              data="{\"label\": \"$LABEL\", \"region\": \"$REGION\", \"vpcs\": [{\"subnet_id\": $SUBNET_ID}]}"
-              if [[ -z $SUBNET_ID ]]; then
-                  data="{\"label\": \"$LABEL\", \"region\": \"$REGION\"}"
-              fi
+              data=$(nodebalancer_payload "$LABEL" "$REGION" "$SUBNET_ID")
 
-              nbid=$(retry 3 5 linode_api POST /v4beta/nodebalancers --data "$data" | jq -er '.id')
+              nbid=$(create_nodebalancer "$data")
 
               if ! [[ $nbid =~ $re ]]; then
                   fail "Nodebalancer id [$nbid] is incorrect, failed to create nodebalancer"
@@ -172,31 +166,19 @@ spec:
 
                   kubectl annotate --overwrite svc svc-test -n $NAMESPACE service.beta.kubernetes.io/linode-loadbalancer-nodebalancer-id=$nbid
 
-                  matched=false
-                  for _ in {1..10}; do
-                      nbid2=$(KUBECONFIG=$KUBECONFIG NAMESPACE=$NAMESPACE LINODE_TOKEN=$LINODE_TOKEN ../scripts/get-nb-id.sh)
-
-                      if [[ "$nbid" == "$nbid2" ]]; then
-                          echo "Condition met"
-                          matched=true
-                          break
-                      fi
-                      sleep 5
-                  done
-                  if [[ "$matched" != true ]]; then
+                  check_new_nodebalancer_adopted() {
+                      local current_nbid
+                      current_nbid=$(KUBECONFIG=$KUBECONFIG NAMESPACE=$NAMESPACE LINODE_TOKEN=$LINODE_TOKEN ../scripts/get-nb-id.sh) || return 1
+                      [[ "$current_nbid" == "$nbid" ]]
+                  }
+                  if ! retry 10 5 check_new_nodebalancer_adopted; then
                       fail "Service did not adopt new NodeBalancer $nbid after 10 attempts"
                   fi
               fi
 
-              old_nb_resp=$(curl --write-out "%{http_code}\n" \
-                --silent --output /dev/null \
-                -H "Authorization: Bearer $LINODE_TOKEN" \
-                "$LINODE_URL/v4/nodebalancers/$old_nbid")
-
-              if [[ $old_nb_resp == "404" ]]; then
-                  echo "old nodebalancer not found"
-              else
-                  fail "Expected old NodeBalancer $old_nbid to be deleted, got HTTP $old_nb_resp"
+              if ! linode_nodebalancer_absent "$old_nbid"; then
+                  fail "Expected old NodeBalancer $old_nbid to be deleted"
               fi
+              echo "old nodebalancer not found"
             check:
               ($error == null): true