From 2d8817d5e27a61188dcb24498902c561b65041eb Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Tue, 17 Mar 2026 16:15:38 +0100
Subject: [PATCH 01/72] [regression] Multiple bug fixes

Correctly get regression cache path, select architecture, and pass resource prefix
---
 sebs/cli.py        | 28 ++++++++++++++++++----------
 sebs/regression.py | 40 +++++++++++++++++++++++++++-------------
 2 files changed, 45 insertions(+), 23 deletions(-)

diff --git a/sebs/cli.py b/sebs/cli.py
index 4e1cc558..ad312de3 100755
--- a/sebs/cli.py
+++ b/sebs/cli.py
@@ -455,32 +455,40 @@ def package(
     multiple=True,
     help="JSON configuration of deployed storage.",
 )
-@common_params
-@click.option(
-    "--cache",
-    default=os.path.join(os.path.curdir, "regression-cache"),
-    help="Location of experiments cache.",
-)
 @click.option(
-    "--output-dir",
-    default=os.path.join(os.path.curdir, "regression-output"),
-    help="Output directory for results.",
+    "--selected-architecture/--all-architectures",
+    type=bool,
+    default=False,
+    help="Skip non-selected CPU architectures.",
 )
-def regression(benchmark_input_size, benchmark_name, storage_configuration, **kwargs):
+@common_params
+def regression(
+    benchmark_input_size, benchmark_name, storage_configuration, selected_architecture, **kwargs
+):
     """Run regression test suite across benchmarks."""
+
     # for regression, deployment client is initialized locally
     # disable default initialization
+
+    from pathlib import Path
+
+    if Path(kwargs["cache"]) == Path("cache"):
+        kwargs["cache"] = os.path.join(os.path.curdir, "regression-cache")
+
     (config, output_dir, logging_filename, sebs_client, _) = parse_common_params(
         initialize_deployment=False,
         storage_configuration=storage_configuration,
         **kwargs,
     )
+    architecture = config["experiments"]["architecture"] if selected_architecture else None
     regression_suite(
         sebs_client,
         config["experiments"],
         set((config["deployment"]["name"],)),
         config,
+        kwargs["resource_prefix"],
         benchmark_name,
+        architecture,
     )
 
 
diff --git a/sebs/regression.py b/sebs/regression.py
index 53336a2a..6221df7f 100644
--- a/sebs/regression.py
+++ b/sebs/regression.py
@@ -80,6 +80,8 @@
 # User-defined config passed during initialization, set in regression_suite()
 cloud_config: Optional[dict] = None
 
+RESOURCE_PREFIX = "regr"
+
 
 class TestSequenceMeta(type):
     """Metaclass for dynamically generating regression test cases.
@@ -335,7 +337,7 @@ def get_deployment(self, benchmark_name, architecture, deployment_type):
 
         # Synchronize resource initialization with a lock
         with AWSTestSequencePython.lock:
-            deployment_client.initialize(resource_prefix="regr")
+            deployment_client.initialize(resource_prefix=RESOURCE_PREFIX)
         return deployment_client
 
 
@@ -389,7 +391,7 @@ def get_deployment(self, benchmark_name, architecture, deployment_type):
 
         # Synchronize resource initialization with a lock
         with AWSTestSequenceNodejs.lock:
-            deployment_client.initialize(resource_prefix="regr")
+            deployment_client.initialize(resource_prefix=RESOURCE_PREFIX)
         return deployment_client
 
 
@@ -432,7 +434,7 @@ def get_deployment(self, benchmark_name, architecture, deployment_type):
             logging_filename=os.path.join(self.client.output_dir, f),
         )
         with AWSTestSequenceCpp.lock:
-            deployment_client.initialize(resource_prefix="regr")
+            deployment_client.initialize(resource_prefix=RESOURCE_PREFIX)
         return deployment_client
 
 
@@ -483,7 +485,7 @@ def get_deployment(self, benchmark_name, architecture, deployment_type):
             logging_filename=os.path.join(self.client.output_dir, f),
         )
         with AWSTestSequenceJava.lock:
-            deployment_client.initialize(resource_prefix="regr")
+            deployment_client.initialize(resource_prefix=RESOURCE_PREFIX)
         return deployment_client
 
 
@@ -565,7 +567,7 @@ def get_deployment(self, benchmark_name, architecture, deployment_type):
             deployment_client.system_resources.initialize_cli(
                 cli=AzureTestSequencePython.cli, login=True
             )
-            deployment_client.initialize(resource_prefix="regr")
+            deployment_client.initialize(resource_prefix=RESOURCE_PREFIX)
             return deployment_client
 
 
@@ -642,7 +644,7 @@ def get_deployment(self, benchmark_name, architecture, deployment_type):
 
             # Initialize CLI and setup resources (no login needed - reuses Python session)
             deployment_client.system_resources.initialize_cli(cli=AzureTestSequenceNodejs.cli)
-            deployment_client.initialize(resource_prefix="regr")
+            deployment_client.initialize(resource_prefix=RESOURCE_PREFIX)
             return deployment_client
 
 
@@ -716,7 +718,7 @@ def get_deployment(self, benchmark_name, architecture, deployment_type):
             deployment_client.system_resources.initialize_cli(
                 cli=AzureTestSequenceJava.cli, login=needs_login
             )
-            deployment_client.initialize(resource_prefix="regr")
+            deployment_client.initialize(resource_prefix=RESOURCE_PREFIX)
             return deployment_client
 
 
@@ -770,7 +772,7 @@ def get_deployment(self, benchmark_name, architecture, deployment_type):
 
         # Synchronize resource initialization with a lock
         with GCPTestSequencePython.lock:
-            deployment_client.initialize(resource_prefix="regr")
+            deployment_client.initialize(resource_prefix=RESOURCE_PREFIX)
         return deployment_client
 
 
@@ -824,7 +826,7 @@ def get_deployment(self, benchmark_name, architecture, deployment_type):
 
         # Synchronize resource initialization with a lock
         with GCPTestSequenceNodejs.lock:
-            deployment_client.initialize(resource_prefix="regr")
+            deployment_client.initialize(resource_prefix=RESOURCE_PREFIX)
         return deployment_client
 
 
@@ -878,7 +880,7 @@ def get_deployment(self, benchmark_name, architecture, deployment_type):
 
         # Synchronize resource initialization with a lock
         with GCPTestSequenceJava.lock:
-            deployment_client.initialize(resource_prefix="regr")
+            deployment_client.initialize(resource_prefix=RESOURCE_PREFIX)
         return deployment_client
 
 
@@ -936,7 +938,7 @@ def get_deployment(self, benchmark_name, architecture, deployment_type):
 
         # Synchronize resource initialization with a lock
         with OpenWhiskTestSequencePython.lock:
-            deployment_client.initialize(resource_prefix="regr")
+            deployment_client.initialize(resource_prefix=RESOURCE_PREFIX)
         return deployment_client
 
 
@@ -994,7 +996,7 @@ def get_deployment(self, benchmark_name, architecture, deployment_type):
 
         # Synchronize resource initialization with a lock
         with OpenWhiskTestSequenceNodejs.lock:
-            deployment_client.initialize(resource_prefix="regr")
+            deployment_client.initialize(resource_prefix=RESOURCE_PREFIX)
         return deployment_client
 
 
@@ -1048,7 +1050,7 @@ def get_deployment(self, benchmark_name, architecture, deployment_type):
 
         # Synchronize resource initialization with a lock
         with OpenWhiskTestSequenceJava.lock:
-            deployment_client.initialize(resource_prefix="regr")
+            deployment_client.initialize(resource_prefix=RESOURCE_PREFIX)
         return deployment_client
 
 
@@ -1123,6 +1125,7 @@ def filter_out_benchmarks(
     language_version: str,
     architecture: str,
     deployment_type: str,
+    selected_architecture: str | None = None,
 ) -> bool:
     """Filter out benchmarks that are not supported on specific platforms.
 
@@ -1142,6 +1145,10 @@ def filter_out_benchmarks(
     """
     # fmt: off
 
+    # user can asks to use only a selected architecture
+    if selected_architecture is not None and selected_architecture != architecture:
+        return False
+
     # Arm architecture currently not supported for C++
     if (language == "cpp" and architecture == "arm64"):
         return False
@@ -1174,7 +1181,9 @@ def regression_suite(
     experiment_config: dict,
     providers: Set[str],
     deployment_config: dict,
+    resource_prefix: str = "regr",
     benchmark_name: Optional[str] = None,
+    selected_architecture: str | None = None,
 ):
     """Create and run a regression test suite for specified cloud providers.
 
@@ -1195,6 +1204,10 @@ def regression_suite(
     Raises:
         AssertionError: If a requested provider is not in the deployment config
     """
+
+    global RESOURCE_PREFIX
+    RESOURCE_PREFIX = resource_prefix
+
     # Create the test suite
     suite = unittest.TestSuite()
 
@@ -1279,6 +1292,7 @@ def regression_suite(
                 language_version,
                 test_architecture,
                 test_deployment_type,
+                selected_architecture,
             ):
                 print(f"Skip test {test_name} - not supported.")
                 continue

From 7cbc681bd2f820b54b8989358c0f6c9dbd592706 Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Tue, 17 Mar 2026 16:17:40 +0100
Subject: [PATCH 02/72] [ci] Update test job

---
 .circleci/config.yml | 135 ++++++++++++++++++++++++++++++++-----------
 1 file changed, 102 insertions(+), 33 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index b22c4edc..e393dc3a 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -3,6 +3,36 @@ version: 2.1
 orbs:
   python: circleci/python@2.1
 
+# Executor for regression testing jobs
+executors:
+  sebs-regression:
+    docker:
+      - image: cimg/python:3.11
+    resource_class: large
+    environment:
+      RESOURCE_PREFIX: sebs-ci
+
+commands:
+
+  restore-sebs-cache:
+    description: "Restore SeBS cache directory containing cloud resource metadata"
+    steps:
+      - restore_cache:
+          keys:
+            - sebs-cache-{{ .Branch }}
+
+  save-caches:
+    description: "Persist SeBS cache and dependencies"
+    parameters:
+      language:
+        type: enum
+        enum: [python, nodejs, java, cpp]
+    steps:
+      - save_cache:
+          key: sebs-cache-{{ .Branch }}
+          paths:
+            - cache/
+
 jobs:
   linting:
     executor:
@@ -43,46 +73,85 @@ jobs:
       - store_artifacts:
           path: flake-reports
           destination: flake-reports
-  test-aws:
-    executor: python/default
+
+  install-sebs:
+    description: "Install SeBS with platform-specific dependencies"
+    parameters:
+      platform:
+        type: enum
+        enum: [aws, azure, gcp]
     steps:
-      - checkout
-      - setup_remote_docker
-      - restore_cache:
-          key: deps1-{{ .Branch }}-{{ checksum "requirements.txt" }}
-      - run:
-          command: |
-            if [[ -d $HOME/docker ]];
-            then
-              ls $HOME/docker/*.tar.gz | xargs -I {file} sh -c "zcat {file} | docker load";
-            else
-              docker pull mcopik/serverless-benchmarks:build.aws.python.3.7
-              docker pull mcopik/serverless-benchmarks:build.aws.nodejs.12.x
-            fi
-          name: Load Docker images
       - run:
-          command: |
-            python3 install.py --aws
-          name: Install pip dependencies
-      - run:
-          command: |
-            mkdir -p $HOME/docker
-            docker images mcopik/serverless-benchmarks --filter='dangling=false' --format '{{.Repository}}:{{.Tag}} {{.ID}}' |\
-            xargs -n 2 -t sh -c 'test -e $HOME/docker/$1.tar.gz || docker save $0 | gzip -2 > $HOME/docker/$1.tar.gz'
-          name: Save Docker images
-      - save_cache:
-          key: deps1-{{ .Branch }}-{{ checksum "requirements.txt" }}
-          paths:
-            - "sebs-virtualenv"
-            - $HOME/docker
+          name: Install SeBS
+          command: pip install .
+
+  setup-cloud-credentials:
+    description: "Configure cloud authentication"
+    parameters:
+      platform:
+        type: enum
+        enum: [aws, azure, gcp]
+    steps:
+      - when:
+          condition:
+            equal: [gcp, << parameters.platform >>]
+          steps:
+            - run:
+                name: Setup GCP Credentials
+                command: |
+                  echo "$GCP_SERVICE_ACCOUNT_JSON" > /tmp/gcp-credentials.json
+                  echo 'export GOOGLE_APPLICATION_CREDENTIALS=/tmp/gcp-credentials.json' >> $BASH_ENV
+
+  run-regression-tests:
+    description: "Execute regression test suite"
+    parameters:
+      platform:
+        type: enum
+        enum: [aws, azure, gcp]
+      language:
+        type: enum
+        enum: [python, nodejs, java, cpp]
+      version:
+        type: string
+    steps:
       - run:
+          name: Run Regression Tests
           command: |
-            . sebs-virtualenv/bin/activate
-            tests/test_runner.py --deployment aws
-          name: Execute AWS tests
+            sebs benchmark regression \
+              --config config/example.json \
+              --deployment << parameters.platform >> \
+              --language << parameters.language >> \
+              --language-version << parameters.version >>
+          no_output_timeout: 5m
+
+  regression-aws-python311:
+    executor: sebs-regression
+    steps:
+      - checkout
+      - restore-sebs-cache
+      - setup_remote_docker:
+          version: 20.10.24
+      - setup-cloud-credentials:
+          platform: aws
+      - install-sebs:
+          platform: aws
+      - run-regression-tests:
+          platform: aws
+          language: python
+          version: "3.11"
+      - save-results
+      - save-caches:
+          language: python
 
 workflows:
   main:
     jobs:
       - linting
 
+  regression-tests:
+    jobs:
+      # AWS jobs
+      - regression-aws-python311:
+          filters:
+            branches:
+              only: master

From eb4d0e02f99fe02c73c45e2915d029d1877bd6bc Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Tue, 17 Mar 2026 16:24:05 +0100
Subject: [PATCH 03/72] [ci] Update test job

---
 .circleci/config.yml | 115 ++++++++++++++++++++++++++++---------------
 1 file changed, 74 insertions(+), 41 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index e393dc3a..2f9a24b4 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -33,47 +33,6 @@ commands:
           paths:
             - cache/
 
-jobs:
-  linting:
-    executor:
-       name: 'python/default'
-       tag: '3.10'
-    steps:
-      - checkout
-      - restore_cache:
-          key: deps1-{{ .Branch }}-{{ checksum "requirements.txt" }}
-      - run:
-          command: |
-            sudo apt update && sudo apt install libcurl4-openssl-dev
-          name: Install curl-config from Ubuntu APT
-      - run:
-          command: |
-            python3 install.py --aws --azure --gcp --no-local
-          name: Install pip dependencies
-      - run:
-          command: |
-            . python-venv/bin/activate
-            black sebs --check --config .black.toml
-          name: Python code formatting with black
-      - run:
-          command: |
-            . python-venv/bin/activate
-            flake8 sebs --config=.flake8.cfg --tee --output-file flake-reports
-          name: Python code lint with flake8
-      - run:
-          command: |
-            . python-venv/bin/activate
-            mypy sebs --config-file=.mypy.ini
-          name: Python static code verification with mypy
-      - run:
-          command: |
-            . python-venv/bin/activate
-            interrogate -v --fail-under 100 sebs
-          name: Check for Python documentation coverage 
-      - store_artifacts:
-          path: flake-reports
-          destination: flake-reports
-
   install-sebs:
     description: "Install SeBS with platform-specific dependencies"
     parameters:
@@ -124,6 +83,80 @@ jobs:
               --language-version << parameters.version >>
           no_output_timeout: 5m
 
+  save-results:
+    description: "Save benchmark results as artifacts"
+    steps:
+      - run:
+          name: Generate Test Summary
+          command: |
+            echo "Regression Test Summary" > test-summary.txt
+            echo "======================" >> test-summary.txt
+            if ls regression_*.json 1> /dev/null 2>&1; then
+              ls -1 regression_*.json | wc -l | xargs echo "Benchmarks tested:" >> test-summary.txt
+              echo "" >> test-summary.txt
+              echo "Results saved to artifacts/results/" >> test-summary.txt
+            else
+              echo "No benchmark results found" >> test-summary.txt
+            fi
+          when: always
+      - store_artifacts:
+          path: test-summary.txt
+      - run:
+          name: Collect regression results
+          command: |
+            mkdir -p results
+            if ls regression_*.json 1> /dev/null 2>&1; then
+              mv regression_*.json results/ || true
+            fi
+          when: always
+      - store_artifacts:
+          path: results
+          destination: results/
+      - store_artifacts:
+          path: cache
+          destination: cache-snapshot/
+
+jobs:
+  linting:
+    executor:
+       name: 'python/default'
+       tag: '3.10'
+    steps:
+      - checkout
+      - restore_cache:
+          key: deps1-{{ .Branch }}-{{ checksum "requirements.txt" }}
+      - run:
+          command: |
+            sudo apt update && sudo apt install libcurl4-openssl-dev
+          name: Install curl-config from Ubuntu APT
+      - run:
+          command: |
+            python3 install.py --aws --azure --gcp --no-local
+          name: Install pip dependencies
+      - run:
+          command: |
+            . python-venv/bin/activate
+            black sebs --check --config .black.toml
+          name: Python code formatting with black
+      - run:
+          command: |
+            . python-venv/bin/activate
+            flake8 sebs --config=.flake8.cfg --tee --output-file flake-reports
+          name: Python code lint with flake8
+      - run:
+          command: |
+            . python-venv/bin/activate
+            mypy sebs --config-file=.mypy.ini
+          name: Python static code verification with mypy
+      - run:
+          command: |
+            . python-venv/bin/activate
+            interrogate -v --fail-under 100 sebs
+          name: Check for Python documentation coverage 
+      - store_artifacts:
+          path: flake-reports
+          destination: flake-reports
+
   regression-aws-python311:
     executor: sebs-regression
     steps:

From 2b19309455fa45db41b806892d1a3f4e9dc4ad78 Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Tue, 17 Mar 2026 16:28:31 +0100
Subject: [PATCH 04/72] [ci] Enable CI job on feature branches

---
 .circleci/config.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 2f9a24b4..69f7527e 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -187,4 +187,6 @@ workflows:
       - regression-aws-python311:
           filters:
             branches:
-              only: master
+              only:
+                - master
+                - /feature\/.*/

From 09a350ca56b67fc9d6f03148a92f5d0d4912b15e Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Tue, 17 Mar 2026 16:32:50 +0100
Subject: [PATCH 05/72] [ci] Fix CI command

---
 .circleci/config.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 69f7527e..cc1438d8 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -76,11 +76,13 @@ commands:
       - run:
           name: Run Regression Tests
           command: |
-            sebs benchmark regression \
+            sebs benchmark regression test \
               --config config/example.json \
               --deployment << parameters.platform >> \
               --language << parameters.language >> \
-              --language-version << parameters.version >>
+              --language-version << parameters.version >> \
+              --architecture x64 --selected-architecture \
+              --resource-prefix sebs-ci
           no_output_timeout: 5m
 
   save-results:

From e9185a1246a5481f27f4648ac4b9f810085b1b2e Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Tue, 17 Mar 2026 16:36:37 +0100
Subject: [PATCH 06/72] [ci] Fix CI command

---
 .circleci/config.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index cc1438d8..bead91a6 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -76,8 +76,9 @@ commands:
       - run:
           name: Run Regression Tests
           command: |
+            set -euo pipefail
             sebs benchmark regression test \
-              --config config/example.json \
+              --config configs/example.json \
               --deployment << parameters.platform >> \
               --language << parameters.language >> \
               --language-version << parameters.version >> \

From 53341689d34a0443f32c7b978dcfaaf07913c145 Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Tue, 17 Mar 2026 17:15:46 +0100
Subject: [PATCH 07/72] [regression] Non-zero return from failed registration

---
 sebs/cli.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/sebs/cli.py b/sebs/cli.py
index ad312de3..870d8603 100755
--- a/sebs/cli.py
+++ b/sebs/cli.py
@@ -11,6 +11,7 @@
 import logging
 import functools
 import os
+import sys
 import traceback
 from typing import cast, List, Optional
 
@@ -481,7 +482,7 @@ def regression(
         **kwargs,
     )
     architecture = config["experiments"]["architecture"] if selected_architecture else None
-    regression_suite(
+    has_failures = regression_suite(
         sebs_client,
         config["experiments"],
         set((config["deployment"]["name"],)),
@@ -490,6 +491,8 @@ def regression(
         benchmark_name,
         architecture,
     )
+    # Exit with non-zero code if any tests failed
+    sys.exit(1 if has_failures else 0)
 
 
 @cli.group()

From 1fac84bce2922530cf752930e3c7861b7231376d Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Tue, 17 Mar 2026 17:16:07 +0100
Subject: [PATCH 08/72] [aws] Remove unnecessary pkg_resources from container

---
 benchmarks/wrappers/aws/python/setup.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/benchmarks/wrappers/aws/python/setup.py b/benchmarks/wrappers/aws/python/setup.py
index c34245e4..51d9c5f8 100644
--- a/benchmarks/wrappers/aws/python/setup.py
+++ b/benchmarks/wrappers/aws/python/setup.py
@@ -1,14 +1,9 @@
 # Copyright 2020-2025 ETH Zurich and the SeBS authors. All rights reserved.
 from distutils.core import setup
 from glob import glob
-from pkg_resources import parse_requirements
-
-with open('requirements.txt') as f:
-    requirements = [str(r) for r in parse_requirements(f)]
 
 setup(
     name='function',
-    install_requires=requirements,
     packages=['function'],
     package_dir={'function': '.'},
     package_data={'function': glob('**', recursive=True)},

From a0d82f2c039a5388d27b195a6ae00effef051c6a Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Tue, 17 Mar 2026 17:47:50 +0100
Subject: [PATCH 09/72] [ci] Fixes

---
 .circleci/config.yml | 2 +-
 sebs/regression.py   | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index bead91a6..f070c6be 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -31,7 +31,7 @@ commands:
       - save_cache:
           key: sebs-cache-{{ .Branch }}
           paths:
-            - cache/
+            - regression-cache/
 
   install-sebs:
     description: "Install SeBS with platform-specific dependencies"
diff --git a/sebs/regression.py b/sebs/regression.py
index 6221df7f..da8a3e6e 100644
--- a/sebs/regression.py
+++ b/sebs/regression.py
@@ -1181,7 +1181,7 @@ def regression_suite(
     experiment_config: dict,
     providers: Set[str],
     deployment_config: dict,
-    resource_prefix: str = "regr",
+    resource_prefix: str | None = None,
     benchmark_name: Optional[str] = None,
     selected_architecture: str | None = None,
 ):
@@ -1206,7 +1206,8 @@ def regression_suite(
     """
 
     global RESOURCE_PREFIX
-    RESOURCE_PREFIX = resource_prefix
+    if resource_prefix is not None:
+        RESOURCE_PREFIX = resource_prefix
 
     # Create the test suite
     suite = unittest.TestSuite()

From 7454aac5e1f2721d36053063b77d7c7355e6408b Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Wed, 15 Apr 2026 18:20:14 +0200
Subject: [PATCH 10/72] [ci] Migrate from CircleCI to GH Actions

---
 .circleci/config.yml                  | 195 --------------------------
 .github/workflows/_regression-job.yml | 126 +++++++++++++++++
 .github/workflows/lint.yml            |  55 ++++++++
 .github/workflows/regression.yml      |  25 ++++
 4 files changed, 206 insertions(+), 195 deletions(-)
 delete mode 100644 .circleci/config.yml
 create mode 100644 .github/workflows/_regression-job.yml
 create mode 100644 .github/workflows/lint.yml
 create mode 100644 .github/workflows/regression.yml

diff --git a/.circleci/config.yml b/.circleci/config.yml
deleted file mode 100644
index f070c6be..00000000
--- a/.circleci/config.yml
+++ /dev/null
@@ -1,195 +0,0 @@
-version: 2.1
-
-orbs:
-  python: circleci/python@2.1
-
-# Executor for regression testing jobs
-executors:
-  sebs-regression:
-    docker:
-      - image: cimg/python:3.11
-    resource_class: large
-    environment:
-      RESOURCE_PREFIX: sebs-ci
-
-commands:
-
-  restore-sebs-cache:
-    description: "Restore SeBS cache directory containing cloud resource metadata"
-    steps:
-      - restore_cache:
-          keys:
-            - sebs-cache-{{ .Branch }}
-
-  save-caches:
-    description: "Persist SeBS cache and dependencies"
-    parameters:
-      language:
-        type: enum
-        enum: [python, nodejs, java, cpp]
-    steps:
-      - save_cache:
-          key: sebs-cache-{{ .Branch }}
-          paths:
-            - regression-cache/
-
-  install-sebs:
-    description: "Install SeBS with platform-specific dependencies"
-    parameters:
-      platform:
-        type: enum
-        enum: [aws, azure, gcp]
-    steps:
-      - run:
-          name: Install SeBS
-          command: pip install .
-
-  setup-cloud-credentials:
-    description: "Configure cloud authentication"
-    parameters:
-      platform:
-        type: enum
-        enum: [aws, azure, gcp]
-    steps:
-      - when:
-          condition:
-            equal: [gcp, << parameters.platform >>]
-          steps:
-            - run:
-                name: Setup GCP Credentials
-                command: |
-                  echo "$GCP_SERVICE_ACCOUNT_JSON" > /tmp/gcp-credentials.json
-                  echo 'export GOOGLE_APPLICATION_CREDENTIALS=/tmp/gcp-credentials.json' >> $BASH_ENV
-
-  run-regression-tests:
-    description: "Execute regression test suite"
-    parameters:
-      platform:
-        type: enum
-        enum: [aws, azure, gcp]
-      language:
-        type: enum
-        enum: [python, nodejs, java, cpp]
-      version:
-        type: string
-    steps:
-      - run:
-          name: Run Regression Tests
-          command: |
-            set -euo pipefail
-            sebs benchmark regression test \
-              --config configs/example.json \
-              --deployment << parameters.platform >> \
-              --language << parameters.language >> \
-              --language-version << parameters.version >> \
-              --architecture x64 --selected-architecture \
-              --resource-prefix sebs-ci
-          no_output_timeout: 5m
-
-  save-results:
-    description: "Save benchmark results as artifacts"
-    steps:
-      - run:
-          name: Generate Test Summary
-          command: |
-            echo "Regression Test Summary" > test-summary.txt
-            echo "======================" >> test-summary.txt
-            if ls regression_*.json 1> /dev/null 2>&1; then
-              ls -1 regression_*.json | wc -l | xargs echo "Benchmarks tested:" >> test-summary.txt
-              echo "" >> test-summary.txt
-              echo "Results saved to artifacts/results/" >> test-summary.txt
-            else
-              echo "No benchmark results found" >> test-summary.txt
-            fi
-          when: always
-      - store_artifacts:
-          path: test-summary.txt
-      - run:
-          name: Collect regression results
-          command: |
-            mkdir -p results
-            if ls regression_*.json 1> /dev/null 2>&1; then
-              mv regression_*.json results/ || true
-            fi
-          when: always
-      - store_artifacts:
-          path: results
-          destination: results/
-      - store_artifacts:
-          path: cache
-          destination: cache-snapshot/
-
-jobs:
-  linting:
-    executor:
-       name: 'python/default'
-       tag: '3.10'
-    steps:
-      - checkout
-      - restore_cache:
-          key: deps1-{{ .Branch }}-{{ checksum "requirements.txt" }}
-      - run:
-          command: |
-            sudo apt update && sudo apt install libcurl4-openssl-dev
-          name: Install curl-config from Ubuntu APT
-      - run:
-          command: |
-            python3 install.py --aws --azure --gcp --no-local
-          name: Install pip dependencies
-      - run:
-          command: |
-            . python-venv/bin/activate
-            black sebs --check --config .black.toml
-          name: Python code formatting with black
-      - run:
-          command: |
-            . python-venv/bin/activate
-            flake8 sebs --config=.flake8.cfg --tee --output-file flake-reports
-          name: Python code lint with flake8
-      - run:
-          command: |
-            . python-venv/bin/activate
-            mypy sebs --config-file=.mypy.ini
-          name: Python static code verification with mypy
-      - run:
-          command: |
-            . python-venv/bin/activate
-            interrogate -v --fail-under 100 sebs
-          name: Check for Python documentation coverage 
-      - store_artifacts:
-          path: flake-reports
-          destination: flake-reports
-
-  regression-aws-python311:
-    executor: sebs-regression
-    steps:
-      - checkout
-      - restore-sebs-cache
-      - setup_remote_docker:
-          version: 20.10.24
-      - setup-cloud-credentials:
-          platform: aws
-      - install-sebs:
-          platform: aws
-      - run-regression-tests:
-          platform: aws
-          language: python
-          version: "3.11"
-      - save-results
-      - save-caches:
-          language: python
-
-workflows:
-  main:
-    jobs:
-      - linting
-
-  regression-tests:
-    jobs:
-      # AWS jobs
-      - regression-aws-python311:
-          filters:
-            branches:
-              only:
-                - master
-                - /feature\/.*/
diff --git a/.github/workflows/_regression-job.yml b/.github/workflows/_regression-job.yml
new file mode 100644
index 00000000..e232d76f
--- /dev/null
+++ b/.github/workflows/_regression-job.yml
@@ -0,0 +1,126 @@
+name: Regression Job (Reusable)
+
+on:
+  workflow_call:
+    inputs:
+      platform:
+        required: true
+        type: string
+      language:
+        required: true
+        type: string
+      version:
+        required: true
+        type: string
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    env:
+      RESOURCE_PREFIX: sebs-ci
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Restore SeBS cache
+        uses: actions/cache/restore@v4
+        with:
+          path: regression-cache/
+          key: sebs-cache-${{ github.ref_name }}-${{ inputs.platform }}-${{ inputs.language }}-${{ inputs.version }}
+          restore-keys: |
+            sebs-cache-${{ github.ref_name }}-
+
+      - name: Setup GCP credentials
+        if: inputs.platform == 'gcp'
+        run: |
+          echo "${{ secrets.GCP_SERVICE_ACCOUNT_JSON }}" > /tmp/gcp-credentials.json
+          echo "GOOGLE_APPLICATION_CREDENTIALS=/tmp/gcp-credentials.json" >> $GITHUB_ENV
+
+      - name: Setup Azure credentials
+        if: inputs.platform == 'azure'
+        run: |
+          echo "AZURE_SUBSCRIPTION_ID=${{ secrets.AZURE_SUBSCRIPTION_ID }}" >> $GITHUB_ENV
+          echo "AZURE_TENANT_ID=${{ secrets.AZURE_TENANT_ID }}" >> $GITHUB_ENV
+          echo "AZURE_CLIENT_ID=${{ secrets.AZURE_CLIENT_ID }}" >> $GITHUB_ENV
+          echo "AZURE_CLIENT_SECRET=${{ secrets.AZURE_CLIENT_SECRET }}" >> $GITHUB_ENV
+
+      - name: Setup AWS credentials
+        if: inputs.platform == 'aws'
+        run: |
+          echo "AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}" >> $GITHUB_ENV
+          echo "AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}" >> $GITHUB_ENV
+          echo "AWS_DEFAULT_REGION=${{ secrets.AWS_DEFAULT_REGION || 'us-east-1' }}" >> $GITHUB_ENV
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Install SeBS
+        run: uv pip install .
+
+      - name: Run regression tests
+        timeout-minutes: 5
+        run: |
+          sebs benchmark regression test \
+            --config configs/example.json \
+            --deployment ${{ inputs.platform }} \
+            --language ${{ inputs.language }} \
+            --language-version ${{ inputs.version }} \
+            --architecture x64 --selected-architecture \
+            --resource-prefix sebs-ci
+
+      - name: Generate test summary
+        if: always()
+        run: |
+          echo "Regression Test Summary" > test-summary.txt
+          echo "======================" >> test-summary.txt
+          echo "Platform: ${{ inputs.platform }}" >> test-summary.txt
+          echo "Language: ${{ inputs.language }}" >> test-summary.txt
+          echo "Version: ${{ inputs.version }}" >> test-summary.txt
+          echo "" >> test-summary.txt
+          if ls regression_*.json 1> /dev/null 2>&1; then
+            ls -1 regression_*.json | wc -l | xargs echo "Benchmarks tested:" >> test-summary.txt
+            echo "" >> test-summary.txt
+            echo "Results saved to artifacts/results/" >> test-summary.txt
+          else
+            echo "No benchmark results found" >> test-summary.txt
+          fi
+
+      - name: Upload test summary
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-summary-${{ inputs.platform }}-${{ inputs.language }}-${{ inputs.version }}
+          path: test-summary.txt
+
+      - name: Collect and upload regression results
+        if: always()
+        run: |
+          mkdir -p results
+          if ls regression_*.json 1> /dev/null 2>&1; then
+            mv regression_*.json results/ || true
+          fi
+
+      - name: Upload regression results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: results-${{ inputs.platform }}-${{ inputs.language }}-${{ inputs.version }}
+          path: results/
+          if-no-files-found: ignore
+
+      - name: Upload cache snapshot
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: cache-snapshot-${{ inputs.platform }}-${{ inputs.language }}-${{ inputs.version }}
+          path: cache/
+          if-no-files-found: ignore
+
+      - name: Save SeBS cache
+        if: success()
+        uses: actions/cache/save@v4
+        with:
+          path: regression-cache/
+          key: sebs-cache-${{ github.ref_name }}-${{ inputs.platform }}-${{ inputs.language }}-${{ inputs.version }}
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
new file mode 100644
index 00000000..479d9733
--- /dev/null
+++ b/.github/workflows/lint.yml
@@ -0,0 +1,55 @@
+name: Lint
+
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  linting:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Install system dependencies
+        run: sudo apt update && sudo apt install -y libcurl4-openssl-dev
+
+      - name: Cache uv dependencies
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/uv
+          key: uv-${{ runner.os }}-${{ hashFiles('requirements.txt', 'pyproject.toml') }}
+          restore-keys: |
+            uv-${{ runner.os }}-
+
+      - name: Install SeBS with dev dependencies
+        run: uv sync --extra dev
+
+      - name: Python code formatting with black
+        run: uv run black sebs --check --config .black.toml
+
+      - name: Python code lint with flake8
+        run: uv run flake8 sebs --config=.flake8.cfg --tee --output-file flake-reports
+
+      - name: Python static code verification with mypy
+        run: uv run mypy sebs --config-file=.mypy.ini
+
+      - name: Check for Python documentation coverage
+        run: uv run interrogate -v --fail-under 100 sebs
+
+      - name: Upload flake8 reports
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: flake-reports
+          path: flake-reports
diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml
new file mode 100644
index 00000000..adbd70bc
--- /dev/null
+++ b/.github/workflows/regression.yml
@@ -0,0 +1,25 @@
+name: Regression Tests
+
+on:
+  push:
+    branches:
+      - master
+      - 'feature/**'
+  workflow_dispatch:
+
+jobs:
+  regression:
+    strategy:
+      matrix:
+        include:
+          - platform: aws
+            language: python
+            version: "3.11"
+      fail-fast: false
+
+    uses: ./.github/workflows/_regression-job.yml
+    with:
+      platform: ${{ matrix.platform }}
+      language: ${{ matrix.language }}
+      version: ${{ matrix.version }}
+    secrets: inherit

From 62a6f319c8a3a28d58cd2d063fc4b3268acdf614 Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Wed, 15 Apr 2026 18:26:12 +0200
Subject: [PATCH 11/72] [ci] Fixes

---
 .github/workflows/_regression-job.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/_regression-job.yml b/.github/workflows/_regression-job.yml
index e232d76f..105e0a69 100644
--- a/.github/workflows/_regression-job.yml
+++ b/.github/workflows/_regression-job.yml
@@ -57,7 +57,7 @@ jobs:
         uses: astral-sh/setup-uv@v4
 
       - name: Install SeBS
-        run: uv pip install .
+        run: uv pip install --system .
 
       - name: Run regression tests
         timeout-minutes: 5

From 66157927791637f6ad427a5b4cc56deede29eb1a Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Thu, 16 Apr 2026 00:48:15 +0200
Subject: [PATCH 12/72] [dev] Linting

---
 sebs/azure/cli.py        | 2 ++
 sebs/benchmark.py        | 4 ++--
 sebs/gcp/cli.py          | 2 ++
 sebs/storage/scylladb.py | 4 +++-
 4 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/sebs/azure/cli.py b/sebs/azure/cli.py
index a1eade42..7719f9fa 100644
--- a/sebs/azure/cli.py
+++ b/sebs/azure/cli.py
@@ -129,6 +129,8 @@ def execute(self, cmd: str) -> bytes:
             RuntimeError: If command execution fails.
         """
         exit_code, out = self.docker_instance.exec_run(cmd, user="docker_user")
+        # exec_run without stream=True always returns bytes
+        assert isinstance(out, bytes)
         if exit_code != 0:
             raise RuntimeError(
                 "Command {} failed at Azure CLI docker!\n Output {}".format(
diff --git a/sebs/benchmark.py b/sebs/benchmark.py
index 3a94ee07..21bb8d0d 100644
--- a/sebs/benchmark.py
+++ b/sebs/benchmark.py
@@ -1338,14 +1338,14 @@ def ensure_image(name: str) -> None:
                         with open(tar_archive, "rb") as data:
                             container.put_archive("/mnt/function", data.read())
                         # do the build step
-                        exit_code, stdout = container.exec_run(
+                        exit_code, stdout = container.exec_run(  # type: ignore[assignment]
                             cmd="/bin/bash /sebs/installer.sh",
                             user="docker_user",
                             stdout=True,
                             stderr=True,
                         )
                         # copy updated code with package
-                        data, stat = container.get_archive("/mnt/function")
+                        data, stat = container.get_archive("/mnt/function")  # type: ignore[assignment]
                         with open(tar_archive, "wb") as output_filef:
                             for chunk in data:
                                 output_filef.write(chunk)
diff --git a/sebs/gcp/cli.py b/sebs/gcp/cli.py
index 964b3efa..b2b5fb5f 100644
--- a/sebs/gcp/cli.py
+++ b/sebs/gcp/cli.py
@@ -117,6 +117,8 @@ def execute(self, cmd: str) -> bytes:
             RuntimeError: If the command fails (non-zero exit code)
         """
         exit_code, out = self.docker_instance.exec_run(cmd)
+        # exec_run without stream=True always returns bytes
+        assert isinstance(out, bytes)
         if exit_code != 0:
             raise RuntimeError(
                 "Command {} failed at gcloud CLI docker!\n Output {}".format(
diff --git a/sebs/storage/scylladb.py b/sebs/storage/scylladb.py
index c47dcef5..6c532686 100644
--- a/sebs/storage/scylladb.py
+++ b/sebs/storage/scylladb.py
@@ -188,7 +188,9 @@ def start(self) -> None:
 
             if attempts == max_attempts:
                 self.logging.error("Failed to launch ScyllaDB!")
-                self.logging.error(f"Last result of nodetool status: {out}")
+                # exec_run without stream=True always returns bytes
+                assert isinstance(out, bytes)
+                self.logging.error(f"Last result of nodetool status: {out.decode('utf-8')}")
                 raise RuntimeError("Failed to launch ScyllaDB!")
 
             self.configure_connection()

From 6340049b2973747d72efed6fe1d3c49de064f9de Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Thu, 16 Apr 2026 13:09:49 +0200
Subject: [PATCH 13/72] [dev] Update install in GH Actions

---
 .github/workflows/_regression-job.yml | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/_regression-job.yml b/.github/workflows/_regression-job.yml
index 105e0a69..e9a64f6f 100644
--- a/.github/workflows/_regression-job.yml
+++ b/.github/workflows/_regression-job.yml
@@ -56,13 +56,16 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@v4
 
-      - name: Install SeBS
-        run: uv pip install --system .
+      - name: Create virtual environment and install SeBS
+        run: |
+          uv venv
+          uv pip install .
 
       - name: Run regression tests
         timeout-minutes: 5
         run: |
-          sebs benchmark regression test \
+          source .venv/bin/activate
+          uv run sebs benchmark regression test \
             --config configs/example.json \
             --deployment ${{ inputs.platform }} \
             --language ${{ inputs.language }} \

From bd8d6e118b266463d8584cadc5dc99767a73ac2d Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Thu, 16 Apr 2026 13:16:45 +0200
Subject: [PATCH 14/72] [aws] Reenable regression of 411 for containers

---
 sebs/regression.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/sebs/regression.py b/sebs/regression.py
index da8a3e6e..0daf3aa1 100644
--- a/sebs/regression.py
+++ b/sebs/regression.py
@@ -1155,7 +1155,8 @@ def filter_out_benchmarks(
 
     # Filter out image recognition on newer Python versions on AWS
     if (deployment_name == "aws" and language == "python"
-            and language_version in ["3.9", "3.10", "3.11"]):
+            and language_version in ["3.9", "3.10", "3.11"]
+            and deployment_type == "package"):
         return "411.image-recognition" not in benchmark
 
     # C++ code package is too large for this benchmark

From 96ace65694e3a13a4b1f05bfa8e9870980b33bba Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Thu, 16 Apr 2026 13:43:37 +0200
Subject: [PATCH 15/72] [ci] Remove the old Docker copy feature needed on
 CircleCI

---
 sebs/benchmark.py                 | 140 ++++++++++--------------------
 tests/aws/create_function.py      |   6 --
 tests/aws/invoke_function_http.py |   6 --
 tests/aws/invoke_function_sdk.py  |   6 --
 4 files changed, 44 insertions(+), 114 deletions(-)

diff --git a/sebs/benchmark.py b/sebs/benchmark.py
index 21bb8d0d..57c51b7c 100644
--- a/sebs/benchmark.py
+++ b/sebs/benchmark.py
@@ -1247,18 +1247,17 @@ def ensure_image(name: str) -> None:
                 # update `image_name` in the context to the fallback image name
                 image_name = unversioned_image_name
 
-            # Create set of mounted volumes unless Docker volumes are disabled
-            if not self._experiment_config.check_flag("docker_copy_build_files"):
-                volumes = {os.path.abspath(output_dir): {"bind": "/mnt/function", "mode": "rw"}}
-                package_script = os.path.abspath(
-                    os.path.join(self._benchmark_path, self.language_name, "package.sh")
-                )
-                # does this benchmark has package.sh script?
-                if os.path.exists(package_script):
-                    volumes[package_script] = {
-                        "bind": "/mnt/function/package.sh",
-                        "mode": "ro",
-                    }
+            # Create set of mounted volumes
+            volumes = {os.path.abspath(output_dir): {"bind": "/mnt/function", "mode": "rw"}}
+            package_script = os.path.abspath(
+                os.path.join(self._benchmark_path, self.language_name, "package.sh")
+            )
+            # does this benchmark has package.sh script?
+            if os.path.exists(package_script):
+                volumes[package_script] = {
+                    "bind": "/mnt/function/package.sh",
+                    "mode": "ro",
+                }
 
             # run Docker container to install packages
             PACKAGE_FILES = {
@@ -1274,91 +1273,40 @@ def ensure_image(name: str) -> None:
                         "Docker build of benchmark dependencies in container "
                         "of image {repo}:{image}".format(repo=repo_name, image=image_name)
                     )
-                    uid = os.getuid()
-                    # Standard, simplest build
-                    if not self._experiment_config.check_flag("docker_copy_build_files"):
-                        self.logging.info(
-                            "Docker mount of benchmark code from path {path}".format(
-                                path=os.path.abspath(output_dir)
-                            )
-                        )
-                        container = self._docker_client.containers.run(
-                            "{}:{}".format(repo_name, image_name),
-                            volumes=volumes,
-                            environment={
-                                "CONTAINER_UID": str(os.getuid()),
-                                "CONTAINER_GID": str(os.getgid()),
-                                "CONTAINER_USER": "docker_user",
-                                "APP": self.benchmark,
-                                "PLATFORM": self._deployment_name.upper(),
-                                "TARGET_ARCHITECTURE": self._experiment_config._architecture,
-                            },
-                            remove=False,
-                            detach=True,
-                        )
-                        try:
-                            exit_code = container.wait()
-                            stdout = container.logs()
-                            if exit_code["StatusCode"] != 0:
-                                error_log_path = os.path.join(output_dir, "error.log")
-                                with open(error_log_path, "wb") as error_file:
-                                    error_file.write(stdout)
-                                self.logging.error(
-                                    f"Build failed! Container exited with "
-                                    f"code {exit_code['StatusCode']}"
-                                )
-                                self.logging.error(f"Logs saved to {error_log_path}")
-                                raise RuntimeError("Package build failed!")
-                        finally:
-                            container.remove()
-                    # Hack to enable builds on platforms where Docker mounted volumes
-                    # are not supported. Example: CircleCI docker environment
-                    else:
-                        container = self._docker_client.containers.run(
-                            "{}:{}".format(repo_name, image_name),
-                            environment={"APP": self.benchmark},
-                            # user="1000:1000",
-                            user=uid,
-                            remove=True,
-                            detach=True,
-                            tty=True,
-                            command="/bin/bash",
-                        )
-                        # copy application files
-                        import tarfile
-
-                        self.logging.info(
-                            "Send benchmark code from path {path} to "
-                            "Docker instance".format(path=os.path.abspath(output_dir))
-                        )
-                        tar_archive = os.path.join(output_dir, os.path.pardir, "function.tar")
-                        with tarfile.open(tar_archive, "w") as tar:
-                            for f in os.listdir(output_dir):
-                                tar.add(os.path.join(output_dir, f), arcname=f)
-                        with open(tar_archive, "rb") as data:
-                            container.put_archive("/mnt/function", data.read())
-                        # do the build step
-                        exit_code, stdout = container.exec_run(  # type: ignore[assignment]
-                            cmd="/bin/bash /sebs/installer.sh",
-                            user="docker_user",
-                            stdout=True,
-                            stderr=True,
+                    self.logging.info(
+                        "Docker mount of benchmark code from path {path}".format(
+                            path=os.path.abspath(output_dir)
                         )
-                        # copy updated code with package
-                        data, stat = container.get_archive("/mnt/function")  # type: ignore[assignment]
-                        with open(tar_archive, "wb") as output_filef:
-                            for chunk in data:
-                                output_filef.write(chunk)
-                        with tarfile.open(tar_archive, "r") as tar:
-                            tar.extractall(output_dir)
-                            # docker packs the entire directory with basename function
-                            for f in os.listdir(os.path.join(output_dir, "function")):
-                                shutil.move(
-                                    os.path.join(output_dir, "function", f),
-                                    os.path.join(output_dir, f),
-                                )
-                            shutil.rmtree(os.path.join(output_dir, "function"))
-                        container.stop()
+                    )
+                    container = self._docker_client.containers.run(
+                        "{}:{}".format(repo_name, image_name),
+                        volumes=volumes,
+                        environment={
+                            "CONTAINER_UID": str(os.getuid()),
+                            "CONTAINER_GID": str(os.getgid()),
+                            "CONTAINER_USER": "docker_user",
+                            "APP": self.benchmark,
+                            "PLATFORM": self._deployment_name.upper(),
+                            "TARGET_ARCHITECTURE": self._experiment_config._architecture,
+                        },
+                        remove=False,
+                        detach=True,
+                    )
+                    try:
+                        exit_code = container.wait()
+                        stdout = container.logs()
+                        if exit_code["StatusCode"] != 0:
+                            error_log_path = os.path.join(output_dir, "error.log")
+                            with open(error_log_path, "wb") as error_file:
+                                error_file.write(stdout)
+                            self.logging.error(
+                                f"Build failed! Container exited with "
+                                f"code {exit_code['StatusCode']}"
+                            )
+                            self.logging.error(f"Logs saved to {error_log_path}")
+                            raise RuntimeError("Package build failed!")
+                    finally:
+                        container.remove()
 
                     # Pass to output information on optimizing builds.
                     # Useful for AWS where packages have to obey size limits.
diff --git a/tests/aws/create_function.py b/tests/aws/create_function.py
index 2c5b810c..d8fc270d 100644
--- a/tests/aws/create_function.py
+++ b/tests/aws/create_function.py
@@ -17,9 +17,6 @@ class AWSCreateFunction(unittest.TestCase):
                 "update_code": False,
                 "update_storage": False,
                 "download_results": False,
-                "flags": {
-                    "docker_copy_build_files": True
-                }
             },
         },
         "nodejs": {
@@ -29,9 +26,6 @@ class AWSCreateFunction(unittest.TestCase):
                 "update_code": False,
                 "update_storage": False,
                 "download_results": False,
-                "flags": {
-                    "docker_copy_build_files": True
-                }
             }
         }
     }
diff --git a/tests/aws/invoke_function_http.py b/tests/aws/invoke_function_http.py
index 894759a2..9324c3d4 100644
--- a/tests/aws/invoke_function_http.py
+++ b/tests/aws/invoke_function_http.py
@@ -28,9 +28,6 @@ def test_invoke_sync_python(self):
                 "update_code": False,
                 "update_storage": False,
                 "download_results": False,
-                "flags": {
-                    "docker_copy_build_files": True
-                }
             },
         }
         benchmark_name = "110.dynamic-html"
@@ -56,9 +53,6 @@ def test_invoke_sync_nodejs(self):
                 "update_code": False,
                 "update_storage": False,
                 "download_results": False,
-                "flags": {
-                    "docker_copy_build_files": True
-                }
             },
         }
         benchmark_name = "110.dynamic-html"
diff --git a/tests/aws/invoke_function_sdk.py b/tests/aws/invoke_function_sdk.py
index 4353edf5..f52bd40d 100644
--- a/tests/aws/invoke_function_sdk.py
+++ b/tests/aws/invoke_function_sdk.py
@@ -29,9 +29,6 @@ def test_invoke_sync_python(self):
                 "update_code": False,
                 "update_storage": False,
                 "download_results": False,
-                "flags": {
-                    "docker_copy_build_files": True
-                }
             },
         }
         benchmark_name = "110.dynamic-html"
@@ -55,9 +52,6 @@ def test_invoke_sync_nodejs(self):
                 "update_code": False,
                 "update_storage": False,
                 "download_results": False,
-                "flags": {
-                    "docker_copy_build_files": True
-                }
             },
         }
         benchmark_name = "110.dynamic-html"

From eb4f3bc56ed49909407a62438fc649dd557afcd4 Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Thu, 16 Apr 2026 13:56:38 +0200
Subject: [PATCH 16/72] [ci] Ensure benchmarks data is cloned

---
 .github/workflows/_regression-job.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/_regression-job.yml b/.github/workflows/_regression-job.yml
index e9a64f6f..a23367e7 100644
--- a/.github/workflows/_regression-job.yml
+++ b/.github/workflows/_regression-job.yml
@@ -23,6 +23,8 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
+        with:
+          submodules: recursive
 
       - name: Restore SeBS cache
         uses: actions/cache/restore@v4

From 3ed88120056cd4678eaaa49e6da82d9be9d721ea Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Thu, 16 Apr 2026 14:17:44 +0200
Subject: [PATCH 17/72] [aws][ci] Shorten function names

---
 .github/workflows/_regression-job.yml | 2 +-
 sebs/aws/aws.py                       | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/_regression-job.yml b/.github/workflows/_regression-job.yml
index a23367e7..f52e50ec 100644
--- a/.github/workflows/_regression-job.yml
+++ b/.github/workflows/_regression-job.yml
@@ -18,7 +18,7 @@ jobs:
     runs-on: ubuntu-latest
 
     env:
-      RESOURCE_PREFIX: sebs-ci
+      RESOURCE_PREFIX: ci
 
     steps:
       - name: Checkout code
diff --git a/sebs/aws/aws.py b/sebs/aws/aws.py
index 293ed02c..e5741351 100644
--- a/sebs/aws/aws.py
+++ b/sebs/aws/aws.py
@@ -598,9 +598,11 @@ def default_function_name(
         """
         # Create function name
         resource_id = resources.resources_id if resources else self.config.resources.resources_id
+        # Extract benchmark number (e.g., "110" from "110-dynamic-html")
+        benchmark_number = code_package.benchmark.split("-")[0]
         func_name = "sebs-{}-{}-{}-{}-{}".format(
             resource_id,
-            code_package.benchmark,
+            benchmark_number,
             code_package.language_name,
             code_package.language_version,
             code_package.architecture,

From 62f437c5815425a287f5b22acc389bcfc454ef6d Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Thu, 16 Apr 2026 14:31:03 +0200
Subject: [PATCH 18/72] [aws] Fix functio nanem splitting

---
 sebs/aws/aws.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sebs/aws/aws.py b/sebs/aws/aws.py
index e5741351..9bf060b1 100644
--- a/sebs/aws/aws.py
+++ b/sebs/aws/aws.py
@@ -598,8 +598,8 @@ def default_function_name(
         """
         # Create function name
         resource_id = resources.resources_id if resources else self.config.resources.resources_id
-        # Extract benchmark number (e.g., "110" from "110-dynamic-html")
-        benchmark_number = code_package.benchmark.split("-")[0]
+        # Extract benchmark number (e.g., "110" from "110.dynamic-html")
+        benchmark_number = code_package.benchmark.split(".")[0]
         func_name = "sebs-{}-{}-{}-{}-{}".format(
             resource_id,
             benchmark_number,

From 745cdd75bf7828f668bfbe9c3f83c34c2574cf17 Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Thu, 16 Apr 2026 16:47:32 +0200
Subject: [PATCH 19/72] [docker] Ensure that failed builds are not marked as
 success

---
 dockerfiles/nodejs_installer.sh | 2 +-
 dockerfiles/python_installer.sh | 4 +---
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/dockerfiles/nodejs_installer.sh b/dockerfiles/nodejs_installer.sh
index 1dd9fd65..01a3eaf9 100644
--- a/dockerfiles/nodejs_installer.sh
+++ b/dockerfiles/nodejs_installer.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-
+set -e
 if [ -f /nvm/nvm.sh ]; then
   . /nvm/nvm.sh
 fi
diff --git a/dockerfiles/python_installer.sh b/dockerfiles/python_installer.sh
index 35f3228d..c7a3f815 100644
--- a/dockerfiles/python_installer.sh
+++ b/dockerfiles/python_installer.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-
+set -e
 cd /mnt/function
 
 PLATFORM_ARG=""
@@ -24,5 +24,3 @@ fi
 if [[ -f "${SCRIPT_FILE}" ]]; then
   /bin/bash ${SCRIPT_FILE} .python_packages/lib/site-packages
 fi
-
-

From 478428ec0d02675801415c8058429664ddd7fcaf Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Thu, 16 Apr 2026 16:47:59 +0200
Subject: [PATCH 20/72] [system] Add explicit printing of full build log

---
 sebs/benchmark.py | 11 ++++++++++-
 sebs/sebs.py      |  1 +
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/sebs/benchmark.py b/sebs/benchmark.py
index 57c51b7c..1b86e8be 100644
--- a/sebs/benchmark.py
+++ b/sebs/benchmark.py
@@ -578,6 +578,7 @@ def __init__(
         output_dir: str,
         cache_client: Cache,
         docker_client: docker.client.DockerClient,
+        verbose: bool = False,
     ):
         """
         Initialize a Benchmark instance.
@@ -594,6 +595,7 @@ def __init__(
             output_dir: Directory for output files
             cache_client: Cache client for caching code packages
             docker_client: Docker client for building dependencies
+            verbose: Print verbose build logs.
 
         Raises:
             RuntimeError: If the benchmark is not found or doesn't support the language
@@ -608,6 +610,7 @@ def __init__(
         self._language_variant = config.runtime.variant.value
         self._architecture = self._experiment_config.architecture
         self._container_deployment = config.container_deployment
+        self._verbose = verbose
 
         benchmark_path = find_benchmark(self.benchmark, "benchmarks")
         if not benchmark_path:
@@ -1295,16 +1298,22 @@ def ensure_image(name: str) -> None:
                     try:
                         exit_code = container.wait()
                         stdout = container.logs()
-                        if exit_code["StatusCode"] != 0:
+
+                        error_log_path: str = ""
+                        if exit_code["StatusCode"] != 0 or self._verbose:
                             error_log_path = os.path.join(output_dir, "error.log")
                             with open(error_log_path, "wb") as error_file:
                                 error_file.write(stdout)
+
+                        if exit_code["StatusCode"] != 0:
                             self.logging.error(
                                 f"Build failed! Container exited with "
                                 f"code {exit_code['StatusCode']}"
                             )
                             self.logging.error(f"Logs saved to {error_log_path}")
                             raise RuntimeError("Package build failed!")
+
+                        self.logging.debug(f"Build Build logs saved to {error_log_path}")
                     finally:
                         container.remove()
 
diff --git a/sebs/sebs.py b/sebs/sebs.py
index a3dd89a9..5d40b103 100644
--- a/sebs/sebs.py
+++ b/sebs/sebs.py
@@ -368,6 +368,7 @@ def get_benchmark(
             self._output_dir,
             self.cache_client,
             self.docker_client,
+            self.verbose,
         )
 
         # Set up logging

From 6ac11841a051241c16cbcd7ce0f9dd08d504d399 Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Thu, 16 Apr 2026 16:58:36 +0200
Subject: [PATCH 21/72] [docker] Print full output of pip install

---
 dockerfiles/python_installer.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/dockerfiles/python_installer.sh b/dockerfiles/python_installer.sh
index c7a3f815..5525cf87 100644
--- a/dockerfiles/python_installer.sh
+++ b/dockerfiles/python_installer.sh
@@ -9,15 +9,15 @@ fi
 
 if [[ "${TARGET_ARCHITECTURE}" == "arm64" ]] && [[ -f "requirements.txt.arm.${PYTHON_VERSION}" ]]; then
 
-  pip3 -q install ${PLATFORM_ARG} -r requirements.txt.arm.${PYTHON_VERSION} -t .python_packages/lib/site-packages
+  pip3 install ${PLATFORM_ARG} -r requirements.txt.arm.${PYTHON_VERSION} -t .python_packages/lib/site-packages
 
 elif [[ -f "requirements.txt.${PYTHON_VERSION}" ]]; then
 
-  pip3 -q install ${PLATFORM_ARG} -r requirements.txt.${PYTHON_VERSION} -t .python_packages/lib/site-packages
+  pip3 install ${PLATFORM_ARG} -r requirements.txt.${PYTHON_VERSION} -t .python_packages/lib/site-packages
 
 else
 
-  pip3 -q install ${PLATFORM_ARG} -r requirements.txt -t .python_packages/lib/site-packages
+  pip3 install ${PLATFORM_ARG} -r requirements.txt -t .python_packages/lib/site-packages
 
 fi
 

From 2983b2133d559ac5445776f7cba547661af0be16 Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Thu, 16 Apr 2026 17:06:56 +0200
Subject: [PATCH 22/72] [docker] Support fallback to previous version of Docker
 image

---
 configs/systems.json |  3 ++-
 sebs/benchmark.py    | 54 +++++++++++++++++++++++++++++---------------
 sebs/config.py       | 12 ++++++++++
 3 files changed, 50 insertions(+), 19 deletions(-)

diff --git a/configs/systems.json b/configs/systems.json
index 27ad4959..671b99b2 100644
--- a/configs/systems.json
+++ b/configs/systems.json
@@ -1,7 +1,8 @@
 {
   "general": {
     "docker_repository": "spcleth/serverless-benchmarks",
-    "SeBS_version": "1.2.0"
+    "SeBS_version": "1.2.1",
+    "previous_major_version": "1.2.0"
   },
   "local": {
     "experiments": {
diff --git a/sebs/benchmark.py b/sebs/benchmark.py
index 1b86e8be..148fed04 100644
--- a/sebs/benchmark.py
+++ b/sebs/benchmark.py
@@ -1161,22 +1161,33 @@ def directory_size(directory: str) -> int:
     def builder_image_name(self) -> Tuple[str, str]:
         """Image names of builder Docker images for preparing benchmarks.
 
-        We are progressively replacing all unversioned image names with versioned ones.
+        Returns two image names for fallback behavior:
+        - Current version image (tagged with current SeBS version)
+        - Previous version image (tagged with previous major SeBS version)
+
+        This allows new SeBS versions to use images from the previous stable
+        version without requiring a complete rebuild of all images.
 
         Returns:
-            Tuple of unversioned and versioned image names.
+            Tuple of (previous_version_image_name, current_version_image_name).
         """
-        unversioned_image_name = "build.{deployment}.{language}.{runtime}".format(
+        base_image_name = "build.{deployment}.{language}.{runtime}".format(
             deployment=self._deployment_name,
             language=self.language_name,
             runtime=self.language_version,
         )
-        image_name = "{base_image_name}-{sebs_version}".format(
-            base_image_name=unversioned_image_name,
-            sebs_version=self._system_config.version(),
+        # Current version image (try this first)
+        current_version_image_name = "{base}-{version}".format(
+            base=base_image_name,
+            version=self._system_config.version(),
+        )
+        # Previous major version image (fallback)
+        previous_version_image_name = "{base}-{version}".format(
+            base=base_image_name,
+            version=self._system_config.previous_version(),
         )
 
-        return unversioned_image_name, image_name
+        return previous_version_image_name, current_version_image_name
 
     def install_dependencies(self, output_dir: str) -> None:
         """Install benchmark dependencies using Docker.
@@ -1187,10 +1198,11 @@ def install_dependencies(self, output_dir: str) -> None:
         Pulls a pre-built Docker image specific to the deployment, language, and
         runtime version. Mounts the output directory into the container and runs
         an installer script (`/sebs/installer.sh`) within the container.
-        Handles fallbacks to unversioned Docker images if versioned ones are not found.
 
-        Supports copying files to/from Docker for environments where volume mounting
-        is problematic (e.g., CircleCI).
+        Tries current SeBS version image first, falls back to previous major version
+        image if the current version image is not available. This allows new SeBS
+        versions to use images from the previous stable version without requiring
+        a complete rebuild.
 
         Args:
             output_dir: Directory containing the code package to build
@@ -1211,7 +1223,7 @@ def install_dependencies(self, output_dir: str) -> None:
             )
         else:
             repo_name = self._system_config.docker_repository()
-            unversioned_image_name, image_name = self.builder_image_name()
+            previous_version_image_name, current_version_image_name = self.builder_image_name()
 
             def ensure_image(name: str) -> None:
                 """Internal implementation of checking for Docker image existence.
@@ -1220,7 +1232,7 @@ def ensure_image(name: str) -> None:
                     name: image name
 
                 Raises:
-                    RuntimeError: when neither versioned nor unversioned images exists.
+                    RuntimeError: when image does not exist locally or cannot be pulled.
                 """
                 try:
                     self._docker_client.images.get(repo_name + ":" + name)
@@ -1235,20 +1247,22 @@ def ensure_image(name: str) -> None:
                             "Docker pull of image {}:{} failed!".format(repo_name, name)
                         )
 
+            # Try current version image first, fallback to previous version if not available
+            image_name = current_version_image_name
             try:
-                ensure_image(image_name)
+                ensure_image(current_version_image_name)
             except RuntimeError as e:
                 self.logging.warning(
                     "Failed to ensure image {}, falling back to {}: {}".format(
-                        image_name, unversioned_image_name, e
+                        current_version_image_name, previous_version_image_name, e
                     )
                 )
                 try:
-                    ensure_image(unversioned_image_name)
+                    ensure_image(previous_version_image_name)
+                    # update `image_name` to the fallback image name
+                    image_name = previous_version_image_name
                 except RuntimeError:
                     raise
-                # update `image_name` in the context to the fallback image name
-                image_name = unversioned_image_name
 
             # Create set of mounted volumes
             volumes = {os.path.abspath(output_dir): {"bind": "/mnt/function", "mode": "rw"}}
@@ -1300,8 +1314,12 @@ def ensure_image(name: str) -> None:
                         stdout = container.logs()
 
                         error_log_path: str = ""
-                        if exit_code["StatusCode"] != 0 or self._verbose:
+                        if exit_code["StatusCode"] != 0:
                             error_log_path = os.path.join(output_dir, "error.log")
+                        elif self._verbose:
+                            error_log_path = os.path.join(output_dir, "build.log")
+
+                        if exit_code["StatusCode"] != 0 or self._verbose:
                             with open(error_log_path, "wb") as error_file:
                                 error_file.write(stdout)
 
diff --git a/sebs/config.py b/sebs/config.py
index c5d3fe9b..6f66e888 100644
--- a/sebs/config.py
+++ b/sebs/config.py
@@ -213,6 +213,18 @@ def version(self) -> str:
         """
         return self._system_config["general"].get("SeBS_version", "unknown")
 
+    def previous_version(self) -> str:
+        """Get the previous major SeBS framework version.
+
+        This is used as a fallback version for Docker images that haven't been
+        rebuilt for the current version. It allows new SeBS versions to use
+        images from the previous stable version without requiring a full rebuild.
+
+        Returns:
+            str: The previous major version string, or 'unknown' if not configured.
+        """
+        return self._system_config["general"].get("previous_major_version", "unknown")
+
     def docker_image_name(
         self,
         system: str,

From a569837ccaf4df53f836d3ec3c20a1e04d4c3228 Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Thu, 16 Apr 2026 17:18:03 +0200
Subject: [PATCH 23/72] [benchmarks] Fix 504 compatibility issues on Python
 3.11 and older glibc

---
 .../504.dna-visualisation/python/requirements.txt.3.11      | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 benchmarks/500.scientific/504.dna-visualisation/python/requirements.txt.3.11

diff --git a/benchmarks/500.scientific/504.dna-visualisation/python/requirements.txt.3.11 b/benchmarks/500.scientific/504.dna-visualisation/python/requirements.txt.3.11
new file mode 100644
index 00000000..15a151be
--- /dev/null
+++ b/benchmarks/500.scientific/504.dna-visualisation/python/requirements.txt.3.11
@@ -0,0 +1,6 @@
+# Copyright 2020-2025 ETH Zurich and the SeBS authors. All rights reserved.
+squiggle==0.3.1
+# Lambda images use glibc 2.26, for which there are no wheels on new packages
+# we have to fix version to prevent compilation from source
+numpy==2.2.6
+contourpy==1.3.2

From 2fe6edf160fbac200a2e6830c60aae7ab85912fa Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Thu, 16 Apr 2026 18:12:22 +0200
Subject: [PATCH 24/72] [aws] Debugging for regression runs

---
 benchmarks/300.utilities/311.compression/input.py | 1 +
 benchmarks/wrappers/aws/python/storage.py         | 5 +++++
 2 files changed, 6 insertions(+)

diff --git a/benchmarks/300.utilities/311.compression/input.py b/benchmarks/300.utilities/311.compression/input.py
index 10cf8986..fc8bdd5f 100644
--- a/benchmarks/300.utilities/311.compression/input.py
+++ b/benchmarks/300.utilities/311.compression/input.py
@@ -36,4 +36,5 @@ def generate_input(data_dir, size, benchmarks_bucket, input_paths, output_paths,
     input_config['bucket']['bucket'] = benchmarks_bucket
     input_config['bucket']['input'] = input_paths[0]
     input_config['bucket']['output'] = output_paths[0]
+    print(input_config)
     return input_config
diff --git a/benchmarks/wrappers/aws/python/storage.py b/benchmarks/wrappers/aws/python/storage.py
index 13fd471d..95e63d77 100644
--- a/benchmarks/wrappers/aws/python/storage.py
+++ b/benchmarks/wrappers/aws/python/storage.py
@@ -31,7 +31,12 @@ def download(self, bucket, file, filepath):
         self.client.download_file(bucket, file, filepath)
 
     def download_directory(self, bucket, prefix, path):
+        print(f"DEBUG: bucket={bucket}, prefix={prefix}")
         objects = self.client.list_objects_v2(Bucket=bucket, Prefix=prefix)
+        print(f"DEBUG: got {objects.get('KeyCount', 0)} keys, response keys: {list(objects.keys())}")
+        # 'Contents' key is only present when objects are found
+        if 'Contents' not in objects:
+            raise RuntimeError(f"No objects found in bucket '{bucket}' with prefix '{prefix}'")
         for obj in objects['Contents']:
             file_name = obj['Key']
             path_to_file = os.path.dirname(file_name)

From 11c96a3dce50adc4595a2f8431e690daee3ad07a Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Thu, 16 Apr 2026 18:28:29 +0200
Subject: [PATCH 25/72] [benchmarks] Ensure that we always put correct path to
 311 benchmark

---
 .../300.utilities/311.compression/input.py    | 30 +++++++++++++------
 benchmarks/wrappers/aws/python/storage.py     |  2 --
 2 files changed, 21 insertions(+), 11 deletions(-)

diff --git a/benchmarks/300.utilities/311.compression/input.py b/benchmarks/300.utilities/311.compression/input.py
index fc8bdd5f..9eb4ffb6 100644
--- a/benchmarks/300.utilities/311.compression/input.py
+++ b/benchmarks/300.utilities/311.compression/input.py
@@ -1,6 +1,7 @@
 # Copyright 2020-2025 ETH Zurich and the SeBS authors. All rights reserved.
 import glob, os
 
+
 def buckets_count():
     return (1, 1)
 
@@ -10,11 +11,12 @@ def upload_files(data_root, data_dir, upload_func):
     for root, dirs, files in os.walk(data_dir):
         prefix = os.path.relpath(root, data_root)
         for file in files:
-            file_name = prefix + '/' + file
+            file_name = prefix + "/" + file
             filepath = os.path.join(root, file)
             upload_func(0, file_name, filepath)
 
-'''
+
+"""
     Generate test, small and large workload for compression test.
 
     :param data_dir: directory where benchmark data is placed
@@ -22,8 +24,18 @@ def upload_files(data_root, data_dir, upload_func):
     :param input_buckets: input storage containers for this benchmark
     :param output_buckets:
     :param upload_func: upload function taking three params(bucket_idx, key, filepath)
-'''
-def generate_input(data_dir, size, benchmarks_bucket, input_paths, output_paths, upload_func, nosql_func):
+"""
+
+
+def generate_input(
+    data_dir,
+    size,
+    benchmarks_bucket,
+    input_paths,
+    output_paths,
+    upload_func,
+    nosql_func,
+):
 
     # upload different datasets
     datasets = []
@@ -31,10 +43,10 @@ def generate_input(data_dir, size, benchmarks_bucket, input_paths, output_paths,
         datasets.append(dir)
         upload_files(data_dir, os.path.join(data_dir, dir), upload_func)
 
-    input_config = {'object': {}, 'bucket': {}}
-    input_config['object']['key'] = datasets[0]
-    input_config['bucket']['bucket'] = benchmarks_bucket
-    input_config['bucket']['input'] = input_paths[0]
-    input_config['bucket']['output'] = output_paths[0]
+    input_config = {"object": {}, "bucket": {}}
+    input_config["object"]["key"] = "acmart-master"
+    input_config["bucket"]["bucket"] = benchmarks_bucket
+    input_config["bucket"]["input"] = input_paths[0]
+    input_config["bucket"]["output"] = output_paths[0]
     print(input_config)
     return input_config
diff --git a/benchmarks/wrappers/aws/python/storage.py b/benchmarks/wrappers/aws/python/storage.py
index 95e63d77..401947df 100644
--- a/benchmarks/wrappers/aws/python/storage.py
+++ b/benchmarks/wrappers/aws/python/storage.py
@@ -31,9 +31,7 @@ def download(self, bucket, file, filepath):
         self.client.download_file(bucket, file, filepath)
 
     def download_directory(self, bucket, prefix, path):
-        print(f"DEBUG: bucket={bucket}, prefix={prefix}")
         objects = self.client.list_objects_v2(Bucket=bucket, Prefix=prefix)
-        print(f"DEBUG: got {objects.get('KeyCount', 0)} keys, response keys: {list(objects.keys())}")
         # 'Contents' key is only present when objects are found
         if 'Contents' not in objects:
             raise RuntimeError(f"No objects found in bucket '{bucket}' with prefix '{prefix}'")

From 3fa215d69456af0ea4a471f6561432a3aec49cb4 Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Thu, 16 Apr 2026 18:32:59 +0200
Subject: [PATCH 26/72] [system] Bump current version

---
 sebs/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sebs/version.py b/sebs/version.py
index dc7990a0..b0eee8ab 100644
--- a/sebs/version.py
+++ b/sebs/version.py
@@ -1,3 +1,3 @@
 # Copyright 2020-2025 ETH Zurich and the SeBS authors. All rights reserved.
 """Main SeBS version information."""
-__version__ = "1.2.0"
+__version__ = "1.2.1"

From 483b016f5510bfadf157a5fb7690cca0794150c9 Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Thu, 16 Apr 2026 18:33:33 +0200
Subject: [PATCH 27/72] [ci] Simplify resource naming

---
 .github/workflows/_regression-job.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/_regression-job.yml b/.github/workflows/_regression-job.yml
index f52e50ec..90aaca82 100644
--- a/.github/workflows/_regression-job.yml
+++ b/.github/workflows/_regression-job.yml
@@ -73,7 +73,7 @@ jobs:
             --language ${{ inputs.language }} \
             --language-version ${{ inputs.version }} \
             --architecture x64 --selected-architecture \
-            --resource-prefix sebs-ci
+            --resource-prefix ci
 
       - name: Generate test summary
         if: always()

From fdd76397ffb20443f7f8e5d884c8252f09552d16 Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Thu, 16 Apr 2026 18:34:19 +0200
Subject: [PATCH 28/72] [ci] Add Nodejs builds

---
 .github/workflows/regression.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml
index adbd70bc..60ae50ee 100644
--- a/.github/workflows/regression.yml
+++ b/.github/workflows/regression.yml
@@ -15,6 +15,9 @@ jobs:
           - platform: aws
             language: python
             version: "3.11"
+          - platform: aws
+            language: nodejs
+            version: "16"
       fail-fast: false
 
     uses: ./.github/workflows/_regression-job.yml

From 75d5feebbd42f9864bbd82a74e8f2740ac750821 Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Thu, 16 Apr 2026 18:38:12 +0200
Subject: [PATCH 29/72] [ci] Update badges

---
 README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 6c38676a..b51db91a 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,6 @@
 
-[![CircleCI](https://circleci.com/gh/spcl/serverless-benchmarks.svg?style=shield)](https://circleci.com/gh/spcl/serverless-benchmarks)
+[![Code Linting](https://github.com/spcl/serverless-benchmarks/actions/workflows/lint.yml/badge.svg)](https://github.com/spcl/serverless-benchmarks/actions)
+[![Regression](https://github.com/spcl/serverless-benchmarks/actions/workflows/regression.yml/badge.svg)](https://github.com/spcl/serverless-benchmarks/actions)
 [![Documentation Status](https://readthedocs.org/projects/sebs/badge/?version=latest)](https://sebs.readthedocs.io/en/latest/?badge=latest)
 ![Release](https://img.shields.io/github/v/release/spcl/serverless-benchmarks)
 ![License](https://img.shields.io/github/license/spcl/serverless-benchmarks)

From f0948f0d27ca95ede24df9729cd4c42d77de7539 Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Thu, 16 Apr 2026 18:50:27 +0200
Subject: [PATCH 30/72] [ci] Cleanup functions once we are done

---
 .github/workflows/_regression-job.yml | 17 ++++++++++----
 sebs/cli.py                           | 32 ++++++++++++++++++++++-----
 2 files changed, 39 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/_regression-job.yml b/.github/workflows/_regression-job.yml
index 90aaca82..26aa02fb 100644
--- a/.github/workflows/_regression-job.yml
+++ b/.github/workflows/_regression-job.yml
@@ -43,10 +43,9 @@ jobs:
       - name: Setup Azure credentials
         if: inputs.platform == 'azure'
         run: |
-          echo "AZURE_SUBSCRIPTION_ID=${{ secrets.AZURE_SUBSCRIPTION_ID }}" >> $GITHUB_ENV
-          echo "AZURE_TENANT_ID=${{ secrets.AZURE_TENANT_ID }}" >> $GITHUB_ENV
-          echo "AZURE_CLIENT_ID=${{ secrets.AZURE_CLIENT_ID }}" >> $GITHUB_ENV
-          echo "AZURE_CLIENT_SECRET=${{ secrets.AZURE_CLIENT_SECRET }}" >> $GITHUB_ENV
+          echo "AZURE_SECRET_APPLICATION_ID==${{ secrets.AZURE_SECRET_APPLICATION_ID }}" >> $GITHUB_ENV
+          echo "AZURE_SECRET_TENANT=${{ secrets.AZURE_SECRET_TENANT }}" >> $GITHUB_ENV
+          echo "AZURE_SECRET_PASSWORD=${{ secrets.AZURE_SECRET_PASSWORD }}" >> $GITHUB_ENV
 
       - name: Setup AWS credentials
         if: inputs.platform == 'aws'
@@ -75,6 +74,16 @@ jobs:
             --architecture x64 --selected-architecture \
             --resource-prefix ci
 
+      - name: Cleanup deployed functions
+        if: always()
+        run: |
+          source .venv/bin/activate
+          uv run sebs resources cleanup \
+            --config configs/example.json \
+            --deployment ${{ inputs.platform }} \
+            --resource-prefix ci \
+            --resource-type functions
+
       - name: Generate test summary
         if: always()
         run: |
diff --git a/sebs/cli.py b/sebs/cli.py
index 870d8603..7c4d4e24 100755
--- a/sebs/cli.py
+++ b/sebs/cli.py
@@ -897,9 +897,19 @@ def resources_remove(resource, prefix, wait, dry_run, **kwargs):
     default=False,
     help="Simulate run without actual deletions.",
 )
+@click.option(
+    "--resource-type",
+    type=click.Choice(["functions"]),
+    default=None,
+    help="Clean up only specific resource type. If not specified, cleans up all resources.",
+)
 @common_params
-def resources_cleanup(resources_id, dry_run, **kwargs):
-    """Clean up cloud resources created by SeBS experiments."""
+def resources_cleanup(resources_id, dry_run, resource_type, **kwargs):
+    """Clean up cloud resources created by SeBS experiments.
+
+    By default, cleans up all resources (functions, storage, etc.).
+    Use --resource-type to clean up only specific resource types.
+    """
     (
         config,
         output_dir,
@@ -909,10 +919,20 @@ def resources_cleanup(resources_id, dry_run, **kwargs):
     ) = parse_common_params(**kwargs)
 
     try:
-        result = deployment_client.cleanup_resources(dry_run=dry_run)
-        total = sum(len(v) for v in result.values())
-        action = "found" if dry_run else "deleted"
-        sebs_client.logging.info(f"Total resources {action}: {total}")
+        if resource_type == "functions":
+            # Clean up only functions
+            deleted_functions = deployment_client.cleanup_functions(dry_run=dry_run)
+            action = "found" if dry_run else "deleted"
+            sebs_client.logging.info(f"Total functions {action}: {len(deleted_functions)}")
+            if deleted_functions:
+                for func_name in deleted_functions:
+                    sebs_client.logging.info(f"  - {func_name}")
+        else:
+            # Clean up all resources (original behavior)
+            result = deployment_client.cleanup_resources(dry_run=dry_run)
+            total = sum(len(v) for v in result.values())
+            action = "found" if dry_run else "deleted"
+            sebs_client.logging.info(f"Total resources {action}: {total}")
     except NotImplementedError as e:
         sebs_client.logging.error(str(e))
 

From dbbf6772c97950c4943fc961dae26b55234105fa Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Thu, 16 Apr 2026 18:50:33 +0200
Subject: [PATCH 31/72] [ci] Enable GCP and Azure

---
 .github/workflows/regression.yml | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml
index 60ae50ee..f5c59239 100644
--- a/.github/workflows/regression.yml
+++ b/.github/workflows/regression.yml
@@ -18,6 +18,18 @@ jobs:
           - platform: aws
             language: nodejs
             version: "16"
+          - platform: gcp
+            language: python
+            version: "3.11"
+          - platform: gcp
+            language: nodejs
+            version: "18"
+          - platform: azure
+            language: python
+            version: "3.11"
+          - platform: azure
+            language: nodejs
+            version: "20"
       fail-fast: false
 
     uses: ./.github/workflows/_regression-job.yml

From 9d8b39a1d858b2006003e26fc27d7a8e9d086c52 Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Thu, 16 Apr 2026 18:54:51 +0200
Subject: [PATCH 32/72] [system] Update default architecture

---
 configs/example.json | 2 +-
 configs/nodejs.json  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/configs/example.json b/configs/example.json
index f320982b..e4f79b21 100644
--- a/configs/example.json
+++ b/configs/example.json
@@ -4,7 +4,7 @@
     "update_code": false,
     "update_storage": false,
     "download_results": false,
-    "architecture": "arm64", 
+    "architecture": "x64", 
     "container_deployment": true,
     "runtime": {
       "language": "python",
diff --git a/configs/nodejs.json b/configs/nodejs.json
index 4188b537..9abc0ce1 100644
--- a/configs/nodejs.json
+++ b/configs/nodejs.json
@@ -4,7 +4,7 @@
     "update_code": false,
     "update_storage": false,
     "download_results": false,
-    "architecture": "arm64",
+    "architecture": "x64",
     "container_deployment": true,
     "runtime": {
       "language": "nodejs",

From 933923625021f35e8ff8c5899e7af68b3a53c64d Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Thu, 16 Apr 2026 18:58:52 +0200
Subject: [PATCH 33/72] [ci] Add arm64 runs

---
 .github/workflows/_regression-job.yml |  6 +++++-
 .github/workflows/regression.yml      | 14 ++++++++++++++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/_regression-job.yml b/.github/workflows/_regression-job.yml
index 26aa02fb..0e7133d9 100644
--- a/.github/workflows/_regression-job.yml
+++ b/.github/workflows/_regression-job.yml
@@ -12,6 +12,9 @@ on:
       version:
         required: true
         type: string
+      architecture:
+        required: true
+        type: string
 
 jobs:
   test:
@@ -71,7 +74,8 @@ jobs:
             --deployment ${{ inputs.platform }} \
             --language ${{ inputs.language }} \
             --language-version ${{ inputs.version }} \
-            --architecture x64 --selected-architecture \
+            --architecture ${{ inputs.architecture }} \
+            --selected-architecture \
             --resource-prefix ci
 
       - name: Cleanup deployed functions
diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml
index f5c59239..5c16464a 100644
--- a/.github/workflows/regression.yml
+++ b/.github/workflows/regression.yml
@@ -15,21 +15,35 @@ jobs:
           - platform: aws
             language: python
             version: "3.11"
+            architecture: "x64"
+          - platform: aws
+            language: python
+            version: "3.11"
+            architecture: "arm64"
+          - platform: aws
+            language: nodejs
+            version: "16"
+            architecture: "x64"
           - platform: aws
             language: nodejs
             version: "16"
+            architecture: "arm64"
           - platform: gcp
             language: python
             version: "3.11"
+            architecture: "x64"
           - platform: gcp
             language: nodejs
             version: "18"
+            architecture: "x64"
           - platform: azure
             language: python
             version: "3.11"
+            architecture: "x64"
           - platform: azure
             language: nodejs
             version: "20"
+            architecture: "x64"
       fail-fast: false
 
     uses: ./.github/workflows/_regression-job.yml

From 4b1a07e581231992f4b08a2ff39ab3889e5af3af Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Thu, 16 Apr 2026 19:03:00 +0200
Subject: [PATCH 34/72] [ci] Try to fix GCP credentials

---
 .github/workflows/_regression-job.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/_regression-job.yml b/.github/workflows/_regression-job.yml
index 0e7133d9..e2b7fb8d 100644
--- a/.github/workflows/_regression-job.yml
+++ b/.github/workflows/_regression-job.yml
@@ -39,9 +39,9 @@ jobs:
 
       - name: Setup GCP credentials
         if: inputs.platform == 'gcp'
-        run: |
-          echo "${{ secrets.GCP_SERVICE_ACCOUNT_JSON }}" > /tmp/gcp-credentials.json
-          echo "GOOGLE_APPLICATION_CREDENTIALS=/tmp/gcp-credentials.json" >> $GITHUB_ENV
+        uses: google-github-actions/auth@v2
+        with:
+          credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_JSON }}
 
       - name: Setup Azure credentials
         if: inputs.platform == 'azure'

From ffee6d4e832887a1b0d0bed1ad3b6706b93a04dc Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Thu, 16 Apr 2026 19:03:45 +0200
Subject: [PATCH 35/72] [ci] Fix

---
 .github/workflows/regression.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml
index 5c16464a..6b680b07 100644
--- a/.github/workflows/regression.yml
+++ b/.github/workflows/regression.yml
@@ -51,4 +51,5 @@ jobs:
       platform: ${{ matrix.platform }}
       language: ${{ matrix.language }}
       version: ${{ matrix.version }}
+      architecture: ${{ matrix.architecture }}
     secrets: inherit

From ef681b0c79aa686187e10b41155c05e177f0ef89 Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Thu, 16 Apr 2026 19:06:48 +0200
Subject: [PATCH 36/72] [ci] Bump GCP node version

---
 .github/workflows/regression.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml
index 6b680b07..9133a856 100644
--- a/.github/workflows/regression.yml
+++ b/.github/workflows/regression.yml
@@ -34,7 +34,7 @@ jobs:
             architecture: "x64"
           - platform: gcp
             language: nodejs
-            version: "18"
+            version: "20"
             architecture: "x64"
           - platform: azure
             language: python

From b9ff8723260a7eff681129c874f6ba0a66819b28 Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Thu, 16 Apr 2026 19:16:31 +0200
Subject: [PATCH 37/72] [ci] Use ARM runners

---
 .github/workflows/_regression-job.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/_regression-job.yml b/.github/workflows/_regression-job.yml
index e2b7fb8d..c1a1cfa6 100644
--- a/.github/workflows/_regression-job.yml
+++ b/.github/workflows/_regression-job.yml
@@ -18,7 +18,7 @@ on:
 
 jobs:
   test:
-    runs-on: ubuntu-latest
+    runs-on: ${{ inputs.architecture == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }}
 
     env:
       RESOURCE_PREFIX: ci

From 121404457283719ccb51091797d77e5437f9ad0e Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Thu, 16 Apr 2026 19:27:44 +0200
Subject: [PATCH 38/72] [ci] Fix typo in Azure creds

---
 .github/workflows/_regression-job.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/_regression-job.yml b/.github/workflows/_regression-job.yml
index c1a1cfa6..e7009916 100644
--- a/.github/workflows/_regression-job.yml
+++ b/.github/workflows/_regression-job.yml
@@ -46,7 +46,7 @@ jobs:
       - name: Setup Azure credentials
         if: inputs.platform == 'azure'
         run: |
-          echo "AZURE_SECRET_APPLICATION_ID==${{ secrets.AZURE_SECRET_APPLICATION_ID }}" >> $GITHUB_ENV
+          echo "AZURE_SECRET_APPLICATION_ID=${{ secrets.AZURE_SECRET_APPLICATION_ID }}" >> $GITHUB_ENV
           echo "AZURE_SECRET_TENANT=${{ secrets.AZURE_SECRET_TENANT }}" >> $GITHUB_ENV
           echo "AZURE_SECRET_PASSWORD=${{ secrets.AZURE_SECRET_PASSWORD }}" >> $GITHUB_ENV
 

From 307e6aaff1b6d3763a9a5fa3dc5b15b1cf4324ce Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Thu, 16 Apr 2026 20:53:27 +0200
Subject: [PATCH 39/72] [system] Support building multi-platform images

---
 docs/platforms.md      |  12 ++++
 sebs/cli.py            |   8 +++
 sebs/docker_builder.py | 152 ++++++++++++++++++++++++++++++++++++++++-
 3 files changed, 170 insertions(+), 2 deletions(-)

diff --git a/docs/platforms.md b/docs/platforms.md
index 6c993592..21d05ef7 100644
--- a/docs/platforms.md
+++ b/docs/platforms.md
@@ -33,6 +33,18 @@ However, special care is needed to build Docker containers: since installation o
 binaries based on ARM containers on x86 CPUs. To build multi-platform images, we recommend to follow official [Docker guidelines](https://docs.docker.com/build/building/multi-platform/#build-multi-platform-images) and provide static QEMU installation.
 On Ubuntu-based distributions, this requires installing an OS package and executing a single Docker command to provide seamless emulation of ARM containers.
 
+### Multi-platform Docker Images
+
+Build images, which encapsulate package building, are available as both x64 and arm64 for Python and Node.js on AWS Lambda.
+To rebuild multi-plaform images, an additional flag is needed to enable the internal `docker buildx` command:
+
+```bash
+sebs docker build --image-type build --language python --deployment aws --architecture x64 --language-version 3.11 --multi-platform
+```
+
+When rebuilding build images (not necessary for regular users, only for developers), make sure that your Docker installation supports multi-platform images,
+e.g., you use `containerd` image store.
+
 ## Cloud Account Identifiers
 
 SeBS ensures that all locally cached cloud resources are valid by storing a unique identifier associated with each cloud account. Furthermore, we store this identifier in experiment results to easily match results with the cloud account or subscription that was used to obtain them. We use non-sensitive identifiers such as account IDs on AWS, subscription IDs on Azure, and Google Cloud project IDs.
diff --git a/sebs/cli.py b/sebs/cli.py
index 7c4d4e24..2fa729b7 100755
--- a/sebs/cli.py
+++ b/sebs/cli.py
@@ -975,6 +975,12 @@ def docker_cmd():
     type=str,
     help="Optional Docker platform (e.g., linux/amd64) to override host architecture.",
 )
+@click.option(
+    "--multi-platform/--no-multi-platform",
+    default=False,
+    type=bool,
+    help="When true, build multi-platform images (requires QEMU support)",
+)
 @click.option(
     "--dependency-type",
     default=None,
@@ -990,6 +996,7 @@ def docker_build(
     language_version,
     architecture,
     platform,
+    multi_platform,
     dependency_type,
     parallel,
     verbose,
@@ -1013,6 +1020,7 @@ def docker_build(
         architecture=architecture,
         dependency_type=dependency_type,
         platform=platform,
+        multi_platform=multi_platform,
         parallel=parallel,
     )
 
diff --git a/sebs/docker_builder.py b/sebs/docker_builder.py
index 420ad446..e03cdc6c 100644
--- a/sebs/docker_builder.py
+++ b/sebs/docker_builder.py
@@ -9,6 +9,7 @@
 import json
 import logging
 import os
+import subprocess
 from pathlib import Path
 from typing import Dict, List, Optional, Tuple
 
@@ -77,6 +78,95 @@ def __init__(
         else:
             self.docker_client = docker_client
 
+    def _should_use_multiplatform(
+        self, system: str, image_type: str, language: Optional[str] = None
+    ) -> bool:
+        """Check if multi-platform build should be used for this image.
+
+        Multi-platform builds (x64 + arm64) are only enabled for:
+        - AWS platform
+        - Build images only
+        - Python or Node.js languages
+
+        Args:
+            system: Deployment platform (e.g., 'aws', 'gcp')
+            image_type: Type of image (e.g., 'build', 'run')
+            language: Programming language (e.g., 'python', 'nodejs')
+
+        Returns:
+            True if multi-platform build should be used, False otherwise
+        """
+        return system == "aws" and image_type == "build" and language in ["python", "nodejs"]
+
+    def _execute_multiplatform_build(
+        self,
+        image_name: str,
+        dockerfile: str,
+        buildargs: dict,
+    ) -> bool:
+        """Execute multi-platform build using Docker buildx.
+
+        Builds an image for both linux/amd64 and linux/arm64 platforms
+        and pushes it as a multi-platform manifest to the registry.
+
+        Args:
+            image_name: Docker image tag
+            dockerfile: path to Dockerfile
+            buildargs: custom build args
+
+        Returns:
+            True if build succeeded, False otherwise
+        """
+
+        self.logging.info(f"Building multi-platform image: {image_name}")
+        self.logging.info("Platforms: linux/amd64, linux/arm64")
+        self.logging.debug(f"Dockerfile: {dockerfile}")
+        self.logging.debug(f"Build args: {buildargs}")
+
+        # Build buildx command
+        cmd = [
+            "docker",
+            "buildx",
+            "build",
+            "--platform",
+            "linux/amd64,linux/arm64",
+            "--push",  # Multi-platform requires push
+            "-f",
+            dockerfile,
+            "-t",
+            image_name,
+        ]
+
+        # Add build args
+        for key, value in buildargs.items():
+            cmd.extend(["--build-arg", f"{key}={value}"])
+
+        # Add context path
+        cmd.append(str(self.project_dir))
+
+        try:
+            # Run buildx command
+            result = subprocess.run(
+                cmd,
+                check=True,
+                capture_output=True,
+                text=True,
+                cwd=str(self.project_dir),
+            )
+            self.logging.info(f"Successfully built and pushed multi-platform image: {image_name}")
+            self.logging.debug(f"Buildx output: {result.stdout}")
+            return True
+        except subprocess.CalledProcessError as exc:
+            self.logging.error(f"Multi-platform build failed for {image_name}")
+            self.logging.error(f"Exit code: {exc.returncode}")
+            self.logging.error(f"Stdout: {exc.stdout}")
+            self.logging.error(f"Stderr: {exc.stderr}")
+            return False
+        except FileNotFoundError:
+            self.logging.error("docker buildx command not found")
+            self.logging.error("Please ensure Docker buildx is installed")
+            return False
+
     def _execute_build(
         self,
         system: str,
@@ -85,11 +175,15 @@ def _execute_build(
         version: Optional[str] = None,
         version_name: Optional[str] = None,
         platform: Optional[str] = None,
+        multi_platform: bool = False,
         parallel: int = 1,
     ) -> bool:
         """
         Execute the actual build operation for a single image.
 
+        For AWS Python/Node.js build images, uses multi-platform build (x64 + arm64).
+        For all other images, uses standard single-platform build.
+
         Args:
             system: Deployment platform
             image_type: Type of image
@@ -97,11 +191,13 @@ def _execute_build(
             version: Language version, optional
             version_name: Base image name for the version, optional
             platform: Docker platform override, optional
+            multi_platform: If we should build both x64 and arm64
             parallel: Number of parallel workers, default 1
 
         Returns:
             True if build succeeded, False otherwise
         """
+
         # Locate dockerfile
         if language:
             dockerfile = os.path.join(
@@ -127,6 +223,16 @@ def _execute_build(
         if version_name:
             buildargs["BASE_IMAGE"] = version_name
 
+        # Check if multi-platform build should be used
+        if multi_platform and self._should_use_multiplatform(system, image_type, language):
+            if version is None or version_name is None:
+                self.logging.error(
+                    f"Multi-platform build not supported for {system}/{language}/{version}"
+                )
+                return False
+            return self._execute_multiplatform_build(image_name, dockerfile, buildargs)
+
+        # Standard single-platform build
         platform_arg = platform or os.environ.get("DOCKER_DEFAULT_PLATFORM")
 
         self.logging.debug(f"Building {image_name} from {dockerfile}")
@@ -162,6 +268,9 @@ def _execute_push(
         """
         Execute the actual push operation for a single image.
 
+        Multi-platform images (AWS Python/Node.js build images) are skipped
+        as they are already pushed during the build process.
+
         Args:
             system: Deployment platform
             image_type: Type of image
@@ -172,6 +281,14 @@ def _execute_push(
         Returns:
             True if push succeeded, False otherwise
         """
+        # Skip multi-platform images - they're already pushed during build
+        if self._should_use_multiplatform(system, image_type, language):
+            image_name = self.config.docker_image_name(system, image_type, language, version)
+            self.logging.info(
+                f"Skipping push for multi-platform image (already pushed): {image_name}"
+            )
+            return True
+
         # Full Docker image tag
         image_name = self.config.docker_image_name(system, image_type, language, version)
 
@@ -210,6 +327,7 @@ def _process_image(
         version: Optional[str] = None,
         version_name: Optional[str] = None,
         platform: Optional[str] = None,
+        multi_platform: bool = False,
         parallel: int = 1,
     ) -> bool:
         """
@@ -223,6 +341,7 @@ def _process_image(
             version: Language version, optional
             version_name: Base image name for the version, optional
             platform: Docker platform override, optional
+            multi_platform: If we should build both x64 and arm64
             parallel: Number of parallel workers, default 1
 
         Returns:
@@ -239,7 +358,14 @@ def _process_image(
         # Execute the appropriate operation
         if operation == "build":
             return self._execute_build(
-                system, image_type, language, version, version_name, platform, parallel
+                system,
+                image_type,
+                language,
+                version,
+                version_name,
+                platform,
+                multi_platform,
+                parallel,
             )
         elif operation == "push":
             return self._execute_push(system, image_type, language, version)
@@ -256,6 +382,7 @@ def _process_language(
         language_version: Optional[str] = None,
         image_type: Optional[str] = None,
         platform: Optional[str] = None,
+        multi_platform: bool = False,
         parallel: int = 1,
     ) -> None:
         """
@@ -270,6 +397,7 @@ def _process_language(
             language_version: Specific version to process, processes all if None
             image_type: Specific image type to process, processes all if None
             platform: Docker platform override, optional
+            multi_platform: If we should build both x64 and arm64
             parallel: Number of parallel workers, default 1
         """
         # Maps to language_version and Docker base image for that version
@@ -300,6 +428,7 @@ def _process_language(
                         language,
                         *image_config,
                         platform=platform,
+                        multi_platform=multi_platform,
                         parallel=parallel,
                     )
             else:
@@ -310,6 +439,7 @@ def _process_language(
                     language,
                     *image_config,
                     platform=platform,
+                    multi_platform=multi_platform,
                     parallel=parallel,
                 )
 
@@ -324,6 +454,7 @@ def _process_system(
         architecture: str = "x64",
         dependency_type: Optional[str] = None,
         platform: Optional[str] = None,
+        multi_platform: bool = False,
         parallel: int = 1,
     ) -> None:
         """
@@ -339,6 +470,7 @@ def _process_system(
             architecture: Target architecture, default "x64"
             dependency_type: Specific dependency for cpp (opencv, boost, etc.), optional
             platform: Docker platform override, optional
+            multi_platform: If we should build both x64 and arm64
             parallel: Number of parallel workers, default 1
         """
         if image_type == "manage":
@@ -376,6 +508,7 @@ def _process_system(
                         version,
                         base_image,
                         platform=platform,
+                        multi_platform=multi_platform,
                         parallel=parallel,
                     )
             else:
@@ -391,6 +524,7 @@ def _process_system(
                             version,
                             base_image,
                             platform=platform,
+                            multi_platform=multi_platform,
                             parallel=parallel,
                         )
         else:
@@ -407,6 +541,7 @@ def _process_system(
                         language_version=language_version,
                         image_type=image_type,
                         platform=platform,
+                        multi_platform=multi_platform,
                         parallel=parallel,
                     )
                 else:
@@ -423,13 +558,19 @@ def _process_system(
                         language_version=language_version,
                         image_type=image_type,
                         platform=platform,
+                        multi_platform=multi_platform,
                         parallel=parallel,
                     )
                 # No filters - process additional image types supported on the platform
                 if "images" in system_config:
                     for img_type, _ in system_config["images"].items():
                         self._process_image(
-                            operation, system, img_type, platform=platform, parallel=parallel
+                            operation,
+                            system,
+                            img_type,
+                            platform=platform,
+                            multi_platform=multi_platform,
+                            parallel=parallel,
                         )
 
     def _process(
@@ -442,6 +583,7 @@ def _process(
         architecture: str = "x64",
         dependency_type: Optional[str] = None,
         platform: Optional[str] = None,
+        multi_platform: bool = False,
         parallel: int = 1,
     ) -> None:
         """
@@ -459,6 +601,7 @@ def _process(
             architecture: Target architecture, default "x64"
             dependency_type: Specific dependency for cpp, optional
             platform: Docker platform override, optional
+            multi_platform: If we should build both x64 and arm64
             parallel: Number of parallel workers, default 1
         """
         systems_config = self.config._system_config
@@ -475,6 +618,7 @@ def _process(
                     architecture=architecture,
                     dependency_type=dependency_type,
                     platform=platform,
+                    multi_platform=multi_platform,
                     parallel=parallel,
                 )
             else:
@@ -491,6 +635,7 @@ def _process(
                     architecture=architecture,
                     dependency_type=dependency_type,
                     platform=platform,
+                    multi_platform=multi_platform,
                     parallel=parallel,
                 )
 
@@ -503,6 +648,7 @@ def build(
         architecture: str = "x64",
         dependency_type: Optional[str] = None,
         platform: Optional[str] = None,
+        multi_platform: bool = False,
         parallel: int = 1,
     ) -> None:
         """
@@ -522,6 +668,7 @@ def build(
             architecture: Target architecture, default "x64"
             dependency_type: Specific dependency for cpp, optional
             platform: Docker platform override, optional
+            multi_platform: If we should build both x64 and arm64
             parallel: Number of parallel workers, default 1
         """
         self._process(
@@ -533,6 +680,7 @@ def build(
             architecture=architecture,
             dependency_type=dependency_type,
             platform=platform,
+            multi_platform=multi_platform,
             parallel=parallel,
         )
 

From 921f5b667dd3f742cb0a4c81254ad7471a142db0 Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Thu, 16 Apr 2026 21:15:03 +0200
Subject: [PATCH 40/72] [docs] Update docs on container images

---
 docs/platforms.md | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docs/platforms.md b/docs/platforms.md
index 21d05ef7..37069411 100644
--- a/docs/platforms.md
+++ b/docs/platforms.md
@@ -42,8 +42,7 @@ To rebuild multi-plaform images, an additional flag is needed to enable the inte
 sebs docker build --image-type build --language python --deployment aws --architecture x64 --language-version 3.11 --multi-platform
 ```
 
-When rebuilding build images (not necessary for regular users, only for developers), make sure that your Docker installation supports multi-platform images,
-e.g., you use `containerd` image store.
+When rebuilding build images (not necessary for regular users, only for developers), make sure that your Docker installation supports multi-platform images, e.g., [you use `containerd` image store](https://docs.docker.com/engine/storage/containerd/) - old Docker installations might not change the storage type after an upgrade to Docker 29.0, where `containerd` is the default.
 
 ## Cloud Account Identifiers
 

From 106ece608a4ccb79e1e4c2437a940dec203e7fca Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Thu, 16 Apr 2026 21:15:15 +0200
Subject: [PATCH 41/72] [gcp] Tolerate 503 errors

---
 sebs/gcp/gcp.py | 47 +++++++++++++++++++++++++++++++++++++----------
 1 file changed, 37 insertions(+), 10 deletions(-)

diff --git a/sebs/gcp/gcp.py b/sebs/gcp/gcp.py
index c33f3240..17cf3dd7 100644
--- a/sebs/gcp/gcp.py
+++ b/sebs/gcp/gcp.py
@@ -69,6 +69,11 @@ class GCP(System):
         logging_handlers: Logging configuration for status reporting
     """
 
+    """
+        Google API is not the most robust - sometimes we need to retry REST operations.
+    """
+    TRANSIENT_HTTP_CODES = frozenset({429, 503})
+
     def __init__(
         self,
         system_config: SeBSConfig,
@@ -336,6 +341,9 @@ def _wait_for_active_status(
 
         After a build completes, the function may be in DEPLOY_IN_PROGRESS state
         for a short time. This function polls until the status becomes ACTIVE.
+        Furthermore, we handle HTTP errors:
+        * 503 / 429 transient backend errors — GCP Cloud Functions v1
+            can periodically returns these; they are not deployment failures.
 
         Args:
             func_name: Name of the function to check
@@ -352,18 +360,43 @@ def _wait_for_active_status(
             self.config.project_name, self.config.region, func_name
         )
         begin = time.time()
+        last_status: Optional[str] = None
 
         self.logging.info(f"Waiting for function {func_name} to become ACTIVE...")
 
         while True:
-            get_req = (
-                self.function_client.projects().locations().functions().get(name=full_func_name)
-            )
-            func_details = get_req.execute()
+
+            elapsed = time.time() - begin
+            if elapsed > timeout:
+                raise RuntimeError(
+                    f"Timeout waiting for function {func_name} to become ACTIVE "
+                    f"after {elapsed:.0f}s. Last status: {last_status}"
+                )
+
+            try:
+                get_req = (
+                    self.function_client.projects().locations().functions().get(name=full_func_name)
+                )
+                func_details = get_req.execute()
+            except HttpError as e:
+
+                status_code = e.resp.status
+                if status_code in GCP.TRANSIENT_HTTP_CODES:
+                    self.logging.warning(
+                        f"Transient error {status_code} while polling {func_name}, "
+                        f"retrying ({elapsed:.0f}s elapsed)"
+                    )
+                    time.sleep(5)  # back off a bit more for 5xx
+                    continue
+                # 404 past grace window, 403, 400, etc. — real error.
+                raise
 
             status = func_details["status"]
             current_version = int(func_details["versionId"])
 
+            if status != last_status:
+                last_status = status
+
             if status == "ACTIVE":
                 # Check version if specified
                 if expected_version is not None and current_version != expected_version:
@@ -382,12 +415,6 @@ def _wait_for_active_status(
                 self.logging.error(f"Function {func_name} has unexpected status: {status}")
                 raise RuntimeError(f"Function {func_name} deployment failed with status: {status}")
 
-            if time.time() - begin > timeout:
-                raise RuntimeError(
-                    f"Timeout waiting for function {func_name} to become ACTIVE. "
-                    f"Current status: {status}"
-                )
-
             time.sleep(2)
 
     def package_code(

From c6f510116981d41694a628d617d26a7f9ddbe331 Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Thu, 16 Apr 2026 22:54:31 +0200
Subject: [PATCH 42/72] [gcp] Add public access to all functions

---
 sebs/gcp/gcp.py | 80 ++++++++++++++++++++++++-------------------------
 1 file changed, 40 insertions(+), 40 deletions(-)

diff --git a/sebs/gcp/gcp.py b/sebs/gcp/gcp.py
index 17cf3dd7..b663fbc4 100644
--- a/sebs/gcp/gcp.py
+++ b/sebs/gcp/gcp.py
@@ -511,6 +511,44 @@ def package_code(
             bytes_size,
         )
 
+    def _allow_public_access(self, func_name: str, full_func_name: str) -> None:
+
+        """Configure GCP function to be publicly accessible.
+
+        Args:
+            func_name: our function name
+            full_func_name: GCP name
+
+        Raises:
+            RuntimeError:
+        """
+        allow_unauthenticated_req = (
+            self.function_client.projects()
+            .locations()
+            .functions()
+            .setIamPolicy(
+                resource=full_func_name,
+                body={
+                    "policy": {
+                        "bindings": [
+                            {
+                                "role": "roles/cloudfunctions.invoker",
+                                "members": ["allUsers"],
+                            }
+                        ]
+                    }
+                },
+            )
+        )
+        try:
+            allow_unauthenticated_req.execute()
+        except HttpError as e:
+            raise RuntimeError(
+                f"Failed to configure function {full_func_name} "
+                f"for unauthenticated invocations! Error: {e}"
+            )
+        self.logging.info(f"Function {func_name} accepts now unauthenticated invocations!")
+
     def create_function(
         self,
         code_package: Benchmark,
@@ -612,46 +650,7 @@ def create_function(
             # Wait for deployment to become ACTIVE
             self._wait_for_active_status(func_name)
 
-            allow_unauthenticated_req = (
-                self.function_client.projects()
-                .locations()
-                .functions()
-                .setIamPolicy(
-                    resource=full_func_name,
-                    body={
-                        "policy": {
-                            "bindings": [
-                                {
-                                    "role": "roles/cloudfunctions.invoker",
-                                    "members": ["allUsers"],
-                                }
-                            ]
-                        }
-                    },
-                )
-            )
-
-            # Avoid infinite loop
-            MAX_RETRIES = 5
-            counter = 0
-            while counter < MAX_RETRIES:
-                try:
-                    allow_unauthenticated_req.execute()
-                    break
-                except HttpError:
-
-                    self.logging.info(
-                        "Sleeping for 5 seconds because the created functions is not yet available!"
-                    )
-                    time.sleep(5)
-                    counter += 1
-            else:
-                raise RuntimeError(
-                    f"Failed to configure function {full_func_name} "
-                    "for unauthenticated invocations!"
-                )
-
-            self.logging.info(f"Function {func_name} accepts now unauthenticated invocations!")
+            self._allow_public_access(func_name, full_func_name)
 
             function = GCPFunction(
                 func_name, benchmark, code_package.hash, function_cfg, code_bucket
@@ -667,6 +666,7 @@ def create_function(
                 cfg=function_cfg,
                 bucket=code_bucket,
             )
+            self._allow_public_access(func_name, full_func_name)
             self.update_function(function, code_package, container_deployment, container_uri)
 
         # Add LibraryTrigger to a new function

From 5a1f9fa3cf1ad7525a63d519105205d2e6a84727 Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Thu, 16 Apr 2026 22:54:50 +0200
Subject: [PATCH 43/72] [ci] Extend timeout to handle Azure

---
 .github/workflows/_regression-job.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/_regression-job.yml b/.github/workflows/_regression-job.yml
index e7009916..b70377cf 100644
--- a/.github/workflows/_regression-job.yml
+++ b/.github/workflows/_regression-job.yml
@@ -66,7 +66,7 @@ jobs:
           uv pip install .
 
       - name: Run regression tests
-        timeout-minutes: 5
+        timeout-minutes: 10
         run: |
           source .venv/bin/activate
           uv run sebs benchmark regression test \

From 7a8bfb979495e1ace1275f568cc7af57e540e556 Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Thu, 16 Apr 2026 23:38:19 +0200
Subject: [PATCH 44/72] [azure] Ensure that regression always logs in

---
 sebs/regression.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/sebs/regression.py b/sebs/regression.py
index 0daf3aa1..6b11f07b 100644
--- a/sebs/regression.py
+++ b/sebs/regression.py
@@ -542,12 +542,14 @@ def get_deployment(self, benchmark_name, architecture, deployment_type):
                 )
 
             # Initialize Azure CLI if not already done
+            needs_login = False
             if not hasattr(AzureTestSequencePython, "cli"):
                 from sebs.azure.cli import AzureCLI
 
                 AzureTestSequencePython.cli = AzureCLI(
                     self.client.config, self.client.docker_client
                 )
+                needs_login = True
 
             # Create a copy of the config and set architecture and deployment type
             config_copy = copy.deepcopy(cloud_config)
@@ -565,7 +567,7 @@ def get_deployment(self, benchmark_name, architecture, deployment_type):
 
             # Initialize CLI with login and setup resources
             deployment_client.system_resources.initialize_cli(
-                cli=AzureTestSequencePython.cli, login=True
+                cli=AzureTestSequencePython.cli, login=needs_login
             )
             deployment_client.initialize(resource_prefix=RESOURCE_PREFIX)
             return deployment_client
@@ -621,12 +623,14 @@ def get_deployment(self, benchmark_name, architecture, deployment_type):
                 )
 
             # Initialize Azure CLI if not already done
+            needs_login = False
             if not hasattr(AzureTestSequenceNodejs, "cli"):
                 from sebs.azure.cli import AzureCLI
 
                 AzureTestSequenceNodejs.cli = AzureCLI(
                     self.client.config, self.client.docker_client
                 )
+                needs_login = True
 
             # Create a copy of the config and set architecture and deployment type
             config_copy = copy.deepcopy(cloud_config)
@@ -643,7 +647,9 @@ def get_deployment(self, benchmark_name, architecture, deployment_type):
             )
 
             # Initialize CLI and setup resources (no login needed - reuses Python session)
-            deployment_client.system_resources.initialize_cli(cli=AzureTestSequenceNodejs.cli)
+            deployment_client.system_resources.initialize_cli(
+                cli=AzureTestSequenceNodejs.cli, login=needs_login
+            )
             deployment_client.initialize(resource_prefix=RESOURCE_PREFIX)
             return deployment_client
 

From b4a34b54349a1a34c37286b9753903a46787d735 Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Fri, 17 Apr 2026 00:16:00 +0200
Subject: [PATCH 45/72] [docs] Update config paths

---
 docs/platforms.md |  8 ++++----
 docs/storage.md   | 10 +++++-----
 docs/usage.md     | 18 +++++++++---------
 3 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/docs/platforms.md b/docs/platforms.md
index 37069411..9bda1fa7 100644
--- a/docs/platforms.md
+++ b/docs/platforms.md
@@ -3,7 +3,7 @@
 SeBS supports three commercial serverless platforms: AWS Lambda, Azure Functions, and Google Cloud Functions.
 Furthermore, we support the open source FaaS system OpenWhisk.
 
-The file `config/example.json` contains all parameters that users can change
+The file `configs/example.json` contains all parameters that users can change
 to customize the deployment.
 Some of these parameters, such as cloud credentials or storage instance address,
 are required.
@@ -63,7 +63,7 @@ Additionally, the account must have `AmazonAPIGatewayAdministrator` permission t
 automatically AWS HTTP trigger.
 You can provide a [role](https://docs.aws.amazon.com/lambda/latest/dg/lambda-intro-execution-role.html)
 with permissions to access AWS Lambda and S3; otherwise, one will be created automatically.
-To use a user-defined lambda role, set the name in config JSON - see an example in `config/example.json`.
+To use a user-defined lambda role, set the name in config JSON - see an example in `configs/example.json`.
 
 You can pass the credentials either using the default AWS-specific environment variables:
 
@@ -219,7 +219,7 @@ or in the JSON input configuration:
 SeBS expects users to deploy and configure an OpenWhisk instance.
 Below, you will find example of instruction for deploying OpenWhisk instance.
 The configuration parameters of OpenWhisk for SeBS can be found
-in `config/example.json` under the key `['deployment']['openwhisk']`.
+in `configs/example.json` under the key `['deployment']['openwhisk']`.
 In the subsections below, we discuss the meaning and use of each parameter.
 To correctly deploy SeBS functions to OpenWhisk, following the
 subsections on *Toolchain* and *Docker* configuration is particularly important.
@@ -293,7 +293,7 @@ and new language versions, Docker images must be placed in the registry.
 However, pushing the image to the default `spcleth/serverless-benchmarks`
 repository on Docker Hub requires permissions.
 To use a different Docker Hub repository, change the key
-`['general']['docker_repository']` in `config/systems.json`.
+`['general']['docker_repository']` in `configs/systems.json`.
 
 Alternatively, OpenWhisk users can configure the FaaS platform to use a custom and
 private Docker registry and push new images there.
diff --git a/docs/storage.md b/docs/storage.md
index 35bde19f..2f6cc54c 100644
--- a/docs/storage.md
+++ b/docs/storage.md
@@ -19,13 +19,13 @@ You can start the necessary storage services using the `storage` command in SeBS
 
 ```bash
 # Start only object storage
-sebs storage start object config/storage.json --output-json storage_object.json
+sebs storage start object configs/storage.json --output-json storage_object.json
 
 # Start only NoSQL database
-sebs storage start nosql config/storage.json --output-json storage_nosql.json
+sebs storage start nosql configs/storage.json --output-json storage_nosql.json
 
 # Start both storage types
-sebs storage start all config/storage.json --output-json storage.json
+sebs storage start all configs/storage.json --output-json storage.json
 ```
 
 The command deploys the requested storage services as Docker containers and generates a configuration file in JSON format.
@@ -87,7 +87,7 @@ For example, for an external address `10.10.1.15` (a LAN-local address on CloudL
 
 ```bash
 # For a LAN-local address (e.g., on CloudLab)
-jq --slurpfile file1 storage.json '.deployment.openwhisk.storage = $file1[0] | .deployment.openwhisk.storage.object.minio.address = "10.10.1.15:9011"' config/example.json > config/openwhisk.json
+jq --slurpfile file1 storage.json '.deployment.openwhisk.storage = $file1[0] | .deployment.openwhisk.storage.object.minio.address = "10.10.1.15:9011"' configs/example.json > configs/openwhisk.json
 ```
 
 You can validate the configuration of Minio with an HTTP request by using `curl`:
@@ -112,7 +112,7 @@ Here, we again assume the external IP address of the system is `10.10.1.15`, and
 
 ```bash
 # For a LAN-local address (e.g., on CloudLab)
-jq '.deployment.openwhisk.storage.nosql.scylladb.address = "10.10.1.15:9012"' config/openwhisk.json | sponge config/openwhisk.json
+jq '.deployment.openwhisk.storage.nosql.scylladb.address = "10.10.1.15:9012"' configs/openwhisk.json | sponge configs/openwhisk.json
 ```
 
 You can validate the configuration of ScyllaDB with an HTTP request by using `curl`:
diff --git a/docs/usage.md b/docs/usage.md
index f97a549c..9d64dab1 100644
--- a/docs/usage.md
+++ b/docs/usage.md
@@ -21,7 +21,7 @@ If you want to simply build a function deployment, such as a full code package o
 then use the command below.
 
 ```bash
-sebs benchmark build 110.dynamic-html --config config/example.json --deployment aws
+sebs benchmark build 110.dynamic-html --config configs/example.json --deployment aws
 ```
 
 It will create a code package (local) or build and push a container, when `--container-deployment` flag is used (AWS only).
@@ -33,7 +33,7 @@ This command builds, deploys, and executes serverless benchmarks in the cloud.
 The example below invokes the benchmark `110.dynamic-html` on AWS via the standard HTTP trigger.
 
 ```bash
-sebs benchmark invoke 110.dynamic-html test --config config/example.json --deployment aws --verbose
+sebs benchmark invoke 110.dynamic-html test --config configs/example.json --deployment aws --verbose
 ```
 
 The results will be stored in `experiment.json`.
@@ -64,13 +64,13 @@ Additionally, we provide a regression option to execute all benchmarks on a give
 The example below demonstrates how to run the regression suite with `test` input size on AWS.
 
 ```bash
-sebs benchmark regression test --config config/example.json --deployment aws
+sebs benchmark regression test --config configs/example.json --deployment aws
 ```
 
 The regression can be executed on a single benchmark as well:
 
 ```bash
-sebs benchmark regression test --config config/example.json --deployment aws --benchmark-name 120.uploader
+sebs benchmark regression test --config configs/example.json --deployment aws --benchmark-name 120.uploader
 ```
 
 ## Experiment
@@ -78,7 +78,7 @@ sebs benchmark regression test --config config/example.json --deployment aws --b
 This command is used to execute benchmarks described in the paper. The example below runs the experiment **perf-cost**:
 
 ```bash
-sebs experiment invoke perf-cost --config config/example.json --deployment aws
+sebs experiment invoke perf-cost --config configs/example.json --deployment aws
 ```
 
 The configuration specifies that benchmark **110.dynamic-html** is executed 50 times, with 50 concurrent invocations, and both cold and warm invocations are recorded. 
@@ -107,7 +107,7 @@ sebs experiment process perf-cost --config example.json --deployment aws
 You can remove all allocated cloud resources with the following command:
 
 ```bash
-sebs resource clean --config config/example.json
+sebs resource clean --config configs/example.json
 ```
 
 This option is currently supported only on AWS, where it removes Lambda functions and associated HTTP APIs and CloudWatch logs,
@@ -123,7 +123,7 @@ map the container's port to port defined in the configuration on host network, a
 instance configuration to file `out_storage.json`
 
 ```bash
-sebs storage start all config/storage.json --output-json out_storage.json
+sebs storage start all configs/storage.json --output-json out_storage.json
 ```
 
 Then, we need to update the configuration of `local` deployment with information on the storage 
@@ -132,7 +132,7 @@ instance. The `.deployment.local` object in the configuration JSON must contain
 this automatically with a single command by using `jq`:
 
 ```bash
-jq '.deployment.local.storage = input' config/example.json out_storage.json > config/local_deployment.json
+jq '.deployment.local.storage = input' configs/example.json out_storage.json > configs/local_deployment.json
 ```
 
 The output file will contain a JSON object that should look similar to this one:
@@ -183,7 +183,7 @@ The output file will contain a JSON object that should look similar to this one:
 To launch Docker containers, use the following command - this example launches benchmark `110.dynamic-html` with size `test`:
 
 ```bash
-sebs local start 110.dynamic-html test out_benchmark.json --config config/local_deployment.json --deployments 1 --remove-containers --architecture=x64
+sebs local start 110.dynamic-html test out_benchmark.json --config configs/local_deployment.json --deployments 1 --remove-containers --architecture=x64
 ```
 
 The output file `out_benchmark.json` will contain the information on containers deployed and the endpoints that can be used to invoke functions:

From 55adc37596082ba27826c0b7fd11e2b45b31de9f Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Fri, 17 Apr 2026 01:11:05 +0200
Subject: [PATCH 46/72] [aws] Extend build image to arm64 compatibility

---
 dockerfiles/aws/python/Dockerfile.build | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/dockerfiles/aws/python/Dockerfile.build b/dockerfiles/aws/python/Dockerfile.build
index 3c6f2f74..e98b0f45 100755
--- a/dockerfiles/aws/python/Dockerfile.build
+++ b/dockerfiles/aws/python/Dockerfile.build
@@ -2,13 +2,14 @@ ARG BASE_IMAGE
 FROM ${BASE_IMAGE}
 ARG VERSION
 ENV PYTHON_VERSION=${VERSION}
+ARG TARGETARCH
 
 # useradd, groupmod
 RUN yum install -y shadow-utils zip
 ENV GOSU_VERSION 1.14
 # https://github.com/tianon/gosu/releases/tag/1.14
 # key https://keys.openpgp.org/search?q=tianon%40debian.org
-RUN curl -o /usr/local/bin/gosu -SL "https://github.com/tianon/gosu/releases/download/${GOSU_VERSION}/gosu-amd64" \
+RUN curl -o /usr/local/bin/gosu -SL "https://github.com/tianon/gosu/releases/download/${GOSU_VERSION}/gosu-${TARGETARCH}" \
     && chmod +x /usr/local/bin/gosu
 RUN mkdir -p /sebs/
 COPY dockerfiles/python_installer.sh /sebs/installer.sh

From 5e3400adcaaf2a6bc03e18d1f36facb5e0d2a251 Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Fri, 17 Apr 2026 01:17:10 +0200
Subject: [PATCH 47/72] [system] Proper build of multi-platform images

---
 sebs/docker_builder.py | 175 +++++++++++++++++++++++++++++------------
 sebs/utils.py          |   2 +-
 2 files changed, 124 insertions(+), 53 deletions(-)

diff --git a/sebs/docker_builder.py b/sebs/docker_builder.py
index e03cdc6c..a2cf1021 100644
--- a/sebs/docker_builder.py
+++ b/sebs/docker_builder.py
@@ -9,7 +9,6 @@
 import json
 import logging
 import os
-import subprocess
 from pathlib import Path
 from typing import Dict, List, Optional, Tuple
 
@@ -18,7 +17,7 @@
 from rich.progress import Progress, TaskID
 
 from sebs.config import SeBSConfig
-from sebs.utils import LoggingBase
+from sebs.utils import execute, LoggingBase
 
 
 class DockerImageBuilder(LoggingBase):
@@ -100,73 +99,145 @@ def _should_use_multiplatform(
 
     def _execute_multiplatform_build(
         self,
+        system: str,
+        language: str,
+        version: str,
         image_name: str,
         dockerfile: str,
         buildargs: dict,
     ) -> bool:
-        """Execute multi-platform build using Docker buildx.
+        """Execute multi-platform build by building separate images and stitching them.
 
-        Builds an image for both linux/amd64 and linux/arm64 platforms
-        and pushes it as a multi-platform manifest to the registry.
+        Builds separate images for linux/amd64 and linux/arm64 with their respective
+        BASE_IMAGE values, then creates a multi-platform manifest to combine them.
 
         Args:
-            image_name: Docker image tag
-            dockerfile: path to Dockerfile
-            buildargs: custom build args
+            system: Deployment platform (e.g., 'aws')
+            language: Programming language (e.g., 'python', 'nodejs')
+            version: Language version (e.g., '3.11', '16')
+            image_name: Docker image tag (without architecture suffix)
+            dockerfile: Path to Dockerfile
+            buildargs: Base build args (BASE_IMAGE will be overridden per arch)
 
         Returns:
             True if build succeeded, False otherwise
         """
-
         self.logging.info(f"Building multi-platform image: {image_name}")
         self.logging.info("Platforms: linux/amd64, linux/arm64")
         self.logging.debug(f"Dockerfile: {dockerfile}")
-        self.logging.debug(f"Build args: {buildargs}")
 
-        # Build buildx command
-        cmd = [
-            "docker",
-            "buildx",
-            "build",
-            "--platform",
-            "linux/amd64,linux/arm64",
-            "--push",  # Multi-platform requires push
-            "-f",
-            dockerfile,
-            "-t",
-            image_name,
-        ]
+        # Get architecture-specific base images from config
+        systems_config = self.config._system_config
+        language_config = systems_config[system]["languages"][language]
+
+        if "base_images" not in language_config:
+            self.logging.error(f"No base_images found for {system}/{language}")
+            return False
+
+        base_images = language_config["base_images"]
+        if "x64" not in base_images or "arm64" not in base_images:
+            self.logging.error(f"Missing x64 or arm64 base images for {system}/{language}")
+            return False
+
+        if version not in base_images["x64"] or version not in base_images["arm64"]:
+            self.logging.error(f"Version {version} not found in base images")
+            return False
+
+        amd64_base = base_images["x64"][version]
+        arm64_base = base_images["arm64"][version]
+
+        self.logging.debug(f"AMD64 base image: {amd64_base}")
+        self.logging.debug(f"ARM64 base image: {arm64_base}")
+
+        # Architecture-specific image tags
+        amd64_image = f"{image_name}-amd64"
+        arm64_image = f"{image_name}-arm64"
+
+        # Build both architecture images
+        for platform, base_image, arch_image in [
+            ("linux/amd64", amd64_base, amd64_image),
+            ("linux/arm64", arm64_base, arm64_image),
+        ]:
+            self.logging.info(f"Building {platform} image: {arch_image}")
 
-        # Add build args
-        for key, value in buildargs.items():
-            cmd.extend(["--build-arg", f"{key}={value}"])
+            # Override BASE_IMAGE for this architecture
+            arch_buildargs = buildargs.copy()
+            arch_buildargs["BASE_IMAGE"] = base_image
 
-        # Add context path
-        cmd.append(str(self.project_dir))
+            # Build command
+            cmd = [
+                "docker",
+                "build",
+                "--platform",
+                platform,
+                "--provenance",
+                "false",
+                "-f",
+                dockerfile,
+                "-t",
+                arch_image,
+                "--push",
+            ]
+
+            # Add build args
+            for key, value in arch_buildargs.items():
+                cmd.extend(["--build-arg", f"{key}={value}"])
+
+            # Add context path
+            cmd.append(str(self.project_dir))
 
+            try:
+                execute(cmd, cwd=str(self.project_dir))
+                self.logging.info(f"Successfully built {arch_image}")
+            except RuntimeError as exc:
+                self.logging.error(f"Build failed for {arch_image}")
+                self.logging.error(f"Exit: {exc}")
+                return False
+
+        self.logging.info(f"Removing old multi-platform manifest: {image_name}")
+        manifest_cmd = [
+            "docker",
+            "manifest",
+            "rm",
+            image_name,
+        ]
         try:
-            # Run buildx command
-            result = subprocess.run(
-                cmd,
-                check=True,
-                capture_output=True,
-                text=True,
-                cwd=str(self.project_dir),
-            )
-            self.logging.info(f"Successfully built and pushed multi-platform image: {image_name}")
-            self.logging.debug(f"Buildx output: {result.stdout}")
-            return True
-        except subprocess.CalledProcessError as exc:
-            self.logging.error(f"Multi-platform build failed for {image_name}")
-            self.logging.error(f"Exit code: {exc.returncode}")
-            self.logging.error(f"Stdout: {exc.stdout}")
-            self.logging.error(f"Stderr: {exc.stderr}")
+            execute(manifest_cmd)
+            self.logging.info(f"Successfully removed old manifest {image_name}")
+        except RuntimeError:
+            # ignore, manifest might not have existed
+            pass
+
+        # Create multi-platform manifest
+        self.logging.info(f"Creating multi-platform manifest: {image_name}")
+        manifest_cmd = [
+            "docker",
+            "manifest",
+            "create",
+            image_name,
+            amd64_image,
+            arm64_image,
+        ]
+        try:
+            execute(manifest_cmd)
+            self.logging.info(f"Successfully created manifest: {image_name}")
+        except RuntimeError as exc:
+            self.logging.error(f"Manifest creation failed for {image_name}")
+            self.logging.error(f"Exit: {exc}")
             return False
-        except FileNotFoundError:
-            self.logging.error("docker buildx command not found")
-            self.logging.error("Please ensure Docker buildx is installed")
+
+        self.logging.info(f"Pushing multi-platform manifest: {image_name}")
+        push_cmd = ["docker", "manifest", "push", image_name]
+        try:
+            execute(push_cmd)
+            self.logging.info(f"Successfully created manifest: {image_name}")
+        except RuntimeError as exc:
+            self.logging.error(f"Manifest creation failed for {image_name}")
+            self.logging.error(f"Exit: {exc}")
             return False
 
+        return True
+
     def _execute_build(
         self,
         system: str,
@@ -225,12 +296,12 @@ def _execute_build(
 
         # Check if multi-platform build should be used
         if multi_platform and self._should_use_multiplatform(system, image_type, language):
-            if version is None or version_name is None:
-                self.logging.error(
-                    f"Multi-platform build not supported for {system}/{language}/{version}"
-                )
+            if version is None or version_name is None or language is None:
+                self.logging.error("Multi-platform build requires version and language")
                 return False
-            return self._execute_multiplatform_build(image_name, dockerfile, buildargs)
+            return self._execute_multiplatform_build(
+                system, language, version, image_name, dockerfile, buildargs
+            )
 
         # Standard single-platform build
         platform_arg = platform or os.environ.get("DOCKER_DEFAULT_PLATFORM")
diff --git a/sebs/utils.py b/sebs/utils.py
index 538194ae..3f7e158e 100644
--- a/sebs/utils.py
+++ b/sebs/utils.py
@@ -148,7 +148,7 @@ def execute(cmd, shell=False, cwd=None) -> str:
     Raises:
         RuntimeError: If command execution fails
     """
-    if not shell:
+    if not shell and isinstance(cmd, str):
         cmd = cmd.split()
     ret = subprocess.run(
         cmd, shell=shell, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT

From 54bc65a34c7dcbb7eed67f791509eb7d84fa1ab8 Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Fri, 17 Apr 2026 01:36:34 +0200
Subject: [PATCH 48/72] [aws] Update Node.js build image for arm

---
 dockerfiles/aws/nodejs/Dockerfile.build | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/dockerfiles/aws/nodejs/Dockerfile.build b/dockerfiles/aws/nodejs/Dockerfile.build
index 29bc8985..c5ef797e 100755
--- a/dockerfiles/aws/nodejs/Dockerfile.build
+++ b/dockerfiles/aws/nodejs/Dockerfile.build
@@ -1,12 +1,13 @@
 ARG BASE_IMAGE
 FROM ${BASE_IMAGE}
+ARG TARGETARCH
 
 # useradd, groupmod
 RUN yum install -y shadow-utils cmake curl libcurl libcurl-devel
 ENV GOSU_VERSION 1.14
 # https://github.com/tianon/gosu/releases/tag/1.14
 # key https://keys.openpgp.org/search?q=tianon%40debian.org
-RUN curl -o /usr/local/bin/gosu -SL "https://github.com/tianon/gosu/releases/download/${GOSU_VERSION}/gosu-amd64" \
+RUN curl -o /usr/local/bin/gosu -SL "https://github.com/tianon/gosu/releases/download/${GOSU_VERSION}/gosu-${TARGETARCH}" \
     && chmod +x /usr/local/bin/gosu
 RUN mkdir -p /sebs/
 COPY dockerfiles/nodejs_installer.sh /sebs/installer.sh

From 31a775dc10be2d68aa0ef87d362101fbcee24703 Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Fri, 17 Apr 2026 02:08:31 +0200
Subject: [PATCH 49/72] [aws] Properly cleanup function resources

---
 sebs/aws/aws.py    |  1 +
 sebs/aws/config.py | 14 +++++---------
 2 files changed, 6 insertions(+), 9 deletions(-)

diff --git a/sebs/aws/aws.py b/sebs/aws/aws.py
index 9bf060b1..ca63e9ef 100644
--- a/sebs/aws/aws.py
+++ b/sebs/aws/aws.py
@@ -638,6 +638,7 @@ def delete_function(self, func_name: str) -> None:
         self.logging.info("Deleting function {}".format(func_name))
         try:
             self.client.delete_function(FunctionName=func_name)
+            self.config.resources.delete_function_url(func_name, self.session, self.cache_client)
         except Exception:
             self.logging.error("Function {} does not exist!".format(func_name))
 
diff --git a/sebs/aws/config.py b/sebs/aws/config.py
index b56b61b7..5a0fd127 100644
--- a/sebs/aws/config.py
+++ b/sebs/aws/config.py
@@ -655,7 +655,6 @@ def cleanup_function_urls(
         """
 
         deleted: List[str] = []
-        deleted_functions: List[str] = []
         dry_run_tag = "[DRY-RUN] " if dry_run else ""
 
         dict_copy = self._function_urls.copy()
@@ -664,14 +663,8 @@ def cleanup_function_urls(
             self.logging.info(f"{dry_run_tag}Deleting Function URL for: {func_name}")
 
             if not dry_run:
-                self.delete_function_url(func_name, boto3_session)
+                self.delete_function_url(func_name, boto3_session, cache_client)
             deleted.append(func_url.url)
-            deleted_functions.append(func_name)
-
-        if not dry_run:
-            for func_name in deleted_functions:
-                cache_client.remove_config_key(["aws", "resources", "function-urls", func_name])
-                self._function_urls.pop(func_name, None)
 
         return deleted
 
@@ -776,7 +769,9 @@ def function_url(
         self._function_urls[func.name] = function_url_obj
         return function_url_obj
 
-    def delete_function_url(self, function_name: str, boto3_session: boto3.session.Session) -> bool:
+    def delete_function_url(
+        self, function_name: str, boto3_session: boto3.session.Session, cache_client: Cache
+    ) -> bool:
         """
         Delete a Lambda Function URL for the given function.
         Returns True if deleted successfully, False if it didn't exist.
@@ -812,6 +807,7 @@ def delete_function_url(self, function_name: str, boto3_session: boto3.session.S
             # Only runs if no exception was raised - cleanup cache
             if function_name in self._function_urls:
                 del self._function_urls[function_name]
+                cache_client.remove_config_key(["aws", "resources", "function-urls", function_name])
             return True
 
     def check_ecr_repository_exists(

From 7bb7e4af45f0722691a93074370ef28d9c20e441 Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Fri, 17 Apr 2026 02:21:34 +0200
Subject: [PATCH 50/72] [gcp] Delete functions

---
 sebs/gcp/gcp.py | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/sebs/gcp/gcp.py b/sebs/gcp/gcp.py
index b663fbc4..8773039d 100644
--- a/sebs/gcp/gcp.py
+++ b/sebs/gcp/gcp.py
@@ -955,6 +955,31 @@ def update_function_configuration(
 
         return current_version
 
+    def delete_function(self, func_name: str) -> None:
+        """Delete a Google Cloud Function.
+
+        Args:
+            func_name: Name of the function to delete
+        """
+        self.logging.info(f"Deleting function {func_name}")
+
+        full_func_name = GCP.get_full_function_name(
+            self.config.project_name, self.config.region, func_name
+        )
+
+        try:
+            delete_req = (
+                self.function_client.projects().locations().functions().delete(name=full_func_name)
+            )
+            delete_req.execute()
+            self.logging.info(f"Function {func_name} deleted successfully")
+        except HttpError as e:
+            if e.resp.status == 404:
+                self.logging.error(f"Function {func_name} does not exist!")
+            else:
+                self.logging.error(f"Failed to delete function {func_name}: {e}")
+                raise
+
     @staticmethod
     def get_full_function_name(project_name: str, location: str, func_name: str) -> str:
         """Generate the fully qualified function name for GCP API calls.

From 3567ac5f78c47dddbf6ea6ffd096b058545d22d6 Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Fri, 17 Apr 2026 03:09:11 +0200
Subject: [PATCH 51/72] [aws] Proper cleaning of function URLs

---
 sebs/aws/config.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/sebs/aws/config.py b/sebs/aws/config.py
index 5a0fd127..83230641 100644
--- a/sebs/aws/config.py
+++ b/sebs/aws/config.py
@@ -807,7 +807,6 @@ def delete_function_url(
             # Only runs if no exception was raised - cleanup cache
             if function_name in self._function_urls:
                 del self._function_urls[function_name]
-                cache_client.remove_config_key(["aws", "resources", "function-urls", function_name])
             return True
 
     def check_ecr_repository_exists(
@@ -1082,8 +1081,13 @@ def update_cache(self, cache: Cache) -> None:
             keys=["aws", "resources", "container_repository"],
         )
         cache.update_config(val=self._lambda_role, keys=["aws", "resources", "lambda-role"])
+
+        # remove old entries before writing new data.
+        cache.remove_config_key(["aws", "resources", "http-apis"])
         for name, api in self._http_apis.items():
             cache.update_config(val=api.serialize(), keys=["aws", "resources", "http-apis", name])
+
+        cache.remove_config_key(["aws", "resources", "function-urls"])
         for name, func_url in self._function_urls.items():
             cache.update_config(
                 val=func_url.serialize(),

From 379942f051793251302ebf0ae36a2bbeba17ae2e Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Fri, 17 Apr 2026 03:31:41 +0200
Subject: [PATCH 52/72] [azure] Add function deletion option

---
 sebs/azure/azure.py  | 37 +++++++++++++++++++++++++++++++++++++
 sebs/azure/config.py | 33 +++++++++++++++++++++++++++++++--
 2 files changed, 68 insertions(+), 2 deletions(-)

diff --git a/sebs/azure/azure.py b/sebs/azure/azure.py
index 13138431..c3fb7fbf 100644
--- a/sebs/azure/azure.py
+++ b/sebs/azure/azure.py
@@ -587,6 +587,43 @@ def update_function_configuration(self, function: Function, code_package: Benchm
             "Updating function's memory and timeout configuration is not supported."
         )
 
+    def delete_function(self, func_name: str) -> None:
+        """Delete an Azure Function App and its associated storage account.
+
+        Args:
+            func_name: Name of the Azure Function App to delete
+        """
+        self.logging.info(f"Deleting function app {func_name}")
+
+        """
+            For Azure, we need to retrieve the associated storage account.
+            Each function has its own storage account.
+        """
+        all_functions = self.cache_client.get_all_functions(self.name())
+        if func_name not in all_functions:
+            self.logging.error(
+                f"Failed to find function {func_name} in functions: {all_functions.keys()}."
+            )
+            raise RuntimeError(f"Failed to find function {func_name} in cache.")
+
+        function = cast(AzureFunction, self.function_type().deserialize(all_functions[func_name]))
+
+        try:
+            self.cli_instance.execute(
+                f"az functionapp delete --name {func_name} "
+                f"--resource-group {self.config.resources.resource_group(self.cli_instance)}"
+            )
+            self.logging.info(f"Function app {func_name} deleted successfully")
+        except RuntimeError as e:
+            self.logging.error(f"Failed to delete the function app {func_name}!")
+            raise e
+
+        self.logging.info(
+            f"Deleting storage account {function.function_storage.account_name} "
+            f"associated with function {func_name}"
+        )
+        self.config.resources.delete_storage_account(self.cli_instance, function.function_storage)
+
     def _mount_function_code(self, code_package: Benchmark) -> str:
         """Mount function code package in Azure CLI container.
 
diff --git a/sebs/azure/config.py b/sebs/azure/config.py
index 44f1d4f2..11ec58ed 100644
--- a/sebs/azure/config.py
+++ b/sebs/azure/config.py
@@ -11,6 +11,7 @@
     AzureResources: Manages Azure resource allocation and lifecycle
     AzureConfig: Combines credentials and resources for Azure deployment
 """
+from __future__ import annotations
 
 import json
 import logging
@@ -444,6 +445,35 @@ def delete_resource_group(self, cli_instance: AzureCLI, name: str, wait: bool =
             self.logging.error(ret.decode())
             raise RuntimeError("Failed to delete the resource group!")
 
+    def delete_storage_account(
+        self, cli_instance: AzureCLI, account: AzureResources.Storage
+    ) -> None:
+        """Delete Azure storage account.
+
+        Args:
+            cli_instance: Azure CLI instance for executing deletion
+            account: Storage account to delete
+
+        Raises:
+            RuntimeError: If deletion fails.
+        """
+
+        storage_account_name = account.account_name
+        try:
+            cli_instance.execute(
+                f"az storage account delete --name {storage_account_name} "
+                f"--resource-group {self._resource_group} --yes"
+            )
+            self.logging.info(f"Storage account {storage_account_name} deleted successfully.")
+
+            # delete the account from our list
+            self._storage_accounts = [
+                acc for acc in self._storage_accounts if acc.account_name != storage_account_name
+            ]
+        except RuntimeError as e:
+            self.logging.error(f"Failed to delete the storage account {storage_account_name}!")
+            self.logging.error(f"Error: {e}")
+
     def cosmosdb_account(self, cli_instance: AzureCLI) -> CosmosDBAccount:
         """Get or create CosmosDB account for NoSQL storage.
 
@@ -674,8 +704,7 @@ def serialize(self) -> dict:
             Dictionary containing all resource configuration data.
         """
         out = super().serialize()
-        if len(self._storage_accounts) > 0:
-            out["storage_accounts"] = [x.serialize() for x in self._storage_accounts]
+        out["storage_accounts"] = [x.serialize() for x in self._storage_accounts]
         if self._resource_group:
             out["resource_group"] = self._resource_group
         if self._cosmosdb_account:

From f56f85867d3aae86f8410d4ce3f522240ef05f68 Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Fri, 17 Apr 2026 13:41:04 +0200
Subject: [PATCH 53/72] [ci] More langues

---
 .github/workflows/regression.yml | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml
index 9133a856..a3d49105 100644
--- a/.github/workflows/regression.yml
+++ b/.github/workflows/regression.yml
@@ -28,6 +28,18 @@ jobs:
             language: nodejs
             version: "16"
             architecture: "arm64"
+          - platform: aws
+            language: cpp
+            version: "all"
+            architecture: "x64"
+          - platform: aws
+            language: java
+            version: "17"
+            architecture: "x64"
+          - platform: aws
+            language: java
+            version: "17"
+            architecture: "arm64"
           - platform: gcp
             language: python
             version: "3.11"
@@ -36,6 +48,10 @@ jobs:
             language: nodejs
             version: "20"
             architecture: "x64"
+          - platform: gcp
+            language: java
+            version: "17"
+            architecture: "x64"
           - platform: azure
             language: python
             version: "3.11"
@@ -44,6 +60,10 @@ jobs:
             language: nodejs
             version: "20"
             architecture: "x64"
+          - platform: azure
+            language: java
+            version: "17"
+            architecture: "x64"
       fail-fast: false
 
     uses: ./.github/workflows/_regression-job.yml

From 2f6c67e4caf20ae1f46c51d48478ff1ff4d3e2ea Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Fri, 17 Apr 2026 13:56:19 +0200
Subject: [PATCH 54/72] [aws] Update Java build image for arm64

---
 dockerfiles/aws/java/Dockerfile.build | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dockerfiles/aws/java/Dockerfile.build b/dockerfiles/aws/java/Dockerfile.build
index bd977d69..36ffd5dc 100644
--- a/dockerfiles/aws/java/Dockerfile.build
+++ b/dockerfiles/aws/java/Dockerfile.build
@@ -13,7 +13,7 @@ ENV PATH=/opt/maven/bin:$PATH
 ENV GOSU_VERSION 1.14
 # https://github.com/tianon/gosu/releases/tag/1.14
 # key https://keys.openpgp.org/search?q=tianon%40debian.org
-RUN curl -o /usr/local/bin/gosu -SL "https://github.com/tianon/gosu/releases/download/${GOSU_VERSION}/gosu-amd64" \
+RUN curl -o /usr/local/bin/gosu -SL "https://github.com/tianon/gosu/releases/download/${GOSU_VERSION}/gosu-${TARGETARCH}" \
     && chmod +x /usr/local/bin/gosu
 RUN mkdir -p /sebs/
 COPY dockerfiles/java_installer.sh /sebs/installer.sh

From 7c4418bf23bb10a3fdd10b78f7de627b66a1c49a Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Fri, 17 Apr 2026 14:02:01 +0200
Subject: [PATCH 55/72] [system] Add language variants for Java and Cpp

---
 sebs/faas/function.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/sebs/faas/function.py b/sebs/faas/function.py
index 651c1f65..f5602e17 100644
--- a/sebs/faas/function.py
+++ b/sebs/faas/function.py
@@ -553,6 +553,22 @@ class NodeJS(Enum):
         BUN = "bun"
         LLRT = "llrt"
 
+    class Java(Enum):
+        """Java runtime variants.
+        Currently only JDK.
+        """
+
+        DEFAULT = "default"
+
+    class Cpp(Enum):
+        """Cpp runtime variants.
+
+        Currently only one variant,
+        compiled with gcc.
+        """
+
+        DEFAULT = "default"
+
     @classmethod
     def for_language(cls, language: Language, val: str) -> Enum:
         """Deserialize a variant string for the given language."""
@@ -574,6 +590,8 @@ def default(cls, language: Language) -> Enum:
 Variant._LANG_MAP = {
     Language.PYTHON: Variant.Python,
     Language.NODEJS: Variant.NodeJS,
+    Language.JAVA: Variant.Java,
+    Language.CPP: Variant.Cpp,
 }
 
 

From 01529c00b2850dc7d06d38ad00987bb3abc17f8a Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Fri, 17 Apr 2026 15:11:27 +0200
Subject: [PATCH 56/72] [cpp][aws] Support fallback to dependency images from
 previous release

---
 sebs/benchmark.py        |  3 +++
 sebs/cpp_dependencies.py | 31 ++++++++++++++++++++++++++++++-
 2 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/sebs/benchmark.py b/sebs/benchmark.py
index 148fed04..62aed9ba 100644
--- a/sebs/benchmark.py
+++ b/sebs/benchmark.py
@@ -1471,6 +1471,9 @@ def build(
                     self._benchmark_config._cpp_dependencies,
                     dockerfile_template,
                     self._system_config.version(),
+                    previous_version=self._system_config.previous_version(),
+                    docker_client=self._docker_client,
+                    docker_repository=self._system_config.docker_repository(),
                 )
                 dockerfile_path = os.path.join(self._output_dir, "Dockerfile")
                 with open(dockerfile_path, "w") as f:
diff --git a/sebs/cpp_dependencies.py b/sebs/cpp_dependencies.py
index 58673ca0..e4667829 100644
--- a/sebs/cpp_dependencies.py
+++ b/sebs/cpp_dependencies.py
@@ -255,6 +255,9 @@ def generate_dockerfile(
         cpp_dependencies: list[CppDependencies],
         dockerfile_template: str,
         sebs_version: str,
+        previous_version: str | None = None,
+        docker_client=None,
+        docker_repository: str | None = None,
     ) -> str:
         """
         Generate a custom Dockerfile for C++ Lambda functions with selective dependencies.
@@ -263,9 +266,16 @@ def generate_dockerfile(
         1. FROM statements for required dependency images
         2. COPY statements to only copy needed libraries from each dependency.
 
+        Supports version fallback: if a dependency image doesn't exist with the current
+        SeBS version, falls back to the previous major version.
+
         Args:
             cpp_dependencies: List of explicit dependencies from benchmark config
             dockerfile_template: Content of Dockerfile.function template
+            sebs_version: Current SeBS version
+            previous_version: Previous major SeBS version for fallback
+            docker_client: Docker client for checking image availability
+            docker_repository: Docker repository name
 
         Returns:
             Complete Dockerfile content with placeholders replaced
@@ -276,9 +286,28 @@ def generate_dockerfile(
         from_statements = []
         for dep in required_deps:
             config = dep_dict[dep]
+
+            # Determine which version to use (current or previous fallback)
+            version_to_use = sebs_version
+            if docker_client and docker_repository and previous_version:
+                current_image = f"{docker_repository}:{config.docker_img}-{sebs_version}"
+                previous_image = f"{docker_repository}:{config.docker_img}-{previous_version}"
+
+                # Try current version first
+                try:
+                    docker_client.images.get(current_image)
+                except Exception:
+                    # Current version not available, try previous version
+                    try:
+                        docker_client.images.get(previous_image)
+                        version_to_use = previous_version
+                    except Exception:
+                        # Neither version available - use current and let it fail later
+                        pass
+
             # Use the short name (e.g., "sdk") as the stage alias
             from_statements.append(
-                f"FROM ${{BASE_REPOSITORY}}:{config.docker_img}-{sebs_version} as {dep.value}"
+                f"FROM ${{BASE_REPOSITORY}}:{config.docker_img}-{version_to_use} as {dep.value}"
             )
 
         copy_statements = []

From 8cf2d2cce5b8f444a1e150a036920bf8ff3f7232 Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Fri, 17 Apr 2026 15:22:30 +0200
Subject: [PATCH 57/72] [aws] Build multi-platform images for Java

---
 sebs/docker_builder.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sebs/docker_builder.py b/sebs/docker_builder.py
index a2cf1021..1fbb5bf8 100644
--- a/sebs/docker_builder.py
+++ b/sebs/docker_builder.py
@@ -95,7 +95,7 @@ def _should_use_multiplatform(
         Returns:
             True if multi-platform build should be used, False otherwise
         """
-        return system == "aws" and image_type == "build" and language in ["python", "nodejs"]
+        return system == "aws" and image_type == "build" and language in ["python", "nodejs", "java"]
 
     def _execute_multiplatform_build(
         self,

From fca1c94b8101533aa9dc730c4fed2948706a501d Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Fri, 17 Apr 2026 15:54:11 +0200
Subject: [PATCH 58/72] [aws] Fix multi-platform images for Java

---
 dockerfiles/aws/java/Dockerfile.build | 1 +
 1 file changed, 1 insertion(+)

diff --git a/dockerfiles/aws/java/Dockerfile.build b/dockerfiles/aws/java/Dockerfile.build
index 36ffd5dc..b2e1058c 100644
--- a/dockerfiles/aws/java/Dockerfile.build
+++ b/dockerfiles/aws/java/Dockerfile.build
@@ -2,6 +2,7 @@ ARG BASE_IMAGE
 FROM ${BASE_IMAGE}
 ARG VERSION
 ENV JAVA_VERSION=${VERSION}
+ARG TARGETARCH
 
 # useradd, groupmod, build tooling
 RUN yum install -y shadow-utils unzip tar gzip zip

From 436a91a32ee2e4ae3f45fc7bf25bf9bfdefb405d Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Fri, 17 Apr 2026 16:03:20 +0200
Subject: [PATCH 59/72] [cpp] Fall back to previous release image for container
 builds

---
 sebs/benchmark.py        |  1 +
 sebs/cpp_dependencies.py | 45 +++++++++++++++++++++++++++++++++++-----
 sebs/docker_builder.py   |  4 +++-
 3 files changed, 44 insertions(+), 6 deletions(-)

diff --git a/sebs/benchmark.py b/sebs/benchmark.py
index 62aed9ba..3b0fcd29 100644
--- a/sebs/benchmark.py
+++ b/sebs/benchmark.py
@@ -1474,6 +1474,7 @@ def build(
                     previous_version=self._system_config.previous_version(),
                     docker_client=self._docker_client,
                     docker_repository=self._system_config.docker_repository(),
+                    logger=self.logging,
                 )
                 dockerfile_path = os.path.join(self._output_dir, "Dockerfile")
                 with open(dockerfile_path, "w") as f:
diff --git a/sebs/cpp_dependencies.py b/sebs/cpp_dependencies.py
index e4667829..6cdb8cd2 100644
--- a/sebs/cpp_dependencies.py
+++ b/sebs/cpp_dependencies.py
@@ -258,6 +258,7 @@ def generate_dockerfile(
         previous_version: str | None = None,
         docker_client=None,
         docker_repository: str | None = None,
+        logger=None,
     ) -> str:
         """
         Generate a custom Dockerfile for C++ Lambda functions with selective dependencies.
@@ -276,6 +277,7 @@ def generate_dockerfile(
             previous_version: Previous major SeBS version for fallback
             docker_client: Docker client for checking image availability
             docker_repository: Docker repository name
+            logger: Logger instance for logging fallback information
 
         Returns:
             Complete Dockerfile content with placeholders replaced
@@ -290,20 +292,53 @@ def generate_dockerfile(
             # Determine which version to use (current or previous fallback)
             version_to_use = sebs_version
             if docker_client and docker_repository and previous_version:
-                current_image = f"{docker_repository}:{config.docker_img}-{sebs_version}"
-                previous_image = f"{docker_repository}:{config.docker_img}-{previous_version}"
+                current_image_tag = f"{config.docker_img}-{sebs_version}"
+                previous_image_tag = f"{config.docker_img}-{previous_version}"
+                current_image = f"{docker_repository}:{current_image_tag}"
+                previous_image = f"{docker_repository}:{previous_image_tag}"
 
-                # Try current version first
+                # Try current version first (check locally, then pull)
+                current_available = False
                 try:
                     docker_client.images.get(current_image)
+                    current_available = True
                 except Exception:
+                    # Not available locally, try to pull it
+                    try:
+                        docker_client.images.pull(docker_repository, current_image_tag)
+                        current_available = True
+                    except Exception:
+                        pass
+
+                if not current_available:
                     # Current version not available, try previous version
                     try:
                         docker_client.images.get(previous_image)
                         version_to_use = previous_version
+                        if logger:
+                            logger.info(
+                                f"Using previous version {previous_version} for "
+                                f"dependency {dep.value} (current version not available)"
+                            )
                     except Exception:
-                        # Neither version available - use current and let it fail later
-                        pass
+                        # Previous version not local, try to pull it
+                        try:
+                            docker_client.images.pull(docker_repository, previous_image_tag)
+                            version_to_use = previous_version
+                            if logger:
+                                logger.info(
+                                    f"Using previous version {previous_version} for "
+                                    f"dependency {dep.value} (current version not available)"
+                                )
+                        except Exception:
+                            # Neither version available - use current and let build fail
+                            if logger:
+                                logger.warning(
+                                    f"Neither current ({sebs_version}) nor previous "
+                                    f"({previous_version}) version available for "
+                                    f"dependency {dep.value}, build may fail"
+                                )
+                            pass
 
             # Use the short name (e.g., "sdk") as the stage alias
             from_statements.append(
diff --git a/sebs/docker_builder.py b/sebs/docker_builder.py
index 1fbb5bf8..de81fa5b 100644
--- a/sebs/docker_builder.py
+++ b/sebs/docker_builder.py
@@ -95,7 +95,9 @@ def _should_use_multiplatform(
         Returns:
             True if multi-platform build should be used, False otherwise
         """
-        return system == "aws" and image_type == "build" and language in ["python", "nodejs", "java"]
+        return (
+            system == "aws" and image_type == "build" and language in ["python", "nodejs", "java"]
+        )
 
     def _execute_multiplatform_build(
         self,

From 4f006897878d8239b69b8bf1068f38ad77372fb2 Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Fri, 17 Apr 2026 16:18:52 +0200
Subject: [PATCH 60/72] [cpp] More fall back

---
 sebs/benchmark.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 47 insertions(+), 1 deletion(-)

diff --git a/sebs/benchmark.py b/sebs/benchmark.py
index 3b0fcd29..9576713f 100644
--- a/sebs/benchmark.py
+++ b/sebs/benchmark.py
@@ -1450,7 +1450,53 @@ def build(
             assert container_client is not None
 
             repo_name = self._system_config.docker_repository()
-            _, image_name = self.builder_image_name()
+            previous_version_image_name, current_version_image_name = self.builder_image_name()
+
+            # Try current version build image first, fallback to previous version
+            image_name = current_version_image_name
+            current_available = False
+            try:
+                self._docker_client.images.get(f"{repo_name}:{current_version_image_name}")
+                current_available = True
+            except Exception:
+                # Not available locally, try to pull it
+                try:
+                    self.logging.info(
+                        f"Docker pull of build image {repo_name}:{current_version_image_name}"
+                    )
+                    self._docker_client.images.pull(repo_name, current_version_image_name)
+                    current_available = True
+                except Exception:
+                    pass
+
+            if not current_available:
+                # Current version not available, try previous version
+                try:
+                    self._docker_client.images.get(f"{repo_name}:{previous_version_image_name}")
+                    image_name = previous_version_image_name
+                    self.logging.info(
+                        f"Using previous version build image {previous_version_image_name} "
+                        "(current version not available)"
+                    )
+                except Exception:
+                    # Previous version not local, try to pull it
+                    try:
+                        self.logging.info(
+                            f"Docker pull of build image {repo_name}:{previous_version_image_name}"
+                        )
+                        self._docker_client.images.pull(repo_name, previous_version_image_name)
+                        image_name = previous_version_image_name
+                        self.logging.info(
+                            f"Using previous version build image {previous_version_image_name} "
+                            "(current version not available)"
+                        )
+                    except Exception:
+                        # Neither version available - use current and let build fail
+                        self.logging.warning(
+                            f"Neither current ({current_version_image_name}) nor previous "
+                            f"({previous_version_image_name}) version build image available, "
+                            "build may fail"
+                        )
 
             """
                 Generate custom Dockerfile for C++ benchmarks

From 711bc4aa6fc8f26bbb0d44c447dfbcef0c81402d Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Fri, 17 Apr 2026 17:59:54 +0200
Subject: [PATCH 61/72] [azure] Add the check for existing storage accounts

---
 sebs/azure/config.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/sebs/azure/config.py b/sebs/azure/config.py
index 11ec58ed..2edbab2c 100644
--- a/sebs/azure/config.py
+++ b/sebs/azure/config.py
@@ -630,7 +630,7 @@ def _create_storage_account(
         """Internal method to create storage account.
 
         Creates a new Azure storage account with the specified name.
-        This one can be usedboth for data storage and function storage.
+        This one can be used both for data storage and function storage.
         This method does NOT update cache or add to resource collections.
 
         Args:
@@ -640,6 +640,21 @@ def _create_storage_account(
         Returns:
             New Storage instance for the created account.
         """
+
+        resource_group = self.resource_group(cli_instance)
+        ret = cli_instance.execute(
+            f"az storage account list --resource-group {resource_group} "
+            f"--query \"[?starts_with(name,'{account_name}') && location=='{self._region}']\""
+        )
+        print(ret)
+        resp = json.loads(ret)
+        if len(resp) > 0:
+            self.logging.info(f"Using existing storage account {account_name}")
+            """
+                List does not return connection string, so we need to query it separately.
+            """
+            return AzureResources.Storage.from_allocation(account_name, cli_instance)
+
         sku = "Standard_LRS"
         self.logging.info("Starting allocation of storage account {}.".format(account_name))
         cli_instance.execute(

From 094d4e2d53b92da67690307f8315f149e52f4222 Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Fri, 17 Apr 2026 18:01:04 +0200
Subject: [PATCH 62/72] [ci] Try to correctly clean up functions

---
 .github/workflows/_regression-job.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/_regression-job.yml b/.github/workflows/_regression-job.yml
index b70377cf..21a3f63e 100644
--- a/.github/workflows/_regression-job.yml
+++ b/.github/workflows/_regression-job.yml
@@ -86,7 +86,8 @@ jobs:
             --config configs/example.json \
             --deployment ${{ inputs.platform }} \
             --resource-prefix ci \
-            --resource-type functions
+            --resource-type functions \
+            --cache regression-cache
 
       - name: Generate test summary
         if: always()

From 643f5c08e52999414e9d84cff83cd02c493f0664 Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Fri, 17 Apr 2026 18:05:49 +0200
Subject: [PATCH 63/72] [ci] Dump all the caches - we should retrieve cloud
 resources on every attempt

---
 .github/workflows/_regression-job.yml | 16 +---------------
 1 file changed, 1 insertion(+), 15 deletions(-)

diff --git a/.github/workflows/_regression-job.yml b/.github/workflows/_regression-job.yml
index 21a3f63e..69d8fd2d 100644
--- a/.github/workflows/_regression-job.yml
+++ b/.github/workflows/_regression-job.yml
@@ -29,14 +29,6 @@ jobs:
         with:
           submodules: recursive
 
-      - name: Restore SeBS cache
-        uses: actions/cache/restore@v4
-        with:
-          path: regression-cache/
-          key: sebs-cache-${{ github.ref_name }}-${{ inputs.platform }}-${{ inputs.language }}-${{ inputs.version }}
-          restore-keys: |
-            sebs-cache-${{ github.ref_name }}-
-
       - name: Setup GCP credentials
         if: inputs.platform == 'gcp'
         uses: google-github-actions/auth@v2
@@ -134,12 +126,6 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: cache-snapshot-${{ inputs.platform }}-${{ inputs.language }}-${{ inputs.version }}
-          path: cache/
+          path: regression-cache/
           if-no-files-found: ignore
 
-      - name: Save SeBS cache
-        if: success()
-        uses: actions/cache/save@v4
-        with:
-          path: regression-cache/
-          key: sebs-cache-${{ github.ref_name }}-${{ inputs.platform }}-${{ inputs.language }}-${{ inputs.version }}

From 9e4518b9b06181e7eb2b9f62ff8a920fcd1b08cf Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Fri, 17 Apr 2026 18:09:03 +0200
Subject: [PATCH 64/72] [azure] Additional exception handling for malformed CLI
 output

---
 sebs/azure/config.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/sebs/azure/config.py b/sebs/azure/config.py
index 2edbab2c..d7229f47 100644
--- a/sebs/azure/config.py
+++ b/sebs/azure/config.py
@@ -646,14 +646,16 @@ def _create_storage_account(
             f"az storage account list --resource-group {resource_group} "
             f"--query \"[?starts_with(name,'{account_name}') && location=='{self._region}']\""
         )
-        print(ret)
-        resp = json.loads(ret)
-        if len(resp) > 0:
-            self.logging.info(f"Using existing storage account {account_name}")
-            """
-                List does not return connection string, so we need to query it separately.
-            """
-            return AzureResources.Storage.from_allocation(account_name, cli_instance)
+        try:
+            resp = json.loads(ret)
+            if len(resp) > 0:
+                self.logging.info(f"Using existing storage account {account_name}")
+                """
+                    List does not return connection string, so we need to query it separately.
+                """
+                return AzureResources.Storage.from_allocation(account_name, cli_instance)
+        except:
+            pass
 
         sku = "Standard_LRS"
         self.logging.info("Starting allocation of storage account {}.".format(account_name))

From dc51bbdbe3ff0d0a7c83cb7940c28b9bc72a46e4 Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Fri, 17 Apr 2026 18:11:42 +0200
Subject: [PATCH 65/72] [dev] Linting

---
 sebs/azure/config.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sebs/azure/config.py b/sebs/azure/config.py
index d7229f47..2c5fb5e6 100644
--- a/sebs/azure/config.py
+++ b/sebs/azure/config.py
@@ -654,7 +654,7 @@ def _create_storage_account(
                     List does not return connection string, so we need to query it separately.
                 """
                 return AzureResources.Storage.from_allocation(account_name, cli_instance)
-        except:
+        except json.JSONDecodeError:
             pass
 
         sku = "Standard_LRS"

From f841cc0d7152d6d54097ed5a682c8298ff6e862e Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Fri, 17 Apr 2026 22:55:50 +0200
Subject: [PATCH 66/72] [aws] Additional handling for partially broken cold
 starts.

---
 sebs/aws/aws.py      | 14 ++++++++++++--
 sebs/aws/config.py   |  1 +
 sebs/aws/triggers.py | 11 ++++++++++-
 3 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/sebs/aws/aws.py b/sebs/aws/aws.py
index ca63e9ef..d0ad1721 100644
--- a/sebs/aws/aws.py
+++ b/sebs/aws/aws.py
@@ -645,7 +645,7 @@ def delete_function(self, func_name: str) -> None:
     @staticmethod
     def parse_aws_report(
         log: str, requests: Union[ExecutionResult, Dict[str, ExecutionResult]]
-    ) -> str:
+    ) -> str | None:
         """Parse AWS Lambda execution report from CloudWatch logs.
 
         Extracts execution metrics from AWS Lambda log entries and updates
@@ -671,8 +671,11 @@ def parse_aws_report(
                 aws_vals[split[0]] = split[1].split()[0]
         if "START RequestId" in aws_vals:
             request_id = aws_vals["START RequestId"]
-        else:
+        elif "REPORT RequestId" in aws_vals:
             request_id = aws_vals["REPORT RequestId"]
+        else:
+            return None
+
         if isinstance(requests, ExecutionResult):
             output = cast(ExecutionResult, requests)
         else:
@@ -857,6 +860,13 @@ def download_metrics(
             for result_part in val:
                 if result_part["field"] == "@message":
                     request_id = AWS.parse_aws_report(result_part["value"], requests)
+
+                    if request_id is None:
+                        self.logging.error(
+                            "Request incomplete, cannot identify ID! "
+                            f"Request: {result_part['value']}"
+                        )
+
                     if request_id in requests:
                         results_processed += 1
                         requests_ids.remove(request_id)
diff --git a/sebs/aws/config.py b/sebs/aws/config.py
index 83230641..590dc1c4 100644
--- a/sebs/aws/config.py
+++ b/sebs/aws/config.py
@@ -756,6 +756,7 @@ def function_url(
                     Action="lambda:InvokeFunctionUrl",
                     Principal="*",
                     FunctionUrlAuthType="NONE",
+                    InvokedViaFunctionUrl=True,
                 )
                 self.logging.info(
                     f"Applied public access permission for Function URL on {func.name}"
diff --git a/sebs/aws/triggers.py b/sebs/aws/triggers.py
index 301f3606..bfc66e31 100644
--- a/sebs/aws/triggers.py
+++ b/sebs/aws/triggers.py
@@ -129,7 +129,16 @@ def sync_invoke(self, payload: dict) -> ExecutionResult:
         function_output = json.loads(ret["Payload"].read().decode("utf-8"))
 
         # AWS-specific parsing
-        AWS.parse_aws_report(log.decode("utf-8"), aws_result)
+        req_id = AWS.parse_aws_report(log.decode("utf-8"), aws_result)
+        if not req_id:
+            """
+            This problem sometimes happens on very long cold starts - the execution
+            works but AWS returns too early.
+            """
+            self.logging.error(
+                f"Unexpected AWS log format! Missing RequestID. Log: {log.decode('utf-8')}"
+            )
+
         # General benchmark output parsing
         # For some reason, the body is dict for NodeJS but a serialized JSON for Python
         if isinstance(function_output["body"], dict):

From fb61c87fdf0cda039d7bace76397099f72e8b955 Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Fri, 17 Apr 2026 23:07:27 +0200
Subject: [PATCH 67/72] [azure] Extra debugging

---
 sebs/azure/azure.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/sebs/azure/azure.py b/sebs/azure/azure.py
index c3fb7fbf..9344b089 100644
--- a/sebs/azure/azure.py
+++ b/sebs/azure/azure.py
@@ -372,6 +372,9 @@ def publish_function(
                         function.name, self.AZURE_RUNTIMES[code_package.language_name]
                     )
                 )
+                self.logging.error(
+                    f"Function app publish of {function.name}, ret {ret.decode('utf-8')}"
+                )
                 url = ""
                 ret_str = ret.decode("utf-8")
                 for line in ret_str.split("\n"):
@@ -393,6 +396,9 @@ def publish_function(
                         "az functionapp function show --function-name handler "
                         f"--name {function.name} --resource-group {resource_group}"
                     )
+                    self.logging.error(
+                        f"Function query for {function.name}! Return {ret.decode('utf-8')}"
+                    )
                     try:
                         url = json.loads(ret.decode("utf-8"))["invokeUrlTemplate"]
                     except json.decoder.JSONDecodeError:
@@ -403,6 +409,7 @@ def publish_function(
                 success = True
             except RuntimeError as e:
                 error = str(e)
+                self.logging.error(f"Failed to publish function {function.name}! Error {error}")
                 # app not found
                 # Azure changed the description as some point
                 if ("find app with name" in error or "NotFound" in error) and repeat_on_failure:
@@ -414,6 +421,16 @@ def publish_function(
                             function.name
                         )
                     )
+                elif ("TooManyRequests" in error) and repeat_on_failure:
+                    """
+                    One error can be "Error calling sync triggers (TooManyRequests)".
+                    """
+                    time.sleep(10)
+                    self.logging.info(
+                        "Sleep 10 seconds for Azure due too many requests for function {}".format(
+                            function.name
+                        )
+                    )
                 # escape loop. we failed!
                 else:
                     raise e
@@ -746,7 +763,7 @@ def create_function(
             while True:
                 try:
                     # create function app
-                    self.cli_instance.execute(
+                    ret = self.cli_instance.execute(
                         (
                             " az functionapp create --resource-group {resource_group} "
                             " --os-type Linux --consumption-plan-location {region} "
@@ -755,6 +772,7 @@ def create_function(
                             " --functions-version 4 "
                         ).format(**config)
                     )
+                    self.logging.error(f"Function app {func_name}, ret {ret.decode('utf-8')}")
                     self.logging.info("Azure: Created function app {}".format(func_name))
                     break
                 except RuntimeError as e:

From 9d18d6bcb8b116ce19c668638352569b334f6a40 Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Fri, 17 Apr 2026 23:12:14 +0200
Subject: [PATCH 68/72] [gcp] First attempt to introduce back-off for all GCP
 ops

---
 sebs/gcp/gcp.py      | 116 ++++++++++++++++++++++++++++++++-----------
 sebs/gcp/triggers.py |   2 +-
 2 files changed, 88 insertions(+), 30 deletions(-)

diff --git a/sebs/gcp/gcp.py b/sebs/gcp/gcp.py
index 8773039d..71eabf7a 100644
--- a/sebs/gcp/gcp.py
+++ b/sebs/gcp/gcp.py
@@ -28,6 +28,7 @@
 import docker
 import os
 import logging
+import random
 import re
 import shutil
 import time
@@ -165,6 +166,76 @@ def get_function_client(self):
         """
         return self.function_client
 
+    def _execute_with_retry(
+        self,
+        request,
+        max_retries: int = 5,
+        base_delay: float = 1.0,
+        max_delay: float = 32.0,
+    ) -> Dict:
+        """Execute a googleapiclient request with retry logic for transient errors.
+
+        Handles transient HTTP errors (503, 429) by retrying with exponential backoff
+        and jitter. Non-transient errors are raised immediately without retry.
+
+        Args:
+            request: googleapiclient request object to execute
+            max_retries: Maximum number of retry attempts (default: 5)
+            base_delay: Base delay in seconds for exponential backoff (default: 1.0)
+            max_delay: Maximum delay between retries in seconds (default: 32.0)
+
+        Returns:
+            Response dictionary from the API call
+
+        Raises:
+            HttpError: If the request fails with a non-transient error or after
+                      exhausting all retry attempts
+        """
+        attempt = 0
+        last_error = None
+
+        while attempt <= max_retries:
+            try:
+                result = request.execute()
+                if attempt > 0:
+                    self.logging.info(f"Request succeeded after {attempt} retries")
+                return result
+            except HttpError as e:
+                status_code = e.resp.status
+                last_error = e
+
+                # Only retry on transient errors
+                if status_code not in GCP.TRANSIENT_HTTP_CODES:
+                    raise
+
+                # Check if we have retries left
+                if attempt >= max_retries:
+                    self.logging.error(
+                        f"Max retries ({max_retries}) exhausted, failing with status {status_code}"
+                    )
+                    raise
+
+                # Calculate delay with exponential backoff and jitter
+                delay = min(base_delay * (2**attempt) + random.uniform(0, 1), max_delay)
+
+                if attempt == 0:
+                    self.logging.warning(
+                        f"Transient error {status_code}, retrying "
+                        f"(attempt {attempt + 1}/{max_retries})"
+                    )
+                else:
+                    self.logging.info(
+                        f"Retry {attempt + 1}/{max_retries} after {delay:.1f}s backoff"
+                    )
+
+                time.sleep(delay)
+                attempt += 1
+
+        # This should not be reached, but just in case
+        if last_error:
+            raise last_error
+        raise RuntimeError("Unexpected state in retry logic")
+
     def default_function_name(
         self, code_package: Benchmark, resources: Optional[Resources] = None
     ) -> str:
@@ -287,7 +358,7 @@ def _wait_for_build_and_poll(
             get_req = (
                 self.function_client.projects().locations().functions().get(name=full_func_name)
             )
-            func_details = get_req.execute()
+            func_details = self._execute_with_retry(get_req)
             if "buildId" in func_details:
                 previous_build_id = func_details["buildId"]
         except HttpError:
@@ -308,7 +379,7 @@ def _wait_for_build_and_poll(
                 get_req = (
                     self.function_client.projects().locations().functions().get(name=full_func_name)
                 )
-                func_details = get_req.execute()
+                func_details = self._execute_with_retry(get_req)
 
                 # Check if there's a new build in progress
                 if "buildId" in func_details:
@@ -373,23 +444,10 @@ def _wait_for_active_status(
                     f"after {elapsed:.0f}s. Last status: {last_status}"
                 )
 
-            try:
-                get_req = (
-                    self.function_client.projects().locations().functions().get(name=full_func_name)
-                )
-                func_details = get_req.execute()
-            except HttpError as e:
-
-                status_code = e.resp.status
-                if status_code in GCP.TRANSIENT_HTTP_CODES:
-                    self.logging.warning(
-                        f"Transient error {status_code} while polling {func_name}, "
-                        f"retrying ({elapsed:.0f}s elapsed)"
-                    )
-                    time.sleep(5)  # back off a bit more for 5xx
-                    continue
-                # 404 past grace window, 403, 400, etc. — real error.
-                raise
+            get_req = (
+                self.function_client.projects().locations().functions().get(name=full_func_name)
+            )
+            func_details = self._execute_with_retry(get_req)
 
             status = func_details["status"]
             current_version = int(func_details["versionId"])
@@ -541,7 +599,7 @@ def _allow_public_access(self, func_name: str, full_func_name: str) -> None:
             )
         )
         try:
-            allow_unauthenticated_req.execute()
+            self._execute_with_retry(allow_unauthenticated_req)
         except HttpError as e:
             raise RuntimeError(
                 f"Failed to configure function {full_func_name} "
@@ -607,7 +665,7 @@ def create_function(
         get_req = self.function_client.projects().locations().functions().get(name=full_func_name)
 
         try:
-            get_req.execute()
+            self._execute_with_retry(get_req)
         except HttpError:
 
             envs = self._generate_function_envs(code_package)
@@ -637,7 +695,7 @@ def create_function(
                     },
                 )
             )
-            create_req.execute()
+            self._execute_with_retry(create_req)
             self.logging.info(
                 f"Function {func_name} is creating - GCP build&deployment is started!"
             )
@@ -707,7 +765,7 @@ def create_trigger(self, function: Function, trigger_type: Trigger.TriggerType)
             get_req = (
                 self.function_client.projects().locations().functions().get(name=full_func_name)
             )
-            func_details = get_req.execute()
+            func_details = self._execute_with_retry(get_req)
             invoke_url = func_details["httpsTrigger"]["url"]
             self.logging.info(f"Function {function.name} - HTTP trigger ready at {invoke_url}")
 
@@ -808,7 +866,7 @@ def update_function(
                 },
             )
         )
-        res = req.execute()
+        res = self._execute_with_retry(req)
 
         self.logging.info(f"Function {function.name} code update initiated")
 
@@ -842,7 +900,7 @@ def _update_envs(self, full_function_name: str, envs: Dict) -> Dict:
         get_req = (
             self.function_client.projects().locations().functions().get(name=full_function_name)
         )
-        response = get_req.execute()
+        response = self._execute_with_retry(get_req)
 
         # preserve old variables while adding new ones.
         # but for conflict, we select the new one
@@ -943,7 +1001,7 @@ def update_function_configuration(
                 )
             )
 
-        res = req.execute()
+        res = self._execute_with_retry(req)
         expected_version = int(res["metadata"]["versionId"])
 
         self.logging.info(f"Function {function.name} configuration update initiated")
@@ -971,7 +1029,7 @@ def delete_function(self, func_name: str) -> None:
             delete_req = (
                 self.function_client.projects().locations().functions().delete(name=full_func_name)
             )
-            delete_req.execute()
+            self._execute_with_retry(delete_req)
             self.logging.info(f"Function {func_name} deleted successfully")
         except HttpError as e:
             if e.resp.status == 404:
@@ -1256,7 +1314,7 @@ def is_deployed(self, func_name: str, versionId: int = -1) -> Tuple[bool, int]:
         name = GCP.get_full_function_name(self.config.project_name, self.config.region, func_name)
         function_client = self.get_function_client()
         status_req = function_client.projects().locations().functions().get(name=name)
-        status_res = status_req.execute()
+        status_res = self._execute_with_retry(status_req)
         if versionId == -1:
             return (status_res["status"] == "ACTIVE", status_res["versionId"])
         else:
@@ -1274,7 +1332,7 @@ def deployment_version(self, func: Function) -> int:
         name = GCP.get_full_function_name(self.config.project_name, self.config.region, func.name)
         function_client = self.get_function_client()
         status_req = function_client.projects().locations().functions().get(name=name)
-        status_res = status_req.execute()
+        status_res = self._execute_with_retry(status_req)
         return int(status_res["versionId"])
 
     @staticmethod
diff --git a/sebs/gcp/triggers.py b/sebs/gcp/triggers.py
index 5a855166..2eaaba41 100644
--- a/sebs/gcp/triggers.py
+++ b/sebs/gcp/triggers.py
@@ -129,7 +129,7 @@ def sync_invoke(self, payload: Dict) -> ExecutionResult:
             .call(name=full_func_name, body={"data": json.dumps(payload)})
         )
         begin = datetime.datetime.now()
-        res = req.execute()
+        res = self.deployment_client._execute_with_retry(req)
         end = datetime.datetime.now()
 
         gcp_result = ExecutionResult.from_times(begin, end)

From c4f1a747a9d6a17df3433c4228308db90ee2e644 Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Fri, 17 Apr 2026 23:41:14 +0200
Subject: [PATCH 69/72] [aws] Fix wrong parameter

---
 sebs/aws/config.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/sebs/aws/config.py b/sebs/aws/config.py
index 590dc1c4..4f0f5028 100644
--- a/sebs/aws/config.py
+++ b/sebs/aws/config.py
@@ -755,8 +755,7 @@ def function_url(
                     StatementId="FunctionURLAllowPublicAccess",
                     Action="lambda:InvokeFunctionUrl",
                     Principal="*",
-                    FunctionUrlAuthType="NONE",
-                    InvokedViaFunctionUrl=True,
+                    FunctionUrlAuthType="NONE"
                 )
                 self.logging.info(
                     f"Applied public access permission for Function URL on {func.name}"

From 6b5804c164015bf9b5eb88b89bace250f43a4de0 Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Sat, 18 Apr 2026 00:12:29 +0200
Subject: [PATCH 70/72] [dev] Linting

---
 sebs/aws/config.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sebs/aws/config.py b/sebs/aws/config.py
index 4f0f5028..83230641 100644
--- a/sebs/aws/config.py
+++ b/sebs/aws/config.py
@@ -755,7 +755,7 @@ def function_url(
                     StatementId="FunctionURLAllowPublicAccess",
                     Action="lambda:InvokeFunctionUrl",
                     Principal="*",
-                    FunctionUrlAuthType="NONE"
+                    FunctionUrlAuthType="NONE",
                 )
                 self.logging.info(
                     f"Applied public access permission for Function URL on {func.name}"

From 75cb23519541d771c3138ff981bcd95281e25728 Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Sat, 18 Apr 2026 00:13:11 +0200
Subject: [PATCH 71/72] [gcp] Additional checks for propagation of triggers

---
 sebs/gcp/gcp.py | 79 ++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 78 insertions(+), 1 deletion(-)

diff --git a/sebs/gcp/gcp.py b/sebs/gcp/gcp.py
index 71eabf7a..7daa9ccb 100644
--- a/sebs/gcp/gcp.py
+++ b/sebs/gcp/gcp.py
@@ -30,6 +30,7 @@
 import logging
 import random
 import re
+import requests
 import shutil
 import time
 import math
@@ -236,6 +237,79 @@ def _execute_with_retry(
             raise last_error
         raise RuntimeError("Unexpected state in retry logic")
 
+    def _wait_for_trigger_propagation(
+        self,
+        trigger_url: str,
+        func_name: str,
+        max_retries: int = 10,
+        base_delay: float = 2.0,
+        max_delay: float = 60.0,
+    ) -> None:
+        """Wait for HTTP trigger URL to be propagated and accessible.
+
+        After creating a trigger, GCP may return the URL before it's fully
+        propagated and accessible. This method polls the URL until it responds
+        with a valid function response (415 with function-execution-id header)
+        or times out.
+
+        Args:
+            trigger_url: HTTP trigger URL to check
+            func_name: Function name for logging purposes
+            max_retries: Maximum number of retry attempts (default: 10)
+            base_delay: Base delay in seconds for exponential backoff (default: 2.0)
+            max_delay: Maximum delay between retries in seconds (default: 60.0)
+
+        Raises:
+            RuntimeError: If trigger URL is not accessible after max retries
+        """
+        attempt = 0
+        self.logging.info(f"Verifying trigger URL propagation for {func_name}")
+
+        while attempt <= max_retries:
+            try:
+                # Use HEAD request to check if function endpoint is ready
+                # A ready function returns 415 (Unsupported Media Type) for HEAD requests
+                # and includes "function-execution-id" in response headers
+                response = requests.head(trigger_url, timeout=10)
+
+                # Check for the expected response indicating function is ready
+                if response.status_code == 415 and "function-execution-id" in response.headers:
+                    self.logging.info(
+                        f"Trigger URL for {func_name} is accessible and ready "
+                        f"(verified after {attempt} attempts)"
+                    )
+                    return
+
+                # Function exists but may not be fully ready yet
+                if response.status_code != 404:
+                    self.logging.debug(
+                        f"Trigger URL returned status {response.status_code}, "
+                        f"waiting for 415 with function-execution-id header"
+                    )
+
+            except requests.exceptions.RequestException as e:
+                self.logging.debug(f"Request to trigger URL failed: {e}")
+
+            # Check if we've exhausted retries
+            if attempt >= max_retries:
+                self.logging.error(
+                    f"Trigger URL for {func_name} not accessible after {max_retries} retries"
+                )
+                raise RuntimeError(
+                    f"Trigger URL {trigger_url} not propagated after {max_retries} attempts"
+                )
+
+            # Calculate delay with exponential backoff and jitter
+            delay = min(base_delay * (2**attempt) + random.uniform(0, 1), max_delay)
+
+            self.logging.info(
+                f"Trigger URL not yet ready for {func_name}, "
+                f"retrying in {delay:.1f}s (attempt {attempt + 1}/{max_retries})"
+            )
+
+            time.sleep(delay)
+            attempt += 1
+
     def default_function_name(
         self, code_package: Benchmark, resources: Optional[Resources] = None
     ) -> str:
@@ -767,7 +841,10 @@ def create_trigger(self, function: Function, trigger_type: Trigger.TriggerType)
             )
             func_details = self._execute_with_retry(get_req)
             invoke_url = func_details["httpsTrigger"]["url"]
-            self.logging.info(f"Function {function.name} - HTTP trigger ready at {invoke_url}")
+            self.logging.info(f"Function {function.name} - HTTP trigger URL: {invoke_url}")
+
+            # Wait for trigger URL to be propagated and accessible
+            self._wait_for_trigger_propagation(invoke_url, function.name)
 
             trigger = HTTPTrigger(invoke_url)
         else:

From 5a7d2ce9cb8a6eedffdaa20f9b5cc71bc02c0830 Mon Sep 17 00:00:00 2001
From: Marcin Copik <mcopik@gmail.com>
Date: Sat, 18 Apr 2026 00:20:05 +0200
Subject: [PATCH 72/72] [azure] retries for azure publish

---
 sebs/azure/azure.py | 207 +++++++++++++++++++++++++++++---------------
 1 file changed, 135 insertions(+), 72 deletions(-)

diff --git a/sebs/azure/azure.py b/sebs/azure/azure.py
index 9344b089..dd3803bb 100644
--- a/sebs/azure/azure.py
+++ b/sebs/azure/azure.py
@@ -33,6 +33,7 @@
 
 import datetime
 import json
+import random
 import re
 import os
 import shutil
@@ -335,6 +336,91 @@ def package_code(
         execute("zip -qu -r9 {}.zip * .".format(benchmark), shell=True, cwd=directory)
         return directory, code_size
 
+    def _execute_cli_with_retry(
+        self,
+        cmd: str,
+        max_retries: int = 5,
+        base_delay: float = 1.0,
+        max_delay: float = 32.0,
+        retryable_errors: Optional[Set[str]] = None,
+    ) -> bytes:
+        """Execute Azure CLI command with retry logic for transient errors.
+
+        Handles transient CLI errors by retrying with exponential backoff
+        and jitter. Specific error patterns can be configured for retry.
+
+        Args:
+            cmd: Azure CLI command to execute
+            max_retries: Maximum number of retry attempts (default: 5)
+            base_delay: Base delay in seconds for exponential backoff (default: 1.0)
+            max_delay: Maximum delay between retries in seconds (default: 32.0)
+            retryable_errors: Set of error patterns to trigger retries
+                             (default: NotFound, TooManyRequests, find app with name)
+
+        Returns:
+            Command output as bytes
+
+        Raises:
+            RuntimeError: If the command fails with a non-retryable error or after
+                         exhausting all retry attempts
+        """
+        if retryable_errors is None:
+            retryable_errors = {
+                "NotFound",
+                "TooManyRequests",
+                "find app with name",
+                "ServiceUnavailable",
+                "InternalServerError",
+            }
+
+        attempt = 0
+        last_error = None
+
+        while attempt <= max_retries:
+            try:
+                result = self.cli_instance.execute(cmd)
+                if attempt > 0:
+                    self.logging.info(f"CLI command succeeded after {attempt} retries")
+                return result
+            except RuntimeError as e:
+                error_message = str(e)
+                last_error = e
+
+                # Check if error is retryable
+                is_retryable = any(pattern in error_message for pattern in retryable_errors)
+
+                if not is_retryable:
+                    raise
+
+                # Check if we have retries left
+                if attempt >= max_retries:
+                    self.logging.error(
+                        f"Max retries ({max_retries}) exhausted for CLI command, "
+                        f"failing with error: {error_message}"
+                    )
+                    raise
+
+                # Calculate delay with exponential backoff and jitter
+                delay = min(base_delay * (2**attempt) + random.uniform(0, 1), max_delay)
+
+                if attempt == 0:
+                    self.logging.warning(
+                        f"Transient CLI error, retrying (attempt {attempt + 1}/{max_retries}): "
+                        f"{error_message[:100]}"
+                    )
+                else:
+                    self.logging.info(
+                        f"Retry {attempt + 1}/{max_retries} after {delay:.1f}s backoff"
+                    )
+
+                time.sleep(delay)
+                attempt += 1
+
+        # This should not be reached, but just in case
+        if last_error:
+            raise last_error
+        raise RuntimeError("Unexpected state in retry logic")
+
     def publish_function(
         self,
         function: Function,
@@ -345,9 +431,9 @@ def publish_function(
         """Publish function code to Azure Functions.
 
         Deploys the packaged function code to Azure Functions using the
-        Azure Functions CLI tools. Handles retries and URL extraction.
-        Will repeat on failure, which is useful to handle delays in
-        Azure cache updates - it can take between 30 and 60 seconds.
+        Azure Functions CLI tools. Handles retries with exponential backoff
+        and jitter for transient errors. This is useful to handle delays in
+        Azure cache updates and service availability issues.
 
         Args:
             function: Function instance to publish
@@ -361,79 +447,56 @@ def publish_function(
         Raises:
             RuntimeError: If function publication fails or URL cannot be found.
         """
-        success = False
-        url = ""
         self.logging.info("Attempting publish of function {}".format(function.name))
-        while not success:
-            try:
-                ret = self.cli_instance.execute(
-                    f"bash -c 'cd {container_dest} "
-                    "&& func azure functionapp publish {} --{} --no-build'".format(
-                        function.name, self.AZURE_RUNTIMES[code_package.language_name]
-                    )
-                )
-                self.logging.error(
-                    f"Function app publish of {function.name}, ret {ret.decode('utf-8')}"
-                )
-                url = ""
-                ret_str = ret.decode("utf-8")
-                for line in ret_str.split("\n"):
-                    if "Invoke url:" in line:
-                        url = line.split("Invoke url:")[1].strip()
-                        break
-
-                # We failed to find the URL the normal way
-                # Sometimes, the output does not include functions.
-                if url == "":
-                    self.logging.warning(
-                        "Couldnt find function URL in the output: {}".format(ret.decode("utf-8"))
-                    )
 
-                    self.logging.info("Sleeping 30 seconds before attempting another query.")
+        publish_cmd = (
+            f"bash -c 'cd {container_dest} "
+            "&& func azure functionapp publish {} --{} --no-build'".format(
+                function.name, self.AZURE_RUNTIMES[code_package.language_name]
+            )
+        )
 
-                    resource_group = self.config.resources.resource_group(self.cli_instance)
-                    ret = self.cli_instance.execute(
-                        "az functionapp function show --function-name handler "
-                        f"--name {function.name} --resource-group {resource_group}"
-                    )
-                    self.logging.error(
-                        f"Function query for {function.name}! Return {ret.decode('utf-8')}"
-                    )
-                    try:
-                        url = json.loads(ret.decode("utf-8"))["invokeUrlTemplate"]
-                    except json.decoder.JSONDecodeError:
-                        raise RuntimeError(
-                            f"Couldn't find the function URL in {ret.decode('utf-8')}"
-                        )
+        # Execute publish command with retry if requested
+        if repeat_on_failure:
+            ret = self._execute_cli_with_retry(publish_cmd)
+        else:
+            ret = self.cli_instance.execute(publish_cmd)
+
+        self.logging.info(f"Function app publish of {function.name}, ret {ret.decode('utf-8')}")
+
+        # Extract URL from publish output
+        url = ""
+        ret_str = ret.decode("utf-8")
+        for line in ret_str.split("\n"):
+            if "Invoke url:" in line:
+                url = line.split("Invoke url:")[1].strip()
+                break
+
+        # Fallback: query function details if URL not found in publish output
+        if url == "":
+            self.logging.warning(
+                "Couldn't find function URL in the publish output: {}".format(ret.decode("utf-8"))
+            )
+            self.logging.info("Querying function details to retrieve URL")
+
+            resource_group = self.config.resources.resource_group(self.cli_instance)
+            query_cmd = (
+                "az functionapp function show --function-name handler "
+                f"--name {function.name} --resource-group {resource_group}"
+            )
+
+            # Use retry for the query as well if repeat_on_failure is enabled
+            if repeat_on_failure:
+                ret = self._execute_cli_with_retry(query_cmd)
+            else:
+                ret = self.cli_instance.execute(query_cmd)
+
+            self.logging.info(f"Function query for {function.name}! Return {ret.decode('utf-8')}")
+            try:
+                url = json.loads(ret.decode("utf-8"))["invokeUrlTemplate"]
+            except json.decoder.JSONDecodeError:
+                raise RuntimeError(f"Couldn't find the function URL in {ret.decode('utf-8')}")
 
-                success = True
-            except RuntimeError as e:
-                error = str(e)
-                self.logging.error(f"Failed to publish function {function.name}! Error {error}")
-                # app not found
-                # Azure changed the description as some point
-                if ("find app with name" in error or "NotFound" in error) and repeat_on_failure:
-                    # Sleep because of problems when publishing immediately
-                    # after creating function app.
-                    time.sleep(30)
-                    self.logging.info(
-                        "Sleep 30 seconds for Azure to register function app {}".format(
-                            function.name
-                        )
-                    )
-                elif ("TooManyRequests" in error) and repeat_on_failure:
-                    """
-                    One error can be "Error calling sync triggers (TooManyRequests)".
-                    """
-                    time.sleep(10)
-                    self.logging.info(
-                        "Sleep 10 seconds for Azure due too many requests for function {}".format(
-                            function.name
-                        )
-                    )
-                # escape loop. we failed!
-                else:
-                    raise e
         return url
 
     def update_function(