Migrate stellar-core Ingress to Gateway API HTTPRoute via a scalable nginx proxy

.github/workflows/build-and-test.yml

@@ -12,17 +12,55 @@ jobs:
     runs-on: ubuntu-24.04
 
     steps:
-    - uses: actions/checkout@v2
-    - name: Setup k3s
-      uses: debianmaster/actions-k3s@master
-      with:
-        version: 'v1.23.17-k3s1'
-    - name: Setup nginx-ingress
+    - uses: actions/checkout@v4
+    - name: Setup k3d cluster (k3s, bundled traefik disabled)
+      run: |
+        set -euo pipefail
+        curl -s https://raw.githubusercontent.com/k3d-io/k3d/v5.9.0/install.sh | bash
+        # Disable k3s's bundled traefik; the gateway controller (envoy) is installed below.
+        # Map host :8081 -> the gateway LoadBalancer :80 so the mission driver can reach it.
+        k3d cluster create ssc \
+          --image rancher/k3s:v1.31.5-k3s1 \
+          --k3s-arg "--disable=traefik@server:*" \
+          -p "8081:80@loadbalancer" \
+          --wait
+        k3d kubeconfig get ssc > "$RUNNER_TEMP/kubeconfig"
+        echo "KUBECONFIG=$RUNNER_TEMP/kubeconfig" >> "$GITHUB_ENV"
+    - name: Setup Envoy Gateway + Gateway
       run: |
-        kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.5.1/deploy/static/provider/cloud/deploy.yaml
-        kubectl wait --namespace ingress-nginx --for=condition=ready pod --selector=app.kubernetes.io/component=controller --timeout=120s
+        set -euo pipefail
+        # Envoy Gateway ships its own matching gateway-api CRDs + controller.
+        helm install eg oci://docker.io/envoyproxy/gateway-helm --version v1.2.6 \
+          -n envoy-gateway-system --create-namespace --wait --timeout 4m
+        # GatewayClass + the Gateway the per-mission HTTPRoutes attach to.
+        # Missions target it via --gateway-name / --gateway-namespace.
+        kubectl create namespace gateway
+        kubectl apply -f - <<'YAML'
+        apiVersion: gateway.networking.k8s.io/v1
+        kind: GatewayClass
+        metadata:
+          name: eg
+        spec:
+          controllerName: gateway.envoyproxy.io/gatewayclass-controller
+        ---
+        apiVersion: gateway.networking.k8s.io/v1
+        kind: Gateway
+        metadata:
+          name: ssc-gateway
+          namespace: gateway
+        spec:
+          gatewayClassName: eg
+          listeners:
+          - name: http
+            port: 80
+            protocol: HTTP
+            allowedRoutes:
+              namespaces:
+                from: All
+        YAML
+        kubectl wait --namespace gateway --for=condition=Programmed gateway/ssc-gateway --timeout=180s
     - name: Setup .NET SDK 8
-      uses: actions/setup-dotnet@v1
+      uses: actions/setup-dotnet@v4
       with:
         dotnet-version: '8.0.x'
     - name: Install dependencies
@@ -36,7 +74,7 @@ jobs:
     - name: Test
       run: dotnet test --no-restore --verbosity normal
     - name: Run BootAndSync mission
-      run: dotnet run --project src/App/App.fsproj --configuration Release -- mission BootAndSync --image stellar/stellar-core:stable --kubeconfig $KUBECONFIG --namespace default --ingress-class nginx --ingress-internal-domain local --ingress-external-host localhost --uneven-sched
+      run: dotnet run --project src/App/App.fsproj --configuration Release -- mission BootAndSync --image stellar/stellar-core:stable --kubeconfig $KUBECONFIG --namespace default --gateway-name ssc-gateway --gateway-namespace gateway --ingress-internal-domain local --ingress-external-host localhost --ingress-external-port 8081 --uneven-sched
     - uses: actions/upload-artifact@v4
       with:
         name: destination

doc/k3s.md

@@ -32,5 +32,5 @@ net.ipv6.neigh.default.gc_thresh3 = 100000
 
   - Build supercluster normally (see [getting-started.md](getting-started.md))
 
-  - Run supercluster with these additional arguments: `--kubeconfig $KUBECONFIG --namespace default --ingress-class nginx --ingress-internal-domain local --ingress-external-host localhost --uneven-sched`
+  - Run supercluster with these additional arguments: `--kubeconfig $KUBECONFIG --namespace default --ingress-internal-domain local --ingress-external-host localhost --uneven-sched`
 

doc/theoretical-max-tps.md

@@ -11,7 +11,7 @@ To run the test, first [set up an EKS cluster](eks.md). Accepting the default
 settings will produce a topology identical to what we use in our test setup.
 Then, run a `MaxTPSClassic` mission with the following template:
 ```bash
-dotnet run --project src/App/App.fsproj --configuration Release -- mission MaxTPSClassic --image=<core-image> --pubnet-data=<path-to-repo>/topologies/theoretical-max-tps.json --tx-rate=<min-tx-rate> --max-tx-rate=<max-tx-rate> --namespace default --ingress-internal-domain=<domain> --ingress-class=nginx --run-for-max-tps=classic --enable-tcp-tuning
+dotnet run --project src/App/App.fsproj --configuration Release -- mission MaxTPSClassic --image=<core-image> --pubnet-data=<path-to-repo>/topologies/theoretical-max-tps.json --tx-rate=<min-tx-rate> --max-tx-rate=<max-tx-rate> --namespace default --ingress-internal-domain=<domain> --run-for-max-tps=classic --enable-tcp-tuning
 ```
 For more information about how to set the parameters in the above command, see
 [Measuring Transaction Throughput](measuring-transaction-throughput.md).

src/App/Program.fs

@@ -42,10 +42,11 @@ type MissionOptions
         logDebugPartitions: seq<string>,
         logTracePartitions: seq<string>,
         namespaceProperty: string option,
-        ingressClass: string,
-        ingressInternalDomain: string,
-        ingressExternalHost: string option,
-        ingressExternalPort: int,
+        gatewayName: string,
+        gatewayNamespace: string,
+        routeInternalDomain: string,
+        routeExternalHost: string option,
+        routeExternalPort: int,
         exportToPrometheus: bool,
         probeTimeout: int,
         missions: string seq,
@@ -70,6 +71,7 @@ type MissionOptions
         avoidNodeLabels: seq<string>,
         tolerateNodeTaints: seq<string>,
         apiRateLimit: int,
+        httpProxyReplicas: int,
         pubnetData: string option,
         flatQuorum: bool option,
         tier1Keys: string option,
@@ -155,28 +157,34 @@ type MissionOptions
     [<Option("namespace", HelpText = "Namespace to use, overriding kubeconfig.", Required = false)>]
     member self.NamespaceProperty = namespaceProperty
 
-    [<Option("ingress-class",
-             HelpText = "Value for kubernetes.io/ingress.class, on ingress",
+    [<Option("gateway-name",
+             HelpText = "Name of the Gateway (gateway.networking.k8s.io) the per-mission HTTPRoute attaches to",
              Required = false,
-             Default = "ingress-private")>]
-    member self.IngressClass = ingressClass
+             Default = "traefik-gateway-private")>]
+    member self.GatewayName = gatewayName
+
+    [<Option("gateway-namespace",
+             HelpText = "Namespace of the Gateway the per-mission HTTPRoute attaches to",
+             Required = false,
+             Default = "traefik")>]
+    member self.GatewayNamespace = gatewayNamespace
 
     [<Option("ingress-internal-domain",
-             HelpText = "Cluster-internal DNS domain in which to configure ingress",
+             HelpText = "Cluster-internal DNS domain used to form the per-mission Gateway API route hostname (flag name kept for compatibility)",
              Required = false,
              Default = "local")>]
-    member self.IngressInternalDomain = ingressInternalDomain
+    member self.RouteInternalDomain = routeInternalDomain
 
     [<Option("ingress-external-host",
-             HelpText = "Cluster-external hostname to connect to for access to ingress",
+             HelpText = "Cluster-external hostname the driver connects to for the gateway route; defaults to the route hostname (flag name kept for compatibility)",
              Required = false)>]
-    member self.IngressExternalHost = ingressExternalHost
+    member self.RouteExternalHost = routeExternalHost
 
     [<Option("ingress-external-port",
-             HelpText = "Cluster-external port to connect to for access to ingress",
+             HelpText = "Cluster-external port the driver connects to for the gateway route (flag name kept for compatibility)",
              Required = false,
              Default = 80)>]
-    member self.IngressExternalPort = ingressExternalPort
+    member self.RouteExternalPort = routeExternalPort
 
     [<Option("export-to-prometheus", HelpText = "Whether to export core metrics to prometheus")>]
     member self.ExportToPrometheus : bool = exportToPrometheus
@@ -295,6 +303,12 @@ type MissionOptions
              Default = 10)>]
     member self.ApiRateLimit = apiRateLimit
 
+    [<Option("http-proxy-replicas",
+             HelpText = "Max nginx HTTP proxy replicas per mission (cap); actual scales with node count, floor 1",
+             Required = false,
+             Default = 10)>]
+    member self.HttpProxyReplicas = httpProxyReplicas
+
     [<Option("pubnet-data", HelpText = "JSON file containing pubnet connectivity graph data", Required = false)>]
     member self.PubnetData = pubnetData
 
@@ -775,10 +789,11 @@ let main argv =
                                numNodes = mission.NumNodes
                                namespaceProperty = ns
                                logLevels = ll
-                               ingressClass = mission.IngressClass
-                               ingressInternalDomain = mission.IngressInternalDomain
-                               ingressExternalHost = mission.IngressExternalHost
-                               ingressExternalPort = mission.IngressExternalPort
+                               gatewayName = mission.GatewayName
+                               gatewayNamespace = mission.GatewayNamespace
+                               routeInternalDomain = mission.RouteInternalDomain
+                               routeExternalHost = mission.RouteExternalHost
+                               routeExternalPort = mission.RouteExternalPort
                                exportToPrometheus = mission.ExportToPrometheus
                                probeTimeout = mission.ProbeTimeout
                                coreResources = SmallTestResources
@@ -788,6 +803,7 @@ let main argv =
                                avoidNodeLabels = List.map splitLabel (List.ofSeq mission.AvoidNodeLabels)
                                tolerateNodeTaints = List.map splitLabel (List.ofSeq mission.TolerateNodeTaints)
                                apiRateLimit = mission.ApiRateLimit
+                               httpProxyReplicas = mission.HttpProxyReplicas
                                pubnetData = mission.PubnetData
                                flatQuorum = mission.FlatQuorum
                                tier1Keys = mission.Tier1Keys

src/CSLibrary/CSLibrary.csproj

@@ -6,7 +6,7 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.Rest.ClientRuntime" Version="2.3.24" />
-    <PackageReference Include="KubernetesClient" Version="15.0.1" />
+    <PackageReference Include="KubernetesClient" Version="17.0.14" />
     <PackageReference Include="Serilog" Version="2.10.0" />
     <PackageReference Include="Serilog.Sinks.Console" Version="3.1.1" />
     <PackageReference Include="stellar-dotnet-sdk" Version="10.0.0" />

src/CSLibrary/GatewayApi.cs

@@ -0,0 +1,179 @@
+// Typed models for the subset of the Gateway API (gateway.networking.k8s.io/v1)
+// that Supercluster needs to route to per-pod core/history endpoints.
+//
+// The official KubernetesClient package ships no Gateway API models (CRDs are
+// out of scope for it), so these POCOs are hand-maintained to the upstream
+// gateway-api v1 schema. Only the fields Supercluster uses are modelled.
+// Usable with GenericClient<HTTPRoute> thanks to the [KubernetesEntity] attribute.
+
+using System.Collections.Generic;
+using System.Text.Json.Serialization;
+using k8s;
+using k8s.Models;
+
+namespace GatewayApiModels
+{
+    [KubernetesEntity(Group = "gateway.networking.k8s.io", ApiVersion = "v1", Kind = "HTTPRoute", PluralName = "httproutes")]
+    public class HTTPRoute : IKubernetesObject<V1ObjectMeta>, ISpec<HTTPRouteSpec>
+    {
+        [JsonPropertyName("apiVersion")]
+        public string ApiVersion { get; set; } = "gateway.networking.k8s.io/v1";
+
+        [JsonPropertyName("kind")]
+        public string Kind { get; set; } = "HTTPRoute";
+
+        [JsonPropertyName("metadata")]
+        public V1ObjectMeta Metadata { get; set; }
+
+        [JsonPropertyName("spec")]
+        public HTTPRouteSpec Spec { get; set; }
+
+        [JsonPropertyName("status")]
+        public HTTPRouteStatus Status { get; set; }
+    }
+
+    // Minimal route status: each parent (Gateway) the route attached to reports
+    // conditions (notably "Accepted"). Used to wait for the gateway to admit the
+    // route before the driver sends traffic.
+    public class HTTPRouteStatus
+    {
+        [JsonPropertyName("parents")]
+        public IList<RouteParentStatus> Parents { get; set; }
+    }
+
+    public class RouteParentStatus
+    {
+        [JsonPropertyName("conditions")]
+        public IList<V1Condition> Conditions { get; set; }
+    }
+
+    [KubernetesEntity(Group = "gateway.networking.k8s.io", ApiVersion = "v1", Kind = "HTTPRouteList", PluralName = "httproutes")]
+    public class HTTPRouteList : IKubernetesObject<V1ListMeta>, IItems<HTTPRoute>
+    {
+        [JsonPropertyName("apiVersion")]
+        public string ApiVersion { get; set; } = "gateway.networking.k8s.io/v1";
+
+        [JsonPropertyName("kind")]
+        public string Kind { get; set; } = "HTTPRouteList";
+
+        [JsonPropertyName("metadata")]
+        public V1ListMeta Metadata { get; set; }
+
+        [JsonPropertyName("items")]
+        public IList<HTTPRoute> Items { get; set; }
+    }
+
+    public class HTTPRouteSpec
+    {
+        [JsonPropertyName("parentRefs")]
+        public IList<ParentReference> ParentRefs { get; set; }
+
+        [JsonPropertyName("hostnames")]
+        public IList<string> Hostnames { get; set; }
+
+        [JsonPropertyName("rules")]
+        public IList<HTTPRouteRule> Rules { get; set; }
+    }
+
+    public class ParentReference
+    {
+        [JsonPropertyName("group")]
+        public string Group { get; set; }
+
+        [JsonPropertyName("kind")]
+        public string Kind { get; set; }
+
+        [JsonPropertyName("namespace")]
+        public string Namespace { get; set; }
+
+        [JsonPropertyName("name")]
+        public string Name { get; set; }
+
+        [JsonPropertyName("sectionName")]
+        public string SectionName { get; set; }
+
+        [JsonPropertyName("port")]
+        public int? Port { get; set; }
+    }
+
+    public class HTTPRouteRule
+    {
+        [JsonPropertyName("matches")]
+        public IList<HTTPRouteMatch> Matches { get; set; }
+
+        [JsonPropertyName("filters")]
+        public IList<HTTPRouteFilter> Filters { get; set; }
+
+        [JsonPropertyName("backendRefs")]
+        public IList<HTTPBackendRef> BackendRefs { get; set; }
+    }
+
+    public class HTTPRouteMatch
+    {
+        [JsonPropertyName("path")]
+        public HTTPPathMatch Path { get; set; }
+    }
+
+    public class HTTPPathMatch
+    {
+        // "Exact" | "PathPrefix" | "RegularExpression"
+        [JsonPropertyName("type")]
+        public string Type { get; set; }
+
+        [JsonPropertyName("value")]
+        public string Value { get; set; }
+    }
+
+    public class HTTPRouteFilter
+    {
+        // "RequestHeaderModifier" | "URLRewrite" | ...
+        [JsonPropertyName("type")]
+        public string Type { get; set; }
+
+        [JsonPropertyName("urlRewrite")]
+        public HTTPURLRewriteFilter UrlRewrite { get; set; }
+    }
+
+    public class HTTPURLRewriteFilter
+    {
+        [JsonPropertyName("hostname")]
+        public string Hostname { get; set; }
+
+        [JsonPropertyName("path")]
+        public HTTPPathModifier Path { get; set; }
+    }
+
+    public class HTTPPathModifier
+    {
+        // "ReplaceFullPath" | "ReplacePrefixMatch"
+        [JsonPropertyName("type")]
+        public string Type { get; set; }
+
+        [JsonPropertyName("replaceFullPath")]
+        public string ReplaceFullPath { get; set; }
+
+        [JsonPropertyName("replacePrefixMatch")]
+        public string ReplacePrefixMatch { get; set; }
+    }
+
+    public class HTTPBackendRef
+    {
+        [JsonPropertyName("group")]
+        public string Group { get; set; }
+
+        [JsonPropertyName("kind")]
+        public string Kind { get; set; }
+
+        [JsonPropertyName("name")]
+        public string Name { get; set; }
+
+        [JsonPropertyName("namespace")]
+        public string Namespace { get; set; }
+
+        [JsonPropertyName("port")]
+        public int? Port { get; set; }
+
+        [JsonPropertyName("weight")]
+        public int? Weight { get; set; }
+    }
+}