diff --git a/docs/contributing/developer-guide.md b/docs/contributing/developer-guide.md index 5bf2f6cf7..b22f0c48f 100644 --- a/docs/contributing/developer-guide.md +++ b/docs/contributing/developer-guide.md @@ -99,6 +99,74 @@ To run e2e test only: make -C tests/e2e # run e2e tests only ``` +## GPU Operator E2E Tests + +The `tests/k8s-e2e/` directory contains an e2e test suite that installs the GPU Operator via Helm and verifies metrics and health. Tests run against a live Kubernetes cluster. + +### Prerequisites + +- A running Kubernetes cluster with at least one AMD GPU node +- `kubectl` configured (`~/.kube/config` or a custom kubeconfig) +- Docker (to build the test runner image) + +### Test runner image + +```bash +docker build -t gpu-op-k8s-e2e:latest -f tests/k8s-e2e/Dockerfile.e2e tests/k8s-e2e/ +``` + +### Running tests + +#### Full install + verify + teardown + +Pass the helm chart as a local directory path (the `helm-charts-k8s/` directory in the repository root) or an OCI/repo reference if publishing to a registry: + +```bash +docker run --rm \ + -v /path/to/kubeconfig:/kubeconfig:ro \ + -v /path/to/gpu-operator/helm-charts-k8s:/helm-charts:ro \ + gpu-op-k8s-e2e:latest \ + -kubeconfig /kubeconfig \ + -operatorchart /helm-charts \ + -operatortag v1.5.0 \ + -test.timeout 60m +``` + +#### Verify only (pre-deployed cluster) + +```bash +docker run --rm -v /path/to/kubeconfig:/kubeconfig:ro \ + gpu-op-k8s-e2e:latest \ + -kubeconfig /kubeconfig -existing \ + -check.f 'TestOp010|TestOp020|TestOp030|TestOp040|TestOp050|TestOp060|TestOp065|TestOp070' \ + -test.timeout 30m +``` + + +#### Using make + +```bash +# Full install+verify+teardown +make -C tests/k8s-e2e all KUBECONFIG=/path/to/kubeconfig OPERATOR_TAG=v1.5.0 + +# Verify only (pre-deployed) +make -C tests/k8s-e2e verify KUBECONFIG=/path/to/kubeconfig +``` + +### Common flags + +| Flag | Default | Description | +| --- | --- | --- | +| `-kubeconfig` | `~/.kube/config` | Path to kubeconfig | +| `-operatorchart` | OCI registry chart | GPU Operator helm chart (OCI ref or local path) | +| `-operatortag` | `v1.4.1` | GPU Operator chart version | +| `-namespace` | `kube-amd-gpu` | Kubernetes namespace | +| `-existing` | `false` | Skip install/teardown — verify only against pre-deployed cluster | +| `-noteardown` | `false` | Skip teardown after tests (leave operator installed) | +| `-helmset` | _(none)_ | Extra helm `--set` override (repeatable) | +| `-check.f` | _(all)_ | Regex filter for test names (gocheck syntax) | +| `-test.timeout` | `30m` | Overall test timeout | + ## Creating a Pull Request 1. Fork the repository on GitHub. diff --git a/tests/k8s-e2e/.gitignore b/tests/k8s-e2e/.gitignore new file mode 100644 index 000000000..48b8bf907 --- /dev/null +++ b/tests/k8s-e2e/.gitignore @@ -0,0 +1 @@ +vendor/ diff --git a/tests/k8s-e2e/Dockerfile.e2e b/tests/k8s-e2e/Dockerfile.e2e new file mode 100644 index 000000000..956148624 --- /dev/null +++ b/tests/k8s-e2e/Dockerfile.e2e @@ -0,0 +1,41 @@ +# Dockerfile.e2e — containerized runner for GPU Operator e2e tests +# +# Build (from gpu-operator repo root): +# docker build -t gpu-op-k8s-e2e:latest -f tests/k8s-e2e/Dockerfile.e2e tests/k8s-e2e/ +# +# Run full install+verify+teardown: +# docker run --rm \ +# -v /path/to/kubeconfig:/kubeconfig:ro \ +# -v /path/to/gpu-operator/helm-charts-k8s:/helm-charts:ro \ +# gpu-op-k8s-e2e:latest \ +# -kubeconfig /kubeconfig \ +# -operatorchart /helm-charts \ +# -operatortag v1.4.1 -test.timeout 60m +# +# Run verify only (pre-deployed cluster): +# docker run --rm -v /path/to/kubeconfig:/kubeconfig:ro \ +# gpu-op-k8s-e2e:latest \ +# -kubeconfig /kubeconfig -existing \ +# -check.f 'TestOp010|TestOp020|TestOp030|TestOp040|TestOp050|TestOp060|TestOp065|TestOp070' \ +# -test.timeout 30m + +FROM golang:1.25-bookworm + +# Install kubectl +RUN curl -fsSL "https://dl.k8s.io/release/$(curl -Ls https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \ + -o /usr/local/bin/kubectl && chmod +x /usr/local/bin/kubectl + +WORKDIR /src + +# Copy module files first for caching, then download deps +COPY go.mod go.sum ./ +RUN go mod download + +# Copy test sources +COPY clients/ clients/ +COPY doc.go suite_test.go operator_test.go ./ + +# Pre-compile tests to catch errors in _test.go files at image build time +RUN go test -run=^$ ./... + +ENTRYPOINT ["go", "test", "-v", "-test.timeout=30m"] diff --git a/tests/k8s-e2e/Makefile b/tests/k8s-e2e/Makefile new file mode 100644 index 000000000..92e11ecd5 --- /dev/null +++ b/tests/k8s-e2e/Makefile @@ -0,0 +1,33 @@ +.DEFAULT: all +.PHONY: all verify lint + +TEST_ARGS := +KUBECONFIG := +OPERATOR_CHART ?= oci://registry-1.docker.io/rocm/gpu-operator-charts +OPERATOR_TAG ?= v1.4.1 +OPERATOR_NS ?= kube-amd-gpu + +ifdef KUBECONFIG + TEST_ARGS += -kubeconfig=$(KUBECONFIG) +endif + +# all: full install + verify + teardown (TestOp000–TestOp900) +all: + go test -failfast \ + -operatorchart $(OPERATOR_CHART) \ + -operatortag $(OPERATOR_TAG) \ + -namespace $(OPERATOR_NS) \ + -test.timeout=60m \ + -v $(TEST_ARGS) + +# verify: verify DME on a PRE-DEPLOYED GPU Operator cluster (no install/teardown) +verify: + go test -failfast -existing \ + -namespace $(OPERATOR_NS) \ + -check.f 'TestOp010|TestOp020|TestOp030|TestOp040|TestOp050|TestOp060|TestOp065|TestOp070' \ + -test.timeout=30m \ + -v $(TEST_ARGS) + +lint: + @go fmt ./... + @go vet ./... diff --git a/tests/k8s-e2e/clients/helm.go b/tests/k8s-e2e/clients/helm.go new file mode 100644 index 000000000..c745c34ff --- /dev/null +++ b/tests/k8s-e2e/clients/helm.go @@ -0,0 +1,197 @@ +/** +# Copyright (c) Advanced Micro Devices, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the \"License\"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package clients + +import ( + "context" + "fmt" + "log" + "os" + "time" + + helm "github.com/mittwald/go-helm-client" + helmValues "github.com/mittwald/go-helm-client/values" + "helm.sh/helm/v3/pkg/repo" + restclient "k8s.io/client-go/rest" +) + +type HelmClientOpt func(client *HelmClient) + +type HelmClient struct { + client helm.Client + cache string + config string + ns string + restConfig *restclient.Config + relName string +} + +func WithNameSpaceOption(namespace string) HelmClientOpt { + return func(c *HelmClient) { + c.ns = namespace + } +} + +func WithKubeConfigOption(kubeconf *restclient.Config) HelmClientOpt { + return func(c *HelmClient) { + c.restConfig = kubeconf + } +} + +func NewHelmClient(opts ...HelmClientOpt) (*HelmClient, error) { + client := &HelmClient{} + for _, opt := range opts { + opt(client) + } + + var err error + client.cache, err = os.MkdirTemp("", ".hcache") + if err != nil { + return nil, err + } + + configDir, err := os.MkdirTemp("", ".hconfig") + if err != nil { + return nil, err + } + // RepositoryConfig must be a file path (repositories.yaml), not a directory. + client.config = configDir + repoFile := configDir + "/repositories.yaml" + restConfOptions := &helm.RestConfClientOptions{ + Options: &helm.Options{ + Namespace: client.ns, + RepositoryConfig: repoFile, + Debug: true, + RepositoryCache: client.cache, + DebugLog: func(format string, v ...interface{}) { + log.Printf(format, v...) + }, + }, + RestConfig: client.restConfig, + } + + helmClient, err := helm.NewClientFromRestConf(restConfOptions) + if err != nil { + return nil, err + } + client.client = helmClient + return client, nil +} + +func (h *HelmClient) InstallChart(ctx context.Context, chart string, params []string) (string, error) { + values := helmValues.Options{ + Values: params, + } + + chartSpec := &helm.ChartSpec{ + ReleaseName: "e2e-test-k8s", + ChartName: chart, + Namespace: h.ns, + GenerateName: false, + Wait: true, + Timeout: 5 * time.Minute, + CleanupOnFail: false, + DryRun: false, + ValuesOptions: values, + } + + resp, err := h.client.InstallChart(ctx, chartSpec, nil) + if err != nil { + return "", err + } + log.Printf("helm chart install resp: %+v", resp) + h.relName = resp.Name + return resp.Name, err +} + +func (h *HelmClient) UninstallChart() error { + if h.relName == "" { + return fmt.Errorf("helm chart is not installed by client") + } + return h.client.UninstallReleaseByName(h.relName) +} + +// AddRepository adds a helm repository. url is the chart repo URL; name is the local alias. +func (h *HelmClient) AddRepository(name, url string) error { + return h.client.AddOrUpdateChartRepo(repo.Entry{ + Name: name, + URL: url, + }) +} + +// InstallChartWithTimeout is like InstallChart but accepts a custom timeout, release name, and +// optional chart version. version may be empty (uses the latest available version). +func (h *HelmClient) InstallChartWithTimeout(ctx context.Context, releaseName, chart, version string, params []string, timeout time.Duration) (string, error) { + values := helmValues.Options{ + Values: params, + } + + chartSpec := &helm.ChartSpec{ + ReleaseName: releaseName, + ChartName: chart, + Version: version, + Namespace: h.ns, + GenerateName: false, + Wait: false, // individual Op010-Op070 tests verify each component's readiness + Timeout: timeout, + CleanupOnFail: false, + DryRun: false, + SkipCRDs: false, + ValuesOptions: values, + } + + resp, err := h.client.InstallChart(ctx, chartSpec, nil) + if err != nil { + return "", err + } + log.Printf("helm chart install resp: %+v", resp) + h.relName = resp.Name + return resp.Name, nil +} + +// UninstallChartByName uninstalls a helm release by name without requiring it was installed by this client. +func (h *HelmClient) UninstallChartByName(releaseName string) error { + return h.client.UninstallReleaseByName(releaseName) +} + +// UninstallAllReleases uninstalls all helm releases in the client's namespace. +// Errors are logged but not returned so cleanup continues regardless. +func (h *HelmClient) UninstallAllReleases() { + releases, err := h.client.ListDeployedReleases() + if err != nil { + log.Printf("UninstallAllReleases: list: %v", err) + return + } + for _, rel := range releases { + log.Printf("UninstallAllReleases: uninstalling %s", rel.Name) + if err := h.client.UninstallReleaseByName(rel.Name); err != nil { + log.Printf("UninstallAllReleases: %s: %v", rel.Name, err) + } + } +} + +func (h *HelmClient) Cleanup() { + err := os.RemoveAll(h.cache) + if err != nil { + log.Printf("failed to delete directory %s; err: %v", h.cache, err) + } + + err = os.RemoveAll(h.config) + if err != nil { + log.Printf("failed to delete directory %s; err: %v", h.config, err) + } +} diff --git a/tests/k8s-e2e/clients/k8s.go b/tests/k8s-e2e/clients/k8s.go new file mode 100644 index 000000000..7960d4041 --- /dev/null +++ b/tests/k8s-e2e/clients/k8s.go @@ -0,0 +1,620 @@ +/** +# Copyright (c) Advanced Micro Devices, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the \"License\"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package clients + +import ( + "bytes" + "context" + "fmt" + "log" + "strings" + "time" + + "github.com/prometheus/common/expfmt" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" + "k8s.io/client-go/tools/remotecommand" + "k8s.io/kubectl/pkg/scheme" +) + +// moduleGVR is the GroupVersionResource for KMM Module CRs. +var moduleGVR = schema.GroupVersionResource{ + Group: "kmm.sigs.x-k8s.io", + Version: "v1beta1", + Resource: "modules", +} + +// deviceconfigGVR is the GroupVersionResource for AMD DeviceConfig CRs. +var deviceconfigGVR = schema.GroupVersionResource{ + Group: "amd.com", + Version: "v1alpha1", + Resource: "deviceconfigs", +} + +type K8sClient struct { + client *kubernetes.Clientset + dynamic dynamic.Interface +} + +func NewK8sClient(config *restclient.Config) (*K8sClient, error) { + k8sc := K8sClient{} + cs, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, err + } + k8sc.client = cs + dc, err := dynamic.NewForConfig(config) + if err != nil { + return nil, err + } + k8sc.dynamic = dc + return &k8sc, nil +} + +// clearFinalizers removes all finalizers from every CR of the given GVR in namespace. +// This unblocks namespace deletion when the owning controller is no longer running. +func (k *K8sClient) clearFinalizers(ctx context.Context, gvr schema.GroupVersionResource, namespace string) { + list, err := k.dynamic.Resource(gvr).Namespace(namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + if !errors.IsNotFound(err) { + log.Printf("clearFinalizers %s: list: %v", gvr.Resource, err) + } + return + } + for _, item := range list.Items { + name := item.GetName() + if len(item.GetFinalizers()) == 0 { + continue + } + item.SetFinalizers(nil) + if _, err := k.dynamic.Resource(gvr).Namespace(namespace).Update(ctx, &item, metav1.UpdateOptions{}); err != nil { + log.Printf("clearFinalizers %s/%s: clear: %v", gvr.Resource, name, err) + } else { + log.Printf("clearFinalizers %s/%s: finalizers cleared", gvr.Resource, name) + } + } +} + +// deleteKMMWebhook removes the KMM ValidatingWebhookConfiguration so that Module CR +// finalizers can be stripped without the webhook intercepting the PATCH request. +func (k *K8sClient) deleteKMMWebhook(ctx context.Context) { + // The webhook name is chart-dependent; delete any that reference kmm. + list, err := k.client.AdmissionregistrationV1().ValidatingWebhookConfigurations().List(ctx, metav1.ListOptions{}) + if err != nil { + return + } + for _, wh := range list.Items { + for _, w := range wh.Webhooks { + if strings.Contains(w.Name, "kmm") { + log.Printf("deleteKMMWebhook: deleting ValidatingWebhookConfiguration %s", wh.Name) + _ = k.client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(ctx, wh.Name, metav1.DeleteOptions{}) + break + } + } + } +} + +// crdGVR is the GroupVersionResource for CustomResourceDefinitions (cluster-scoped). +var crdGVR = schema.GroupVersionResource{ + Group: "apiextensions.k8s.io", + Version: "v1", + Resource: "customresourcedefinitions", +} + +// CleanupClusterScopedResources removes cluster-scoped resources left behind by a previous +// GPU Operator helm release. These persist after namespace deletion and block a fresh install +// with a different release name. +func (k *K8sClient) CleanupClusterScopedResources(ctx context.Context, oldReleaseName string) { + // Patch helm ownership annotation on CRDs to the new release name so helm install adopts them. + crdNames := []string{ + "modules.kmm.sigs.x-k8s.io", + "nodemodulesconfigs.kmm.sigs.x-k8s.io", + "preflightvalidations.kmm.sigs.x-k8s.io", + "deviceconfigs.amd.com", + "remediationworkflowstatuses.amd.com", + } + for _, name := range crdNames { + _ = k.dynamic.Resource(crdGVR).Delete(ctx, name, metav1.DeleteOptions{}) + } + + // DeviceClass — try both v1 (k8s 1.32+) and v1beta1 (older clusters). + for _, dcVersion := range []string{"v1", "v1beta1"} { + deviceClassGVR := schema.GroupVersionResource{Group: "resource.k8s.io", Version: dcVersion, Resource: "deviceclasses"} + dcs, err := k.dynamic.Resource(deviceClassGVR).List(ctx, metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/managed-by=Helm", + }) + if err != nil { + continue + } + for _, dc := range dcs.Items { + ann := dc.GetAnnotations() + if ann["meta.helm.sh/release-name"] == oldReleaseName { + log.Printf("CleanupClusterScopedResources: deleting DeviceClass %s (resource.k8s.io/%s)", dc.GetName(), dcVersion) + _ = k.dynamic.Resource(deviceClassGVR).Delete(ctx, dc.GetName(), metav1.DeleteOptions{}) + } + } + } + + // ClusterRoles and ClusterRoleBindings. + crList, _ := k.client.RbacV1().ClusterRoles().List(ctx, metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/managed-by=Helm", + }) + if crList != nil { + for _, cr := range crList.Items { + if cr.Annotations["meta.helm.sh/release-name"] == oldReleaseName { + log.Printf("CleanupClusterScopedResources: deleting ClusterRole %s", cr.Name) + _ = k.client.RbacV1().ClusterRoles().Delete(ctx, cr.Name, metav1.DeleteOptions{}) + } + } + } + crbList, _ := k.client.RbacV1().ClusterRoleBindings().List(ctx, metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/managed-by=Helm", + }) + if crbList != nil { + for _, crb := range crbList.Items { + if crb.Annotations["meta.helm.sh/release-name"] == oldReleaseName { + log.Printf("CleanupClusterScopedResources: deleting ClusterRoleBinding %s", crb.Name) + _ = k.client.RbacV1().ClusterRoleBindings().Delete(ctx, crb.Name, metav1.DeleteOptions{}) + } + } + } + + // PriorityClass. + pcList, _ := k.client.SchedulingV1().PriorityClasses().List(ctx, metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/managed-by=Helm", + }) + if pcList != nil { + for _, pc := range pcList.Items { + if pc.Annotations["meta.helm.sh/release-name"] == oldReleaseName { + log.Printf("CleanupClusterScopedResources: deleting PriorityClass %s", pc.Name) + _ = k.client.SchedulingV1().PriorityClasses().Delete(ctx, pc.Name, metav1.DeleteOptions{}) + } + } + } + + // Validating and mutating webhook configurations. + for _, whGVR := range []schema.GroupVersionResource{ + {Group: "admissionregistration.k8s.io", Version: "v1", Resource: "validatingwebhookconfigurations"}, + {Group: "admissionregistration.k8s.io", Version: "v1", Resource: "mutatingwebhookconfigurations"}, + } { + list, err := k.dynamic.Resource(whGVR).List(ctx, metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/managed-by=Helm", + }) + if err != nil { + continue + } + for _, obj := range list.Items { + ann := obj.GetAnnotations() + if ann["meta.helm.sh/release-name"] == oldReleaseName { + log.Printf("CleanupClusterScopedResources: deleting webhook %s %s", whGVR.Resource, obj.GetName()) + _ = k.dynamic.Resource(whGVR).Delete(ctx, obj.GetName(), metav1.DeleteOptions{}) + } + } + } + + // NodeFeatureRule (nfd.k8s-sigs.io/v1) and other custom cluster-scoped resources. + for _, gvr := range []schema.GroupVersionResource{ + {Group: "nfd.k8s-sigs.io", Version: "v1", Resource: "nodefeaturerules"}, + {Group: "argoproj.io", Version: "v1alpha1", Resource: "clusterworkflowtemplates"}, + } { + list, err := k.dynamic.Resource(gvr).List(ctx, metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/managed-by=Helm", + }) + if err != nil { + continue + } + for _, obj := range list.Items { + ann := obj.GetAnnotations() + if ann["meta.helm.sh/release-name"] == oldReleaseName { + log.Printf("CleanupClusterScopedResources: deleting %s/%s %s", gvr.Group, gvr.Resource, obj.GetName()) + _ = k.dynamic.Resource(gvr).Delete(ctx, obj.GetName(), metav1.DeleteOptions{}) + } + } + } +} + +// releaseGPUOperatorCRDs removes the helm release ownership annotations from GPU Operator CRDs +// so that a fresh install with a new release name can adopt them without conflicts. +func (k *K8sClient) releaseGPUOperatorCRDs(ctx context.Context, newReleaseName, newNamespace string) { + names := []string{ + "modules.kmm.sigs.x-k8s.io", + "nodemodulesconfigs.kmm.sigs.x-k8s.io", + "preflightvalidations.kmm.sigs.x-k8s.io", + "deviceconfigs.amd.com", + "remediationworkflowstatuses.amd.com", + } + for _, name := range names { + obj, err := k.dynamic.Resource(crdGVR).Get(ctx, name, metav1.GetOptions{}) + if errors.IsNotFound(err) { + continue + } + if err != nil { + log.Printf("releaseGPUOperatorCRDs: get %s: %v", name, err) + continue + } + ann := obj.GetAnnotations() + if ann == nil { + ann = map[string]string{} + } + ann["meta.helm.sh/release-name"] = newReleaseName + ann["meta.helm.sh/release-namespace"] = newNamespace + obj.SetAnnotations(ann) + if _, err := k.dynamic.Resource(crdGVR).Update(ctx, obj, metav1.UpdateOptions{}); err != nil { + log.Printf("releaseGPUOperatorCRDs: update %s: %v", name, err) + } else { + log.Printf("releaseGPUOperatorCRDs: re-annotated %s → %s/%s", name, newNamespace, newReleaseName) + } + } +} + +// DeleteNamespaceAndWait removes GPU Operator CRs and their finalizers, deletes the namespace, +// and polls until it is fully gone. +func (k *K8sClient) DeleteNamespaceAndWait(ctx context.Context, namespace, _ string, timeout time.Duration) error { + // Remove the KMM validating webhook first so Module finalizer patches succeed. + k.deleteKMMWebhook(ctx) + // Strip finalizers from both CRs so the namespace can terminate cleanly. + k.clearFinalizers(ctx, moduleGVR, namespace) + k.clearFinalizers(ctx, deviceconfigGVR, namespace) + + if err := k.client.CoreV1().Namespaces().Delete(ctx, namespace, metav1.DeleteOptions{}); err != nil { + if errors.IsNotFound(err) { + return nil + } + return err + } + + // Poll until the namespace is gone. + return wait.PollUntilContextTimeout(ctx, 5*time.Second, timeout, true, func(ctx context.Context) (bool, error) { + _, err := k.client.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) + if errors.IsNotFound(err) { + return true, nil + } + log.Printf("DeleteNamespaceAndWait: namespace %s still terminating…", namespace) + return false, nil + }) +} + +func (k *K8sClient) NamespaceExists(ctx context.Context, namespace string) bool { + _, err := k.client.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) + return err == nil +} + +func (k *K8sClient) CreateNamespace(ctx context.Context, namespace string) error { + namespaceObj := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + }, + Status: corev1.NamespaceStatus{}, + } + _, err := k.client.CoreV1().Namespaces().Create(ctx, namespaceObj, metav1.CreateOptions{}) + return err +} + +func (k *K8sClient) DeleteNamespace(ctx context.Context, namespace string) error { + return k.client.CoreV1().Namespaces().Delete(ctx, namespace, metav1.DeleteOptions{}) +} + +func (k *K8sClient) GetPodsByLabel(ctx context.Context, namespace string, labelMap map[string]string) ([]corev1.Pod, error) { + podList, err := k.client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: labels.SelectorFromSet(labelMap).String(), + }) + if err != nil { + return nil, err + } + return podList.Items, nil +} + +func (k *K8sClient) GetNodesByLabel(ctx context.Context, labelMap map[string]string) ([]corev1.Node, error) { + nodeList, err := k.client.CoreV1().Nodes().List(ctx, metav1.ListOptions{ + LabelSelector: labels.SelectorFromSet(labelMap).String(), + }) + if err != nil { + return nil, err + } + return nodeList.Items, nil +} + +func (k *K8sClient) GetServiceByLabel(ctx context.Context, namespace string, labelMap map[string]string) ([]corev1.Service, error) { + nodeList, err := k.client.CoreV1().Services(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: labels.SelectorFromSet(labelMap).String(), + }) + if err != nil { + return nil, err + } + return nodeList.Items, nil +} + +func (k *K8sClient) ValidatePod(ctx context.Context, namespace, podName string) error { + pod, err := k.client.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("unexpected error getting pod %s; err: %w", podName, err) + } + + for _, c := range pod.Status.ContainerStatuses { + if c.State.Waiting != nil && c.State.Waiting.Reason == "CrashLoopBackOff" { + return fmt.Errorf("pod %s in namespace %s is in CrashLoopBackOff", pod.Name, pod.Namespace) + } + } + + return nil +} + +func (k *K8sClient) GetMetricsCmdFromPod(ctx context.Context, rc *restclient.Config, pod *corev1.Pod) (labels []string, fields []string, err error) { + if pod == nil { + return nil, nil, fmt.Errorf("invalid pod") + } + req := k.client.CoreV1().RESTClient().Post().Resource("pods").Name(pod.Name). + Namespace(pod.Namespace). + SubResource("exec") + + cmd := "curl -s localhost:5000/metrics" + req.VersionedParams(&corev1.PodExecOptions{ + Command: []string{"/bin/sh", "-c", cmd}, + Stdin: false, + Stdout: true, + Stderr: false, + TTY: false, + }, scheme.ParameterCodec) + + exec, err := remotecommand.NewSPDYExecutor(rc, "POST", req.URL()) + if err != nil { + return nil, nil, err + } + + buf := &bytes.Buffer{} + err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdout: buf, + Tty: false, + }) + if err != nil { + return nil, nil, fmt.Errorf("%w failed executing command %s on %v/%v", err, cmd, pod.Namespace, pod.Name) + } + //log.Printf("\nbuf : %v\n", buf.String()) + p := expfmt.TextParser{} + m, err := p.TextToMetricFamilies(buf) + if err != nil { + return nil, nil, fmt.Errorf("%w failed parsing to metrics", err) + } + for _, f := range m { + fields = append(fields, *f.Name) + for _, km := range f.Metric { + if len(labels) != 0 { + continue + } + for _, lp := range km.GetLabel() { + labels = append(labels, *lp.Name) + } + } + + } + return +} + +func (k *K8sClient) CreateConfigMap(ctx context.Context, namespace string, name string, json string) error { + mcfgMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Data: map[string]string{ + "config.json": json, + }, + } + + _, err := k.client.CoreV1().ConfigMaps(namespace).Create(ctx, mcfgMap, metav1.CreateOptions{}) + return err +} + +func (k *K8sClient) UpdateConfigMap(ctx context.Context, namespace string, name string, json string) error { + existing, err := k.client.CoreV1().ConfigMaps(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return err + } + if existing.Data == nil { + existing.Data = map[string]string{} + } + existing.Data["config.json"] = json + _, err = k.client.CoreV1().ConfigMaps(namespace).Update(ctx, existing, metav1.UpdateOptions{}) + return err +} + +func (k *K8sClient) DeleteConfigMap(ctx context.Context, namespace string, name string) error { + return k.client.CoreV1().ConfigMaps(namespace).Delete(ctx, name, metav1.DeleteOptions{}) +} + +// WaitForNodeLabel polls until at least one node has the given label key=value, or timeout expires. +func (k *K8sClient) WaitForNodeLabel(ctx context.Context, labelKey, labelValue string, timeout time.Duration) error { + return wait.PollUntilContextTimeout(ctx, 10*time.Second, timeout, true, func(ctx context.Context) (bool, error) { + nodeList, err := k.client.CoreV1().Nodes().List(ctx, metav1.ListOptions{ + LabelSelector: labels.SelectorFromSet(map[string]string{labelKey: labelValue}).String(), + }) + if err != nil { + return false, err + } + if len(nodeList.Items) > 0 { + log.Printf("WaitForNodeLabel: %d node(s) have %s=%s", len(nodeList.Items), labelKey, labelValue) + return true, nil + } + return false, nil + }) +} + +// WaitForDaemonSetReady polls until all desired pods of a DaemonSet are ready, or timeout expires. +func (k *K8sClient) WaitForDaemonSetReady(ctx context.Context, namespace, name string, timeout time.Duration) error { + return wait.PollUntilContextTimeout(ctx, 10*time.Second, timeout, true, func(ctx context.Context) (bool, error) { + ds, err := k.client.AppsV1().DaemonSets(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + return false, nil + } + return false, err + } + if ds.Status.NumberReady > 0 && ds.Status.NumberReady == ds.Status.DesiredNumberScheduled { + return true, nil + } + return false, nil + }) +} + +// GetNodeAllocatableGPUs returns the number of amd.com/gpu allocatable resources on a node. +// Returns -1 if the resource is not present. +func (k *K8sClient) GetNodeAllocatableGPUs(ctx context.Context, nodeName string) (int64, error) { + node, err := k.client.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) + if err != nil { + return 0, err + } + qty, ok := node.Status.Allocatable["amd.com/gpu"] + if !ok { + return -1, nil + } + return qty.Value(), nil +} + +// CreatePod creates a Pod in the given namespace and returns any error. +func (k *K8sClient) CreatePod(ctx context.Context, namespace string, pod *corev1.Pod) error { + _, err := k.client.CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{}) + return err +} + +// WaitForPodSucceeded polls until the pod reaches Succeeded phase or timeout expires. +func (k *K8sClient) WaitForPodSucceeded(ctx context.Context, namespace, podName string, timeout time.Duration) error { + return wait.PollUntilContextTimeout(ctx, 10*time.Second, timeout, true, func(ctx context.Context) (bool, error) { + pod, err := k.client.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + return false, nil + } + return false, err + } + switch pod.Status.Phase { + case corev1.PodSucceeded: + return true, nil + case corev1.PodFailed: + return false, fmt.Errorf("pod %s failed", podName) + } + return false, nil + }) +} + +// DeletePod deletes a pod by name. +func (k *K8sClient) DeletePod(ctx context.Context, namespace, name string) error { + return k.client.CoreV1().Pods(namespace).Delete(ctx, name, metav1.DeleteOptions{}) +} + +// WaitForPodDeleted polls until the named pod no longer exists (or timeout). +func (k *K8sClient) WaitForPodDeleted(ctx context.Context, namespace, podName string, timeout time.Duration) error { + return wait.PollUntilContextTimeout(ctx, 2*time.Second, timeout, true, func(ctx context.Context) (bool, error) { + _, err := k.client.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) + if errors.IsNotFound(err) { + return true, nil + } + return false, nil + }) +} + +// GetPodLogs returns the logs for a pod's first container. +func (k *K8sClient) GetPodLogs(ctx context.Context, namespace, podName string) (string, error) { + req := k.client.CoreV1().Pods(namespace).GetLogs(podName, &corev1.PodLogOptions{}) + result := req.Do(ctx) + raw, err := result.Raw() + if err != nil { + return "", err + } + return string(raw), nil +} + +// GetNodeNames returns all node names in the cluster. +func (k *K8sClient) GetNodeNames(ctx context.Context) ([]string, error) { + nodeList, err := k.client.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, err + } + names := make([]string, 0, len(nodeList.Items)) + for _, n := range nodeList.Items { + names = append(names, n.Name) + } + return names, nil +} + +func (k *K8sClient) ExecCmdOnPod(ctx context.Context, rc *restclient.Config, pod *corev1.Pod, container, execCmd string) (string, error) { + if pod == nil { + return "", fmt.Errorf("No pod specified") + } + req := k.client.CoreV1().RESTClient().Post().Resource("pods").Name(pod.Name).Namespace(pod.Namespace).SubResource("exec") + req.VersionedParams(&corev1.PodExecOptions{ + Container: container, + Command: []string{"/bin/sh", "-c", execCmd}, + Stdin: false, + Stdout: true, + Stderr: false, + TTY: false, + }, scheme.ParameterCodec) + executor, err := remotecommand.NewSPDYExecutor(rc, "POST", req.URL()) + if err != nil { + return "", fmt.Errorf("failed to create command executor. Error:%v", err) + } + buf := &bytes.Buffer{} + err = executor.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdout: buf, + Tty: false, + }) + if err != nil { + return "", fmt.Errorf("failed to run command on pod %v. Error:%v", pod.Name, err) + } + + return buf.String(), nil +} + +// DeleteCertManagerKubeSystemRoles removes cert-manager Roles from kube-system that may +// have been left without helm ownership metadata after a previous run's cleanup. Without +// proper labels/annotations, helm cannot adopt them and fails with "cannot be imported". +func (k *K8sClient) DeleteCertManagerKubeSystemRoles(ctx context.Context) { + certMgrRoles := []string{ + "cert-manager-cainjector:leaderelection", + "cert-manager:leaderelection", + "cert-manager-webhook:dynamic-serving", + } + for _, roleName := range certMgrRoles { + err := k.client.RbacV1().Roles("kube-system").Delete(ctx, roleName, metav1.DeleteOptions{}) + if err != nil { + log.Printf("DeleteCertManagerKubeSystemRoles: %s: %v", roleName, err) + } else { + log.Printf("DeleteCertManagerKubeSystemRoles: deleted kube-system/Role/%s", roleName) + } + } + // Also delete RoleBindings + certMgrRoleBindings := []string{ + "cert-manager-cainjector:leaderelection", + "cert-manager:leaderelection", + "cert-manager-webhook:dynamic-serving", + } + for _, rbName := range certMgrRoleBindings { + err := k.client.RbacV1().RoleBindings("kube-system").Delete(ctx, rbName, metav1.DeleteOptions{}) + if err != nil { + log.Printf("DeleteCertManagerKubeSystemRoles: rolebinding %s: %v", rbName, err) + } else { + log.Printf("DeleteCertManagerKubeSystemRoles: deleted kube-system/RoleBinding/%s", rbName) + } + } +} diff --git a/tests/k8s-e2e/doc.go b/tests/k8s-e2e/doc.go new file mode 100644 index 000000000..77e77cae4 --- /dev/null +++ b/tests/k8s-e2e/doc.go @@ -0,0 +1,43 @@ +/* +Copyright (c) Advanced Micro Devices, Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gpuope2e + +import ( + "context" + + "github.com/ROCm/gpu-operator/tests/k8s-e2e/clients" + restclient "k8s.io/client-go/rest" +) + +// E2ESuite holds configuration shared across all GPU Operator e2e tests. +type E2ESuite struct { + k8sclient *clients.K8sClient + helmClient *clients.HelmClient + restConfig *restclient.Config + ns string + kubeconfig string + + // GPU Operator install parameters. + operatorChart string + operatorTag string + helmSet []string + + existingDeploy bool // true: skip install/teardown (verify only) + + // suiteHook is an optional callback invoked at the end of SetUpSuite. + suiteHook func(ctx context.Context) error +} diff --git a/tests/k8s-e2e/go.mod b/tests/k8s-e2e/go.mod new file mode 100644 index 000000000..3272c0838 --- /dev/null +++ b/tests/k8s-e2e/go.mod @@ -0,0 +1,153 @@ +module github.com/ROCm/gpu-operator/tests/k8s-e2e + +go 1.25.8 + +require ( + github.com/mittwald/go-helm-client v0.12.14 + github.com/prometheus/common v0.62.0 + github.com/stretchr/testify v1.10.0 + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c + helm.sh/helm/v3 v3.16.1 + k8s.io/api v0.34.0 + k8s.io/apimachinery v0.34.0 + k8s.io/client-go v0.34.0 + k8s.io/kubectl v0.34.0 +) + +require ( + dario.cat/mergo v1.0.1 // indirect + github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect + github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect + github.com/BurntSushi/toml v1.3.2 // indirect + github.com/MakeNowJust/heredoc v1.0.0 // indirect + github.com/Masterminds/goutils v1.1.1 // indirect + github.com/Masterminds/semver/v3 v3.3.0 // indirect + github.com/Masterminds/sprig/v3 v3.3.0 // indirect + github.com/Masterminds/squirrel v1.5.4 // indirect + github.com/Microsoft/hcsshim v0.11.4 // indirect + github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/chai2010/gettext-go v1.0.2 // indirect + github.com/containerd/containerd v1.7.12 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/cyphar/filepath-securejoin v0.3.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/cli v25.0.1+incompatible // indirect + github.com/docker/distribution v2.8.3+incompatible // indirect + github.com/docker/docker v25.0.6+incompatible // indirect + github.com/docker/docker-credential-helpers v0.8.0 // indirect + github.com/docker/go-connections v0.5.0 // indirect + github.com/docker/go-metrics v0.0.1 // indirect + github.com/emicklei/go-restful/v3 v3.12.2 // indirect + github.com/evanphx/json-patch v5.9.0+incompatible // indirect + github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect + github.com/fatih/color v1.16.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-errors/errors v1.5.1 // indirect + github.com/go-gorp/gorp/v3 v3.1.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.4 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/gobwas/glob v0.2.3 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/btree v1.1.3 // indirect + github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/gorilla/mux v1.8.1 // indirect + github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect + github.com/gosuri/uitable v0.0.4 // indirect + github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/huandu/xstrings v1.5.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jmoiron/sqlx v1.4.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect + github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect + github.com/lib/pq v1.10.9 // indirect + github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/go-wordwrap v1.0.1 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/moby/locker v1.0.1 // indirect + github.com/moby/spdystream v0.5.0 // indirect + github.com/moby/term v0.5.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.0 // indirect + github.com/peterbourgon/diskv v2.0.1+incompatible // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/client_golang v1.22.0 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + github.com/rivo/uniseg v0.4.4 // indirect + github.com/rogpeppe/go-internal v1.13.1 // indirect + github.com/rubenv/sql-migrate v1.7.0 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/shopspring/decimal v1.4.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/spf13/cast v1.7.0 // indirect + github.com/spf13/cobra v1.9.1 // indirect + github.com/spf13/pflag v1.0.6 // indirect + github.com/x448/float16 v0.8.4 // indirect + github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect + github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + github.com/xeipuuv/gojsonschema v1.2.0 // indirect + github.com/xlab/treeprint v1.2.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect + go.opentelemetry.io/otel v1.35.0 // indirect + go.opentelemetry.io/otel/metric v1.35.0 // indirect + go.opentelemetry.io/otel/trace v1.35.0 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/crypto v0.36.0 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/oauth2 v0.27.0 // indirect + golang.org/x/sync v0.12.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/term v0.30.0 // indirect + golang.org/x/text v0.23.0 // indirect + golang.org/x/time v0.9.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb // indirect + google.golang.org/grpc v1.72.1 // indirect + google.golang.org/protobuf v1.36.5 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiextensions-apiserver v0.31.1 // indirect + k8s.io/apiserver v0.31.1 // indirect + k8s.io/cli-runtime v0.34.0 // indirect + k8s.io/component-base v0.34.0 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect + k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect + oras.land/oras-go v1.2.5 // indirect + sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect + sigs.k8s.io/kustomize/api v0.20.1 // indirect + sigs.k8s.io/kustomize/kyaml v0.20.1 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect +) diff --git a/tests/k8s-e2e/go.sum b/tests/k8s-e2e/go.sum new file mode 100644 index 000000000..8688bb044 --- /dev/null +++ b/tests/k8s-e2e/go.sum @@ -0,0 +1,479 @@ +dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= +dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= +github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= +github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= +github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= +github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= +github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= +github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver/v3 v3.3.0 h1:B8LGeaivUe71a5qox1ICM/JLl0NqZSW5CHyL+hmvYS0= +github.com/Masterminds/semver/v3 v3.3.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/Masterminds/sprig/v3 v3.3.0 h1:mQh0Yrg1XPo6vjYXgtf5OtijNAKJRNcTdOOGZe3tPhs= +github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0= +github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM= +github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8= +github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w= +github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/OJnIp5u0s1SbQ8dVfLCZJsnvazdBP5hS4iRs= +github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/bshuster-repo/logrus-logstash-hook v1.0.0 h1:e+C0SB5R1pu//O4MQ3f9cFuPGoOVeF2fE4Og9otCc70= +github.com/bshuster-repo/logrus-logstash-hook v1.0.0/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= +github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd h1:rFt+Y/IK1aEZkEHchZRSq9OQbsSzIT/OrI8YFFmRIng= +github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= +github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b h1:otBG+dV+YK+Soembjv71DPz3uX/V/6MMlSyD9JBQ6kQ= +github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= +github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0 h1:nvj0OLI3YqYXer/kZD8Ri1aaunCxIEsOst1BVJswV0o= +github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk= +github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA= +github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= +github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= +github.com/containerd/containerd v1.7.12 h1:+KQsnv4VnzyxWcfO9mlxxELaoztsDEjOuCMPAuPqgU0= +github.com/containerd/containerd v1.7.12/go.mod h1:/5OMpE1p0ylxtEUGY8kuCYkDRzJm9NO1TFMWjUpdevk= +github.com/containerd/continuity v0.4.2 h1:v3y/4Yz5jwnvqPKJJ+7Wf93fyWoCB3F5EclWG023MDM= +github.com/containerd/continuity v0.4.2/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/cyphar/filepath-securejoin v0.3.1 h1:1V7cHiaW+C+39wEfpH6XlLBQo3j/PciWFrgfCLS8XrE= +github.com/cyphar/filepath-securejoin v0.3.1/go.mod h1:F7i41x/9cBF7lzCrVsYs9fuzwRZm4NQsGTBdpp6mETc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2 h1:aBfCb7iqHmDEIp6fBvC/hQUddQfg+3qdYjwzaiP9Hnc= +github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2/go.mod h1:WHNsWjnIn2V1LYOrME7e8KxSeKunYHsxEm4am0BUtcI= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/cli v25.0.1+incompatible h1:mFpqnrS6Hsm3v1k7Wa/BO23oz0k121MTbTO1lpcGSkU= +github.com/docker/cli v25.0.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= +github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v25.0.6+incompatible h1:5cPwbwriIcsua2REJe8HqQV+6WlWc1byg2QSXzBxBGg= +github.com/docker/docker v25.0.6+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker-credential-helpers v0.8.0 h1:YQFtbBQb4VrpoPxhFuzEBPQ9E16qz5SpHLS+uswaCp8= +github.com/docker/docker-credential-helpers v0.8.0/go.mod h1:UGFXcuoQ5TxPiB54nHOZ32AWRqQdECoh/Mg0AlEYb40= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8= +github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= +github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8= +github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= +github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 h1:ZClxb8laGDf5arXfYcAtECDFgAgHklGI8CxgjHnXKJ4= +github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= +github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= +github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= +github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f h1:Wl78ApPPB2Wvf/TIe2xdyJxTlb6obmF18d8QdkxNDu4= +github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSYXu++VVOHnXeitef/D8n/6y4QV8uLHSFXX4NeXMGc= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/foxcpp/go-mockdns v1.1.0 h1:jI0rD8M0wuYAxL7r/ynTrCQQq0BVqfB99Vgk7DlmewI= +github.com/foxcpp/go-mockdns v1.1.0/go.mod h1:IhLeSFGed3mJIAXPH2aiRQB+kqz7oqu8ld2qVbOu7Wk= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk= +github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-gorp/gorp/v3 v3.1.0 h1:ItKF/Vbuj31dmV4jxA1qblpSwkl9g1typ24xoe70IGs= +github.com/go-gorp/gorp/v3 v3.1.0/go.mod h1:dLEjIyyRNiXvNZ8PSmzpt1GsWAUK8kjVhEpjH8TixEw= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.4 h1:bKlDxQxQJgwpUSgOENiMPzCTBVuc7vTdXSSgNeAhojU= +github.com/go-openapi/jsonreference v0.20.4/go.mod h1:5pZJyJP2MnYCpoeoMAql78cCHauHj0V9Lhc506VOpw4= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/gomodule/redigo v1.8.2 h1:H5XSIre1MB5NbPYFp+i1NBbb5qN1W8Y8YAQoAYbkm8k= +github.com/gomodule/redigo v1.8.2/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= +github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= +github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY= +github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo= +github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA= +github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI= +github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= +github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw= +github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= +github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk= +github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= +github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM= +github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= +github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/mittwald/go-helm-client v0.12.14 h1:az3GJ4kRmFK609Ic3iHXveNtg92n9jWG0YpKKTIK4oo= +github.com/mittwald/go-helm-client v0.12.14/go.mod h1:2VogAupgnV7FiuoPqtpCYKS/RrMh9fFA3/pD/OmTaLc= +github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= +github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= +github.com/moby/spdystream v0.5.0 h1:7r0J1Si3QO/kjRitvSLVVFUjxMEb/YLj6S9FF62JBCU= +github.com/moby/spdystream v0.5.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= +github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78= +github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= +github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= +github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= +github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 h1:Ii+DKncOVM8Cu1Hc+ETb5K+23HdAMvESYE3ZJ5b5cMI= +github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/poy/onpar v1.1.2 h1:QaNrNiZx0+Nar5dLgTVp5mXkyoVFIbepjyEoGSnhbAY= +github.com/poy/onpar v1.1.2/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjzg= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= +github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= +github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= +github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rubenv/sql-migrate v1.7.0 h1:HtQq1xyTN2ISmQDggnh0c9U3JlP8apWh8YO2jzlXpTI= +github.com/rubenv/sql-migrate v1.7.0/go.mod h1:S4wtDEG1CKn+0ShpTtzWhFpHHI5PvCUtiGI+C+Z2THE= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= +github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= +github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= +github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= +github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43 h1:+lm10QQTNSBd8DVTNGHx7o/IKu9HYDvLMffDhbyLccI= +github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= +github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50 h1:hlE8//ciYMztlGpl/VA+Zm1AcTPHYkHJPbHqE6WJUXE= +github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= +github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f h1:ERexzlUfuTvpE74urLSbIQW0Z/6hF9t8U4NsJLaioAY= +github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 h1:yd02MEjBdJkG3uabWP9apV+OuWRIXGDuJEUJbOHmCFU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0/go.mod h1:umTcuxiv1n/s/S6/c2AT/g2CQ7u5C59sHDNmfSwgz7Q= +go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= +go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= +go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= +go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= +go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= +go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= +golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= +golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= +golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb h1:TLPQVbx1GJ8VKZxz52VAxl1EBgKXXbTiU9Fc5fZeLn4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:LuRYeWDFV6WOn90g357N17oMCaxpgCnbi/44qJvDn2I= +google.golang.org/grpc v1.72.1 h1:HR03wO6eyZ7lknl75XlxABNVLLFc2PAb6mHlYh756mA= +google.golang.org/grpc v1.72.1/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= +gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= +helm.sh/helm/v3 v3.16.1 h1:cER6tI/8PgUAsaJaQCVBUg3VI9KN4oVaZJgY60RIc0c= +helm.sh/helm/v3 v3.16.1/go.mod h1:r+xBHHP20qJeEqtvBXMf7W35QDJnzY/eiEBzt+TfHps= +k8s.io/api v0.34.0 h1:L+JtP2wDbEYPUeNGbeSa/5GwFtIA662EmT2YSLOkAVE= +k8s.io/api v0.34.0/go.mod h1:YzgkIzOOlhl9uwWCZNqpw6RJy9L2FK4dlJeayUoydug= +k8s.io/apiextensions-apiserver v0.31.1 h1:L+hwULvXx+nvTYX/MKM3kKMZyei+UiSXQWciX/N6E40= +k8s.io/apiextensions-apiserver v0.31.1/go.mod h1:tWMPR3sgW+jsl2xm9v7lAyRF1rYEK71i9G5dRtkknoQ= +k8s.io/apimachinery v0.34.0 h1:eR1WO5fo0HyoQZt1wdISpFDffnWOvFLOOeJ7MgIv4z0= +k8s.io/apimachinery v0.34.0/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= +k8s.io/apiserver v0.31.1 h1:Sars5ejQDCRBY5f7R3QFHdqN3s61nhkpaX8/k1iEw1c= +k8s.io/apiserver v0.31.1/go.mod h1:lzDhpeToamVZJmmFlaLwdYZwd7zB+WYRYIboqA1kGxM= +k8s.io/cli-runtime v0.34.0 h1:N2/rUlJg6TMEBgtQ3SDRJwa8XyKUizwjlOknT1mB2Cw= +k8s.io/cli-runtime v0.34.0/go.mod h1:t/skRecS73Piv+J+FmWIQA2N2/rDjdYSQzEE67LUUs8= +k8s.io/client-go v0.34.0 h1:YoWv5r7bsBfb0Hs2jh8SOvFbKzzxyNo0nSb0zC19KZo= +k8s.io/client-go v0.34.0/go.mod h1:ozgMnEKXkRjeMvBZdV1AijMHLTh3pbACPvK7zFR+QQY= +k8s.io/component-base v0.34.0 h1:bS8Ua3zlJzapklsB1dZgjEJuJEeHjj8yTu1gxE2zQX8= +k8s.io/component-base v0.34.0/go.mod h1:RSCqUdvIjjrEm81epPcjQ/DS+49fADvGSCkIP3IC6vg= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOPolHyvO06MXG5TUIj2mNAA= +k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= +k8s.io/kubectl v0.34.0 h1:NcXz4TPTaUwhiX4LU+6r6udrlm0NsVnSkP3R9t0dmxs= +k8s.io/kubectl v0.34.0/go.mod h1:bmd0W5i+HuG7/p5sqicr0Li0rR2iIhXL0oUyLF3OjR4= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +oras.land/oras-go v1.2.5 h1:XpYuAwAb0DfQsunIyMfeET92emK8km3W4yEzZvUbsTo= +oras.land/oras-go v1.2.5/go.mod h1:PuAwRShRZCsZb7g8Ar3jKKQR/2A/qN+pkYxIOd/FAoo= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/kustomize/api v0.20.1 h1:iWP1Ydh3/lmldBnH/S5RXgT98vWYMaTUL1ADcr+Sv7I= +sigs.k8s.io/kustomize/api v0.20.1/go.mod h1:t6hUFxO+Ph0VxIk1sKp1WS0dOjbPCtLJ4p8aADLwqjM= +sigs.k8s.io/kustomize/kyaml v0.20.1 h1:PCMnA2mrVbRP3NIB6v9kYCAc38uvFLVs8j/CD567A78= +sigs.k8s.io/kustomize/kyaml v0.20.1/go.mod h1:0EmkQHRUsJxY8Ug9Niig1pUMSCGHxQ5RklbpV/Ri6po= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/tests/k8s-e2e/operator_test.go b/tests/k8s-e2e/operator_test.go new file mode 100644 index 000000000..ec68e88c5 --- /dev/null +++ b/tests/k8s-e2e/operator_test.go @@ -0,0 +1,634 @@ +/* +Copyright (c) Advanced Micro Devices, Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// GPU Operator e2e tests — full lifecycle and DME verification. +// +// Test sequence: +// +// Op000 InstallCertManager — install cert-manager (operator prerequisite) +// Op001 InstallGPUOperator — install AMD GPU Operator via Helm +// Op010 VerifyNodeLabeller — NFD labels nodes; Node Labeller DaemonSet ready +// Op020 VerifyDevicePlugin — Device Plugin DaemonSet ready; amd.com/gpu allocatable +// Op030 VerifyKMMDriver — KMM controller pods Running (skipped for DKMS nodes) +// Op040 VerifyDMEDaemonSet — DME DaemonSet ready; key metrics present +// Op050 VerifyGPUHealth — all GPUs report gpu_health=1 +// Op060 VerifyCoreMetrics — VRAM/PCIe/ECC/clock/power/temperature metrics present +// Op065 VerifyPartitionedGPUMetrics — partition labels and per-partition metrics (auto-skips) +// Op070 ScheduleGPUWorkload — rocminfo/amd-smi pod completes successfully (GPU scheduling verified) +// Op900 TearDownOperator — uninstall GPU Operator and cert-manager + +package gpuope2e + +import ( + "context" + "fmt" + "log" + "strings" + "time" + + "github.com/stretchr/testify/assert" + . "gopkg.in/check.v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // certManagerReleaseName is the helm release name for cert-manager. + certManagerReleaseName = "cert-manager" + + // certManagerChart is the public cert-manager helm chart repo URL. + certManagerChart = "https://charts.jetstack.io" + + // dmeMetricsPort is the port DME listens on inside its pod. + dmeMetricsPort = 5000 + + // gpuWorkloadPodName is the name of the test GPU workload pod. + gpuWorkloadPodName = "op-e2e-gpu-workload" + + // operatorPollTimeout is the timeout for waiting on operator-managed resources. + operatorPollTimeout = 15 * time.Minute +) + +// ---- Op000–Op001: install ----------------------------------------------- + +// TestOp000InstallCertManager installs cert-manager, a prerequisite for the GPU Operator. +// Skipped when running in existing-deploy mode or when cert-manager is already present. +func (s *E2ESuite) TestOp000InstallCertManager(c *C) { + if s.existingDeploy { + c.Skip("skipping install: existing deploy mode") + } + ctx := context.Background() + + if s.k8sclient.NamespaceExists(ctx, "cert-manager") { + log.Print("Op000: cert-manager namespace already exists — skipping install") + return + } + + log.Print("Op000: adding cert-manager helm repository") + assert.NoError(c, s.helmClient.AddRepository("jetstack", certManagerChart)) + + log.Print("Op000: installing cert-manager") + _, err := s.helmClient.InstallChartWithTimeout( + ctx, + certManagerReleaseName, + "jetstack/cert-manager", + "", + []string{"installCRDs=true"}, + operatorPollTimeout, + ) + assert.NoError(c, err) + log.Print("Op000: cert-manager installed") +} + +// TestOp001InstallGPUOperator installs the AMD GPU Operator via Helm using +// -operatorchart and -operatortag. Pass extra overrides with -helmset. +// +// Example overrides for local images: +// +// -helmset controllerManager.manager.imagePullPolicy=Never +// -helmset kmm.controller.manager.imagePullPolicy=Never +// -helmset kmm.webhookServer.webhookServer.imagePullPolicy=IfNotPresent +// -helmset 'deviceConfig.spec.metricsExporter.image=rocm/device-metrics-exporter:v1.5.0' +func (s *E2ESuite) TestOp001InstallGPUOperator(c *C) { + if s.existingDeploy { + c.Skip("skipping install: existing deploy mode") + } + ctx := context.Background() + + log.Printf("Op001: installing GPU Operator from chart %q version %s helmset=[%s]", + s.operatorChart, s.operatorTag, helmSetJoin(s.helmSet)) + + params := append([]string{ + fmt.Sprintf("devicePlugin.image.tag=%s", s.operatorTag), + fmt.Sprintf("nodeLabellerImage.tag=%s", s.operatorTag), + }, s.helmSet...) + + _, err := s.helmClient.InstallChartWithTimeout( + ctx, + operatorReleaseName, + s.operatorChart, + s.operatorTag, + params, + operatorPollTimeout, + ) + assert.NoError(c, err) + log.Print("Op001: GPU Operator installed") +} + +// ---- Op010–Op030: infrastructure ---------------------------------------- + +// TestOp010VerifyNodeLabeller waits for NFD to label GPU nodes and verifies +// the Node Labeller DaemonSet is ready. +func (s *E2ESuite) TestOp010VerifyNodeLabeller(c *C) { + ctx := context.Background() + + log.Print("Op010: waiting for NFD to label node feature.node.kubernetes.io/amd-gpu=true") + err := s.k8sclient.WaitForNodeLabel(ctx, "feature.node.kubernetes.io/amd-gpu", "true", operatorPollTimeout) + assert.NoError(c, err, "NFD did not label node with feature.node.kubernetes.io/amd-gpu=true") + + log.Print("Op010: waiting for node-labeller DaemonSet") + err = s.k8sclient.WaitForDaemonSetReady(ctx, s.ns, "default-node-labeller", operatorPollTimeout) + assert.NoError(c, err, "node-labeller DaemonSet did not become ready") + + nodes, err := s.k8sclient.GetNodesByLabel(ctx, map[string]string{ + "feature.node.kubernetes.io/amd-gpu": "true", + }) + assert.NoError(c, err) + assert.True(c, len(nodes) > 0, "expected at least one node with feature.node.kubernetes.io/amd-gpu=true") + log.Printf("Op010: %d node(s) have amd-gpu label", len(nodes)) + for _, n := range nodes { + for k, v := range n.Labels { + if strings.HasPrefix(k, "amd.com/gpu.") { + log.Printf("Op010: node %s label %s=%s", n.Name, k, v) + } + } + } +} + +// TestOp020VerifyDevicePlugin verifies the Device Plugin DaemonSet is ready +// and amd.com/gpu resources are allocatable on GPU nodes. +func (s *E2ESuite) TestOp020VerifyDevicePlugin(c *C) { + ctx := context.Background() + + log.Print("Op020: waiting for device-plugin DaemonSet") + err := s.k8sclient.WaitForDaemonSetReady(ctx, s.ns, "default-device-plugin", operatorPollTimeout) + assert.NoError(c, err, "device-plugin DaemonSet did not become ready") + + gpuNodes, err := s.k8sclient.GetNodesByLabel(ctx, map[string]string{ + "feature.node.kubernetes.io/amd-gpu": "true", + }) + assert.NoError(c, err) + assert.True(c, len(gpuNodes) > 0, "no GPU nodes found") + + // The device-plugin may take a moment after its pod is Running to register + // with kubelet and update node allocatable. Poll until GPU count > 0. + for _, node := range gpuNodes { + pollCtx, cancel := context.WithTimeout(ctx, 2*time.Minute) + var gpuCount int64 + for { + gpuCount, err = s.k8sclient.GetNodeAllocatableGPUs(pollCtx, node.Name) + if err != nil || gpuCount > 0 { + break + } + log.Printf("Op020: node %s allocatable amd.com/gpu = 0, waiting for device-plugin registration…", node.Name) + select { + case <-pollCtx.Done(): + case <-time.After(5 * time.Second): + continue + } + break + } + cancel() + assert.NoError(c, err) + assert.True(c, gpuCount > 0, + "node %s: expected amd.com/gpu > 0 in allocatable resources, got %d", node.Name, gpuCount) + log.Printf("Op020: node %s allocatable amd.com/gpu = %d", node.Name, gpuCount) + } +} + +// TestOp030VerifyKMMDriver verifies KMM controller pods are Running. +// Automatically skipped on clusters using a pre-installed DKMS driver. +func (s *E2ESuite) TestOp030VerifyKMMDriver(c *C) { + ctx := context.Background() + + pods, err := s.k8sclient.GetPodsByLabel(ctx, s.ns, map[string]string{ + "app.kubernetes.io/name": "kmm", + "control-plane": "controller", + }) + if err != nil || len(pods) == 0 { + log.Print("Op030: no KMM controller pods found — assuming pre-installed DKMS driver; skipping") + c.Skip("KMM controller not found — pre-installed driver assumed") + return + } + + log.Printf("Op030: found %d KMM controller pod(s); phase=%s", len(pods), pods[0].Status.Phase) + for _, pod := range pods { + // Skip terminal pods from old ReplicaSet revisions (Succeeded/Failed are expected + // for completed init or evicted pods from prior rollouts). + if pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed { + log.Printf("Op030: skipping terminal pod %s (phase=%s)", pod.Name, pod.Status.Phase) + continue + } + assert.Equal(c, corev1.PodRunning, pod.Status.Phase, + "KMM pod %s is not Running (phase: %s)", pod.Name, pod.Status.Phase) + } +} + +// ---- Op040–Op065: DME verification -------------------------------------- + +// TestOp040VerifyDMEDaemonSet verifies the DME DaemonSet is ready and key +// metric families are present in the metrics endpoint. +func (s *E2ESuite) TestOp040VerifyDMEDaemonSet(c *C) { + ctx := context.Background() + + log.Print("Op040: waiting for metrics-exporter DaemonSet") + err := s.k8sclient.WaitForDaemonSetReady(ctx, s.ns, "default-metrics-exporter", operatorPollTimeout) + assert.NoError(c, err, "metrics-exporter DaemonSet did not become ready") + + pods, err := s.k8sclient.GetPodsByLabel(ctx, s.ns, map[string]string{ + "app.kubernetes.io/name": "metrics-exporter", + }) + assert.NoError(c, err) + assert.True(c, len(pods) > 0, "no metrics-exporter pods found") + + var dmePod *corev1.Pod + for i := range pods { + if pods[i].Status.Phase == corev1.PodRunning { + dmePod = &pods[i] + break + } + } + if dmePod == nil { + assert.Fail(c, "no Running metrics-exporter pod found (pods may be in CrashLoopBackOff)") + return + } + log.Printf("Op040: using DME pod %s (phase=%s)", dmePod.Name, dmePod.Status.Phase) + + var fields []string + assert.Eventually(c, func() bool { + _, f, err := s.k8sclient.GetMetricsCmdFromPod(ctx, s.restConfig, dmePod) + if err != nil { + log.Printf("Op040: waiting for DME metrics endpoint: %v", err) + return false + } + fields = f + return len(fields) > 0 + }, 90*time.Second, 5*time.Second, "DME metrics endpoint did not become ready within 90s") + + fieldSet := make(map[string]bool, len(fields)) + for _, f := range fields { + fieldSet[f] = true + } + for _, m := range []string{"gpu_nodes_total", "gpu_health", "gpu_total_vram", "gpu_used_vram"} { + assert.True(c, fieldSet[m], "required metric %q not found in DME output", m) + } + hasPower := fieldSet["gpu_package_power"] || fieldSet["gpu_average_package_power"] + assert.True(c, hasPower, "no power metric found (expected gpu_package_power or gpu_average_package_power)") + hasTemp := fieldSet["gpu_junction_temperature"] || fieldSet["gpu_edge_temperature"] + assert.True(c, hasTemp, "no temperature metric found (expected gpu_junction_temperature or gpu_edge_temperature)") + log.Printf("Op040: DME returned %d metric families; all required metrics present", len(fields)) +} + +// TestOp050VerifyGPUHealth asserts all GPUs reported by DME have gpu_health=1. +func (s *E2ESuite) TestOp050VerifyGPUHealth(c *C) { + ctx := context.Background() + + pods, err := s.k8sclient.GetPodsByLabel(ctx, s.ns, map[string]string{ + "app.kubernetes.io/name": "metrics-exporter", + }) + assert.NoError(c, err) + assert.True(c, len(pods) > 0, "no metrics-exporter pods found") + + var dmePod *corev1.Pod + for i := range pods { + if pods[i].Status.Phase == corev1.PodRunning { + dmePod = &pods[i] + break + } + } + if dmePod == nil { + assert.Fail(c, "no Running metrics-exporter pod found") + return + } + + var output string + assert.Eventually(c, func() bool { + var err error + output, err = s.k8sclient.ExecCmdOnPod(ctx, s.restConfig, dmePod, "", + fmt.Sprintf("curl -s localhost:%d/metrics", dmeMetricsPort)) + if err != nil { + log.Printf("Op050: waiting for DME metrics: %v", err) + return false + } + return strings.Contains(output, "gpu_health") + }, 90*time.Second, 5*time.Second, "DME metrics endpoint did not return gpu_health within 90s") + + healthy, unhealthy := 0, 0 + for _, line := range strings.Split(output, "\n") { + if strings.HasPrefix(line, "gpu_health{") { + if strings.HasSuffix(strings.TrimSpace(line), " 1") { + healthy++ + } else { + unhealthy++ + log.Printf("Op050: unhealthy GPU: %s", line) + } + } + } + log.Printf("Op050: gpu_health — healthy=%d unhealthy=%d", healthy, unhealthy) + assert.True(c, healthy > 0, "no healthy GPUs found in DME metrics") + assert.Equal(c, 0, unhealthy, "found %d unhealthy GPU(s)", unhealthy) +} + +// TestOp060VerifyCoreMetrics checks VRAM, PCIe, ECC, clock, power and temperature +// metric categories in the DME output. +func (s *E2ESuite) TestOp060VerifyCoreMetrics(c *C) { + ctx := context.Background() + + pods, err := s.k8sclient.GetPodsByLabel(ctx, s.ns, map[string]string{ + "app.kubernetes.io/name": "metrics-exporter", + }) + assert.NoError(c, err) + assert.True(c, len(pods) > 0) + + var dmePod *corev1.Pod + for i := range pods { + if pods[i].Status.Phase == corev1.PodRunning { + dmePod = &pods[i] + break + } + } + if dmePod == nil { + assert.Fail(c, "no Running metrics-exporter pod found") + return + } + + var fields []string + assert.Eventually(c, func() bool { + _, f, err := s.k8sclient.GetMetricsCmdFromPod(ctx, s.restConfig, dmePod) + if err != nil { + log.Printf("Op060: waiting for DME metrics: %v", err) + return false + } + fields = f + return len(fields) > 0 + }, 90*time.Second, 5*time.Second, "DME metrics endpoint did not respond within 90s") + + fieldSet := make(map[string]bool, len(fields)) + for _, f := range fields { + fieldSet[f] = true + } + + categories := map[string][]string{ + "vram": {"gpu_total_vram", "gpu_used_vram", "gpu_free_vram"}, + "pcie": {"pcie_speed", "pcie_max_speed"}, + "ecc": {"gpu_ecc_correct_total", "gpu_ecc_uncorrect_total"}, + "clock": {"gpu_clock"}, + } + for category, metrics := range categories { + for _, m := range metrics { + assert.True(c, fieldSet[m], "category %q: metric %q not found", category, m) + } + log.Printf("Op060: category %q OK", category) + } + hasPower := fieldSet["gpu_package_power"] || fieldSet["gpu_average_package_power"] + assert.True(c, hasPower, "power: no power metric (expected gpu_package_power or gpu_average_package_power)") + log.Printf("Op060: category %q OK", "power") + hasTemp := fieldSet["gpu_junction_temperature"] || fieldSet["gpu_edge_temperature"] + assert.True(c, hasTemp, "temperature: no temperature metric (expected gpu_junction_temperature or gpu_edge_temperature)") + log.Printf("Op060: category %q OK", "temperature") + log.Printf("Op060: %d total metric families verified", len(fields)) +} + +// TestOp065VerifyPartitionedGPUMetrics validates partition labels and per-partition +// metrics when partitioned GPUs are detected (SPX/DPX/TPX/QPX/CPX). +// Automatically skipped on non-partitioned clusters (e.g. Radeon cards). +func (s *E2ESuite) TestOp065VerifyPartitionedGPUMetrics(c *C) { + ctx := context.Background() + + pods, err := s.k8sclient.GetPodsByLabel(ctx, s.ns, map[string]string{ + "app.kubernetes.io/name": "metrics-exporter", + }) + assert.NoError(c, err) + assert.True(c, len(pods) > 0, "no metrics-exporter pods found") + + var dmePod *corev1.Pod + for i := range pods { + if pods[i].Status.Phase == corev1.PodRunning { + dmePod = &pods[i] + break + } + } + if dmePod == nil { + assert.Fail(c, "no Running metrics-exporter pod found") + return + } + + var output string + assert.Eventually(c, func() bool { + var err error + output, err = s.k8sclient.ExecCmdOnPod(ctx, s.restConfig, dmePod, "", + fmt.Sprintf("curl -s localhost:%d/metrics", dmeMetricsPort)) + if err != nil { + log.Printf("Op065: waiting for DME metrics: %v", err) + return false + } + return len(output) > 0 + }, 90*time.Second, 5*time.Second, "DME metrics endpoint did not respond within 90s") + + validComputePartitionTypes := map[string]bool{ + "SPX": true, "DPX": true, "TPX": true, "QPX": true, "CPX": true, + } + + type partitionKey struct { + gpuID, partitionID, computePartition, memoryPartition string + } + partitionedGPUs := map[partitionKey]bool{} + perPartitionMetrics := map[string]map[string]bool{} + + for _, line := range strings.Split(output, "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "#") || line == "" { + continue + } + lbStart := strings.Index(line, "{") + lbEnd := strings.Index(line, "}") + if lbStart < 0 || lbEnd < 0 || lbEnd <= lbStart { + continue + } + metricName := line[:lbStart] + lbls := map[string]string{} + for _, part := range strings.Split(line[lbStart+1:lbEnd], ",") { + part = strings.TrimSpace(part) + eqIdx := strings.Index(part, "=") + if eqIdx < 0 { + continue + } + lbls[strings.TrimSpace(part[:eqIdx])] = strings.Trim(strings.TrimSpace(part[eqIdx+1:]), `"`) + } + + partID := lbls["gpu_partition_id"] + computeType := strings.ToUpper(lbls["gpu_compute_partition_type"]) + if partID == "" || computeType == "" || computeType == "NONE" { + continue + } + pk := partitionKey{ + gpuID: lbls["gpu_id"], + partitionID: partID, + computePartition: computeType, + memoryPartition: strings.ToUpper(lbls["gpu_memory_partition_type"]), + } + partitionedGPUs[pk] = true + for _, pm := range []string{"gpu_gfx_busy_instantaneous", "gpu_total_vram", "gpu_used_vram"} { + if metricName == pm { + if perPartitionMetrics[pm] == nil { + perPartitionMetrics[pm] = map[string]bool{} + } + perPartitionMetrics[pm][partID] = true + } + } + } + + if len(partitionedGPUs) == 0 { + log.Print("Op065: no partitioned GPUs detected — skipping partition-specific validation") + c.Skip("no partitioned GPUs detected (gpu_compute_partition_type=none for all GPUs)") + return + } + log.Printf("Op065: detected %d partitioned GPU instance(s)", len(partitionedGPUs)) + + allPartitionIDs := map[string]bool{} + for pk := range partitionedGPUs { + log.Printf("Op065: GPU %s partition_id=%s compute_type=%s memory_type=%s", + pk.gpuID, pk.partitionID, pk.computePartition, pk.memoryPartition) + assert.True(c, isNonNegativeInt(pk.partitionID), + "GPU %s: gpu_partition_id %q is not a non-negative integer", pk.gpuID, pk.partitionID) + assert.True(c, validComputePartitionTypes[pk.computePartition], + "GPU %s partition %s: unexpected compute partition type %q", pk.gpuID, pk.partitionID, pk.computePartition) + assert.True(c, pk.memoryPartition != "" && pk.memoryPartition != "NONE", + "GPU %s partition %s: gpu_memory_partition_type is %q (expected non-empty, non-NONE)", + pk.gpuID, pk.partitionID, pk.memoryPartition) + allPartitionIDs[pk.partitionID] = true + } + + for _, metricName := range []string{"gpu_gfx_busy_instantaneous", "gpu_total_vram", "gpu_used_vram"} { + for pid := range allPartitionIDs { + present := perPartitionMetrics[metricName] != nil && perPartitionMetrics[metricName][pid] + assert.True(c, present, "per-partition metric %q not found for partition_id=%s", metricName, pid) + } + log.Printf("Op065: per-partition metric %q present for all %d partition(s)", metricName, len(allPartitionIDs)) + } + log.Printf("Op065: partition validation passed for %d GPU partition(s)", len(partitionedGPUs)) +} + +// ---- Op070: GPU workload ------------------------------------------------ + +// TestOp070ScheduleGPUWorkload submits a GPU workload pod (rocminfo + amd-smi) requesting +// amd.com/gpu:1 and asserts it completes successfully, verifying GPU scheduling works end-to-end. +func (s *E2ESuite) TestOp070ScheduleGPUWorkload(c *C) { + ctx := context.Background() + + gpuNodes, err := s.k8sclient.GetNodesByLabel(ctx, map[string]string{ + "feature.node.kubernetes.io/amd-gpu": "true", + }) + assert.NoError(c, err) + assert.True(c, len(gpuNodes) > 0, "no GPU nodes available for workload scheduling") + + log.Printf("Op070: submitting GPU workload pod %s in namespace %s", gpuWorkloadPodName, s.ns) + _ = s.k8sclient.DeletePod(ctx, s.ns, gpuWorkloadPodName) + _ = s.k8sclient.WaitForPodDeleted(ctx, s.ns, gpuWorkloadPodName, 30*time.Second) + + // Use a lightweight ROCm image rather than a full PyTorch image. + // rocm/rocm-terminal has rocminfo + amd-smi pre-installed and is ~2 GB vs ~40 GB for + // full pytorch images. On k3s Docker-mode clusters /dev/kfd is not automatically + // passed by the device plugin, so we mount it via a hostPath volume and run + // privileged to ensure the GPU is accessible. + kfdHostPath := corev1.HostPathType("") + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: gpuWorkloadPodName, + Namespace: s.ns, + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + Volumes: []corev1.Volume{ + { + Name: "kfd", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/dev/kfd", + Type: &kfdHostPath, + }, + }, + }, + }, + Containers: []corev1.Container{ + { + Name: "rocm-workload", + Image: *workloadImage, + Command: []string{"/bin/bash", "-c"}, + Args: []string{ + "rocminfo | grep -E 'Name|gfx' | head -10 && " + + "amd-smi list && " + + "echo 'ROCm available: True' && " + + "echo 'DONE'", + }, + SecurityContext: &corev1.SecurityContext{ + Privileged: func() *bool { b := true; return &b }(), + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: "kfd", MountPath: "/dev/kfd"}, + }, + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + "amd.com/gpu": resource.MustParse("1"), + }, + }, + }, + }, + Tolerations: []corev1.Toleration{ + {Operator: corev1.TolerationOpExists}, + }, + }, + } + + err = s.k8sclient.CreatePod(ctx, s.ns, pod) + assert.NoError(c, err, "failed to create GPU workload pod") + + log.Print("Op070: waiting for GPU workload pod to complete (timeout=15m)") + err = s.k8sclient.WaitForPodSucceeded(ctx, s.ns, gpuWorkloadPodName, 15*time.Minute) + assert.NoError(c, err, "GPU workload pod did not succeed") + + logs, logErr := s.k8sclient.GetPodLogs(ctx, s.ns, gpuWorkloadPodName) + if logErr == nil { + log.Printf("Op070: workload logs:\n%s", logs) + assert.True(c, strings.Contains(logs, "DONE"), "expected 'DONE' in workload output") + assert.True(c, strings.Contains(logs, "ROCm available: True"), "expected ROCm to be available") + } + _ = s.k8sclient.DeletePod(ctx, s.ns, gpuWorkloadPodName) +} + +// ---- Op900: teardown ---------------------------------------------------- + +// TestOp900TearDownOperator uninstalls the GPU Operator and cert-manager. +func (s *E2ESuite) TestOp900TearDownOperator(c *C) { + if s.existingDeploy { + c.Skip("skipping teardown: existing deploy mode") + } + log.Print("Op900: uninstalling GPU Operator") + if err := s.helmClient.UninstallChartByName(operatorReleaseName); err != nil { + log.Printf("Op900: warning — GPU Operator uninstall: %v", err) + } + log.Print("Op900: uninstalling cert-manager") + if err := s.helmClient.UninstallChartByName(certManagerReleaseName); err != nil { + log.Printf("Op900: warning — cert-manager uninstall: %v", err) + } + log.Print("Op900: teardown complete") +} + +// ---- helpers ------------------------------------------------------------ + +// isNonNegativeInt returns true if s is a string representation of a non-negative integer. +func isNonNegativeInt(s string) bool { + if s == "" { + return false + } + for _, ch := range s { + if ch < '0' || ch > '9' { + return false + } + } + return true +} diff --git a/tests/k8s-e2e/suite_test.go b/tests/k8s-e2e/suite_test.go new file mode 100644 index 000000000..5ff3e2923 --- /dev/null +++ b/tests/k8s-e2e/suite_test.go @@ -0,0 +1,167 @@ +/* +Copyright (c) Advanced Micro Devices, Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// GPU Operator e2e tests. +// +// This package contains GPU Operator lifecycle tests (install, infra verification, +// GPU workload, teardown) that exercise the full operator stack with DME deployed +// as a managed DaemonSet. +// +// Run full install+verify+teardown: +// +// go test -v -failfast \ +// -operatorchart helm-charts-k8s \ +// -operatortag v1.4.1 \ +// -kubeconfig /path/to/kubeconfig \ +// -test.timeout 60m +// +// Run verify only against a pre-deployed cluster: +// +// go test -v -failfast -existing \ +// -kubeconfig /path/to/kubeconfig \ +// -check.f 'TestOp010|TestOp020|TestOp030|TestOp040|TestOp050|TestOp060|TestOp065|TestOp070' \ +// -test.timeout 30m + +package gpuope2e + +import ( + "context" + "flag" + "fmt" + "log" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/ROCm/gpu-operator/tests/k8s-e2e/clients" + "github.com/stretchr/testify/assert" + . "gopkg.in/check.v1" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/util/homedir" +) + +// sliceFlag is a repeatable string flag (used for -helmset). +type sliceFlag []string + +func (f *sliceFlag) String() string { return fmt.Sprintf("%v", []string(*f)) } +func (f *sliceFlag) Set(v string) error { + *f = append(*f, v) + return nil +} + +var kubeConfig = flag.String("kubeconfig", filepath.Join(homedir.HomeDir(), ".kube", "config"), "absolute path to the kubeconfig file") +var operatorNS = flag.String("namespace", "kube-amd-gpu", "namespace for GPU Operator deployment") +var operatorChart = flag.String("operatorchart", "helm-charts-k8s", "GPU Operator helm chart (local path, OCI ref, or repo/chart)") +var operatorTag = flag.String("operatortag", "v1.4.1", "GPU Operator chart version/tag") +var existingDeploy = flag.Bool("existing", false, "when true, skip install/teardown (verify only)") +var noTeardown = flag.Bool("noteardown", false, "when true, skip TearDownSuite namespace deletion (leave operator installed)") +var workloadImage = flag.String("workloadimage", "rocm/rocm-terminal:latest", "ROCm workload image for Op070 GPU workload test") +var helmSet sliceFlag + +func init() { + flag.Var(&helmSet, "helmset", "extra helm --set override (repeatable, e.g. -helmset foo=bar)") +} + +// Hook up gocheck into the "go test" runner. +func Test(t *testing.T) { + TestingT(t) +} + +var suite = &E2ESuite{} +var _ = Suite(suite) + +func (s *E2ESuite) SetUpSuite(c *C) { + log.Print("SetUpSuite:") + s.kubeconfig = *kubeConfig + s.ns = *operatorNS + s.operatorChart = *operatorChart + s.operatorTag = *operatorTag + s.existingDeploy = *existingDeploy + s.helmSet = []string(helmSet) + ctx := context.Background() + + config, err := clientcmd.BuildConfigFromFlags("", s.kubeconfig) + c.Assert(err, IsNil) + s.restConfig = config + + cs, err := clients.NewK8sClient(config) + c.Assert(err, IsNil) + s.k8sclient = cs + + hClient, err := clients.NewHelmClient( + clients.WithNameSpaceOption(s.ns), + clients.WithKubeConfigOption(config), + ) + c.Assert(err, IsNil) + s.helmClient = hClient + + if s.existingDeploy { + log.Printf("SetUpSuite: existing deploy mode — skipping namespace delete/create (ns=%s)", s.ns) + } else { + log.Printf("SetUpSuite: deleting namespace %s (if exists)", s.ns) + if err = s.k8sclient.DeleteNamespaceAndWait(ctx, s.ns, "", 3*time.Minute); err != nil { + log.Printf("SetUpSuite: namespace delete/wait: %v (continuing)", err) + } + // Clean up cluster-scoped resources from any previous operator or cert-manager release + // (including differently-named releases like "amd-gpu-operator"). + for _, oldRelease := range []string{operatorReleaseName, "amd-gpu-operator", certManagerReleaseName} { + log.Printf("SetUpSuite: cleaning up cluster-scoped resources for release %q", oldRelease) + s.k8sclient.CleanupClusterScopedResources(ctx, oldRelease) + } + // Also delete the cert-manager namespace if it exists (prevents install conflicts). + if s.k8sclient.NamespaceExists(ctx, "cert-manager") { + log.Print("SetUpSuite: deleting cert-manager namespace (leftover from previous run)") + if err = s.k8sclient.DeleteNamespaceAndWait(ctx, "cert-manager", "", 3*time.Minute); err != nil { + log.Printf("SetUpSuite: cert-manager namespace delete: %v (continuing)", err) + } + } + // Delete cert-manager namespace-scoped Roles in kube-system (these lack helm labels + // after a raw kubectl delete of the cert-manager namespace, preventing helm reinstall). + s.k8sclient.DeleteCertManagerKubeSystemRoles(ctx) + err = s.k8sclient.CreateNamespace(ctx, s.ns) + assert.NoError(c, err) + } + + if s.suiteHook != nil { + if err := s.suiteHook(ctx); err != nil { + assert.NoError(c, err, "suite hook failed") + } + } +} + +func (s *E2ESuite) TearDownSuite(c *C) { + log.Print("TearDownSuite:") + if !s.existingDeploy && !*noTeardown { + log.Printf("TearDownSuite: deleting namespace %s", s.ns) + if err := s.k8sclient.DeleteNamespaceAndWait(context.Background(), s.ns, "", 3*time.Minute); err != nil { + log.Printf("TearDownSuite: namespace delete: %v", err) + } + // Clean up cluster-scoped resources left by the operator helm release. + s.k8sclient.CleanupClusterScopedResources(context.Background(), operatorReleaseName) + } else if *noTeardown { + log.Print("TearDownSuite: skipping teardown (-noteardown flag set)") + } + if s.helmClient != nil { + s.helmClient.Cleanup() + } +} + +// operatorReleaseName is the helm release name used for the GPU Operator. +const operatorReleaseName = "e2e-gpu-operator" + +// helmSetJoin joins -helmset values into a log-friendly string. +func helmSetJoin(vals []string) string { return strings.Join(vals, ", ") }