diff --git a/.claude/projects/-home-ploffay-projects-openshift-cluster-logging-operator/memory/MEMORY.md b/.claude/projects/-home-ploffay-projects-openshift-cluster-logging-operator/memory/MEMORY.md new file mode 100644 index 0000000000..c493a2fd6b --- /dev/null +++ b/.claude/projects/-home-ploffay-projects-openshift-cluster-logging-operator/memory/MEMORY.md @@ -0,0 +1 @@ +- [CLO OTEL Migration](project_clo_otel_migration.md) — Migrating CLO from Vector to OTEL collector, multi-milestone proposal diff --git a/.claude/projects/-home-ploffay-projects-openshift-cluster-logging-operator/memory/project_clo_otel_migration.md b/.claude/projects/-home-ploffay-projects-openshift-cluster-logging-operator/memory/project_clo_otel_migration.md new file mode 100644 index 0000000000..0387a8667e --- /dev/null +++ b/.claude/projects/-home-ploffay-projects-openshift-cluster-logging-operator/memory/project_clo_otel_migration.md @@ -0,0 +1,17 @@ +--- +name: clo-otel-migration +description: Working on proposal to migrate CLO from Vector collector to OpenTelemetry collector - milestone 1 replaces Vector binary with OTEL collector +metadata: + type: project +--- + +User is working on a migration proposal to replace Vector with the OpenTelemetry collector in the Cluster Logging Operator (CLO). + +**Why:** CLO currently uses Vector (Datadog) as the log collection/forwarding engine. The goal is to migrate to the OTEL collector to align with OpenTelemetry standards and potentially consolidate with the OpenTelemetry Operator. + +**How to apply:** When working on CLO code or the migration proposal, understand that: +- Milestone 1: Replace Vector binary with OTEL collector in CLO-managed pods, generate OTEL collector YAML config instead of Vector TOML +- Milestone 2: Use OTEL collector CR instead of CLO-managed deployment +- Milestone 3: Full migration to OpenTelemetry Operator +- The ClusterLogForwarder CRD API stays the same in milestone 1 +- Key files: api/observability/v1/ (CRD types), internal/generator/vector/ (Vector config generation) diff --git a/Makefile b/Makefile index a10c351602..b4c606e782 100644 --- a/Makefile +++ b/Makefile @@ -29,6 +29,7 @@ export NAMESPACE?=openshift-logging export LOKI_OPERATOR_CHANNEL?=stable-6.4 IMAGE_LOGGING_VECTOR?=quay.io/openshift-logging/vector:v0.54.0 +IMAGE_OTEL_COLLECTOR?=ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:0.127.0 IMAGE_LOGFILEMETRICEXPORTER?=quay.io/openshift-logging/log-file-metric-exporter:latest IMAGE_LOGGING_EVENTROUTER?=quay.io/openshift-logging/eventrouter:v0.5.0 IMAGE_TLS_SCANNER?=quay.io/openshift/tls-scanner:latest @@ -129,6 +130,7 @@ run: @mkdir -p $(CURDIR)/tmp LOG_LEVEL=$(LOG_LEVEL) \ RELATED_IMAGE_VECTOR=$(IMAGE_LOGGING_VECTOR) \ + RELATED_IMAGE_OTEL_COLLECTOR=$(IMAGE_OTEL_COLLECTOR) \ RELATED_IMAGE_LOG_FILE_METRIC_EXPORTER=$(IMAGE_LOGFILEMETRICEXPORTER) \ OPERATOR_NAME=$(OPERATOR_NAME) \ WATCH_NAMESPACE="" \ @@ -230,12 +232,14 @@ deploy-catalog: test-env: ## Echo test environment, useful for running tests outside of the Makefile. @echo \ RELATED_IMAGE_VECTOR=$(IMAGE_LOGGING_VECTOR) \ + RELATED_IMAGE_OTEL_COLLECTOR=$(IMAGE_OTEL_COLLECTOR) \ RELATED_IMAGE_LOG_FILE_METRIC_EXPORTER=$(IMAGE_LOGFILEMETRICEXPORTER) \ IMAGE_TLS_SCANNER=$(IMAGE_TLS_SCANNER) \ .PHONY: test-functional test-functional: test-functional-benchmarker-vector RELATED_IMAGE_VECTOR=$(IMAGE_LOGGING_VECTOR) \ + RELATED_IMAGE_OTEL_COLLECTOR=$(IMAGE_OTEL_COLLECTOR) \ RELATED_IMAGE_LOG_FILE_METRIC_EXPORTER=$(IMAGE_LOGFILEMETRICEXPORTER) \ go test -race \ ./test/functional/... \ @@ -258,6 +262,7 @@ test-functional-benchmarker-vector: bin/functional-benchmarker .PHONY: test-unit test-unit: test-forwarder-generator test-unit-api RELATED_IMAGE_VECTOR=$(IMAGE_LOGGING_VECTOR) \ + RELATED_IMAGE_OTEL_COLLECTOR=$(IMAGE_OTEL_COLLECTOR) \ RELATED_IMAGE_LOG_FILE_METRIC_EXPORTER=$(IMAGE_LOGFILEMETRICEXPORTER) \ go test -coverprofile=test.cov -race ./api/... ./internal/... `go list ./test/... | grep -Ev 'test/(e2e|functional|framework|client|helpers)'` @@ -319,6 +324,7 @@ apply: namespace $(OPERATOR_SDK) ## Install kustomized resources directly to the .PHONY: test-upgrade test-upgrade: $(JUNITREPORT) RELATED_IMAGE_VECTOR=$(IMAGE_LOGGING_VECTOR) \ + RELATED_IMAGE_OTEL_COLLECTOR=$(IMAGE_OTEL_COLLECTOR) \ RELATED_IMAGE_LOG_FILE_METRIC_EXPORTER=$(IMAGE_LOGFILEMETRICEXPORTER) \ IMAGE_LOGGING_EVENTROUTER=$(IMAGE_LOGGING_EVENTROUTER) \ exit 0 @@ -326,6 +332,7 @@ test-upgrade: $(JUNITREPORT) .PHONY: test-e2e test-e2e: $(JUNITREPORT) RELATED_IMAGE_VECTOR=$(IMAGE_LOGGING_VECTOR) \ + RELATED_IMAGE_OTEL_COLLECTOR=$(IMAGE_OTEL_COLLECTOR) \ RELATED_IMAGE_LOG_FILE_METRIC_EXPORTER=$(IMAGE_LOGFILEMETRICEXPORTER) \ IMAGE_LOGGING_EVENTROUTER=$(IMAGE_LOGGING_EVENTROUTER) \ IMAGE_TLS_SCANNER=$(IMAGE_TLS_SCANNER) \ @@ -336,6 +343,7 @@ test-e2e-local: $(JUNITREPORT) deploy-image LOG_LEVEL=3 \ LOKI_OPERATOR_CHANNEL=$(LOKI_OPERATOR_CHANNEL) \ RELATED_IMAGE_VECTOR=$(IMAGE_LOGGING_VECTOR) \ + RELATED_IMAGE_OTEL_COLLECTOR=$(IMAGE_OTEL_COLLECTOR) \ RELATED_IMAGE_LOG_FILE_METRIC_EXPORTER=$(IMAGE_LOGFILEMETRICEXPORTER) \ IMAGE_LOGGING_EVENTROUTER=$(IMAGE_LOGGING_EVENTROUTER) \ IMAGE_TLS_SCANNER=$(IMAGE_TLS_SCANNER) \ diff --git a/internal/collector/collector.go b/internal/collector/collector.go index 47ac59a064..305821aaa9 100644 --- a/internal/collector/collector.go +++ b/internal/collector/collector.go @@ -14,6 +14,7 @@ import ( configv1 "github.com/openshift/api/config/v1" obs "github.com/openshift/cluster-logging-operator/api/observability/v1" internalobs "github.com/openshift/cluster-logging-operator/internal/api/observability" + "github.com/openshift/cluster-logging-operator/internal/collector/otel" "github.com/openshift/cluster-logging-operator/internal/collector/vector" "github.com/openshift/cluster-logging-operator/internal/constants" "github.com/openshift/cluster-logging-operator/internal/factory" @@ -106,29 +107,40 @@ func New(confHash, clusterID string, collectorSpec *obs.CollectorSpec, secrets i if collectorSpec == nil { collectorSpec = &obs.CollectorSpec{} } - factory := &Factory{ + + imageName := constants.VectorName + visit := Visitor(vector.CollectorVisitor) + podLabelVisitor := PodLabelVisitor(vector.PodLogExcludeLabel) + + if annotations[constants.AnnotationCollectorType] == constants.OTELCollectorName { + imageName = constants.OTELCollectorName + visit = otel.CollectorVisitor + podLabelVisitor = otel.PodLogExcludeLabel + } + + f := &Factory{ ClusterID: clusterID, ConfigHash: confHash, CollectorSpec: *collectorSpec, - ImageName: constants.VectorName, - Visit: vector.CollectorVisitor, + ImageName: imageName, + Visit: visit, ConfigMaps: configMaps, Secrets: secrets, ForwarderSpec: forwarderSpec, CommonLabelInitializer: func(o runtime.Object) { - runtime.SetCommonLabels(o, constants.VectorName, resNames.ForwarderName, constants.CollectorName) + runtime.SetCommonLabels(o, imageName, resNames.ForwarderName, constants.CollectorName) }, ResourceNames: resNames, - PodLabelVisitor: vector.PodLogExcludeLabel, + PodLabelVisitor: podLabelVisitor, isDaemonset: isDaemonset, annotations: annotations, } - return factory + return f } func (f *Factory) NewDaemonSet(namespace, name string, trustedCABundle *v1.ConfigMap, tlsProfileSpec configv1.TLSProfileSpec) *apps.DaemonSet { podSpec := f.NewPodSpec(trustedCABundle, f.ForwarderSpec, f.ClusterID, tlsProfileSpec, namespace) - ds := factory.NewDaemonSet(namespace, name, name, constants.CollectorName, constants.VectorName, f.MaxUnavailable(), *podSpec, f.CommonLabelInitializer, f.PodLabelVisitor) + ds := factory.NewDaemonSet(namespace, name, name, constants.CollectorName, f.ImageName, f.MaxUnavailable(), *podSpec, f.CommonLabelInitializer, f.PodLabelVisitor) ds.Spec.Template.Annotations[constants.AnnotationSecretHash] = f.Secrets.Hash64a() ds.Spec.Template.Annotations[constants.AnnotationConfigMapHash] = f.ConfigMaps.Hash64a() return ds @@ -136,7 +148,7 @@ func (f *Factory) NewDaemonSet(namespace, name string, trustedCABundle *v1.Confi func (f *Factory) NewDeployment(namespace, name string, trustedCABundle *v1.ConfigMap, tlsProfileSpec configv1.TLSProfileSpec) *apps.Deployment { podSpec := f.NewPodSpec(trustedCABundle, f.ForwarderSpec, f.ClusterID, tlsProfileSpec, namespace) - dpl := factory.NewDeployment(namespace, name, constants.CollectorName, constants.VectorName, 2, *podSpec, f.CommonLabelInitializer, f.PodLabelVisitor) + dpl := factory.NewDeployment(namespace, name, constants.CollectorName, f.ImageName, 2, *podSpec, f.CommonLabelInitializer, f.PodLabelVisitor) dpl.Spec.Template.Annotations[constants.AnnotationSecretHash] = f.Secrets.Hash64a() dpl.Spec.Template.Annotations[constants.AnnotationConfigMapHash] = f.ConfigMaps.Hash64a() return dpl diff --git a/internal/collector/config.go b/internal/collector/config.go index b3d90bf8fa..3b264cbe1f 100644 --- a/internal/collector/config.go +++ b/internal/collector/config.go @@ -2,8 +2,11 @@ package collector import ( "fmt" + log "github.com/ViaQ/logerr/v2/log/static" + "github.com/openshift/cluster-logging-operator/internal/collector/otel" "github.com/openshift/cluster-logging-operator/internal/collector/vector" + "github.com/openshift/cluster-logging-operator/internal/constants" "github.com/openshift/cluster-logging-operator/internal/reconcile" "github.com/openshift/cluster-logging-operator/internal/runtime" "github.com/openshift/cluster-logging-operator/internal/utils" @@ -12,16 +15,25 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +func (f *Factory) configFileData(namespace, collectorConfig string) map[string]string { + if f.ImageName == constants.OTELCollectorName { + return map[string]string{ + otel.ConfigFile: collectorConfig, + } + } + return map[string]string{ + vector.ConfigFile: collectorConfig, + vector.RunVectorFile: fmt.Sprintf(vector.RunVectorScript, vector.GetDataPath(namespace, f.ResourceNames.ForwarderName)), + } +} + // ReconcileCollectorConfig reconciles a collector config specifically for the collector defined by the factory func (f *Factory) ReconcileCollectorConfig(k8sClient client.Client, reader client.Reader, namespace, collectorConfig string, owner metav1.OwnerReference) error { log.V(3).Info("Updating ConfigMap and Secrets") configMap := runtime.NewConfigMap( namespace, f.ResourceNames.ConfigMap, - map[string]string{ - vector.ConfigFile: collectorConfig, - vector.RunVectorFile: fmt.Sprintf(vector.RunVectorScript, vector.GetDataPath(namespace, f.ResourceNames.ForwarderName)), - }, + f.configFileData(namespace, collectorConfig), f.CommonLabelInitializer) utils.AddOwnerRefToObject(configMap, owner) diff --git a/internal/collector/otel/utils.go b/internal/collector/otel/utils.go new file mode 100644 index 0000000000..7fabbbfd02 --- /dev/null +++ b/internal/collector/otel/utils.go @@ -0,0 +1,20 @@ +package otel + +import ( + "path" + + "github.com/openshift/cluster-logging-operator/internal/constants" +) + +const ( + ConfigFile = "config.yaml" + DefaultDataPath = "/var/lib/otelcol" + configPath = "/etc/otelcol" +) + +func GetDataPath(namespace, forwarderName string) string { + if constants.OpenshiftNS == namespace && constants.SingletonName == forwarderName { + return DefaultDataPath + } + return path.Join(DefaultDataPath, namespace, forwarderName) +} diff --git a/internal/collector/otel/visitors.go b/internal/collector/otel/visitors.go new file mode 100644 index 0000000000..8796aab266 --- /dev/null +++ b/internal/collector/otel/visitors.go @@ -0,0 +1,33 @@ +package otel + +import ( + "github.com/openshift/cluster-logging-operator/internal/collector/common" + "github.com/openshift/cluster-logging-operator/internal/factory" + "github.com/openshift/cluster-logging-operator/internal/runtime" + corev1 "k8s.io/api/core/v1" +) + +func CollectorVisitor(collectorContainer *corev1.Container, podSpec *corev1.PodSpec, resNames *factory.ForwarderResourceNames, namespace, logLevel string) { + collectorContainer.Env = append(collectorContainer.Env, + corev1.EnvVar{Name: "OTEL_LOG_LEVEL", Value: logLevel}, + ) + + dataPath := GetDataPath(namespace, resNames.ForwarderName) + collectorContainer.VolumeMounts = append(collectorContainer.VolumeMounts, + corev1.VolumeMount{Name: common.ConfigVolumeName, ReadOnly: true, MountPath: configPath}, + corev1.VolumeMount{Name: common.DataDir, ReadOnly: false, MountPath: dataPath}, + ) + + collectorContainer.Command = []string{"/otelcol-contrib"} + collectorContainer.Args = []string{"--config=" + configPath + "/" + ConfigFile} + + podSpec.Volumes = append(podSpec.Volumes, + corev1.Volume{Name: common.ConfigVolumeName, VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{LocalObjectReference: corev1.LocalObjectReference{Name: resNames.ConfigMap}}}}, + corev1.Volume{Name: common.DataDir, VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: dataPath}}}, + ) +} + +func PodLogExcludeLabel(o runtime.Object) { + // OTEL collector's filelog receiver uses exclude patterns in config rather than pod labels. + // No pod label needed for self-exclusion. +} diff --git a/internal/constants/annotations.go b/internal/constants/annotations.go index b50c6b2b3b..c03922d21d 100644 --- a/internal/constants/annotations.go +++ b/internal/constants/annotations.go @@ -19,4 +19,7 @@ const ( // AnnotationMaxUnavailable (Deprecated) configures the maximum number of DaemonSet pods that can be unavailable during a rolling update. // This can be an absolute number (e.g., 1) or a percentage (e.g., 10%). Default is 100%. AnnotationMaxUnavailable = "observability.openshift.io/max-unavailable-rollout" + + // AnnotationCollectorType selects the collector implementation. Supported values: "vector" (default), "otelcol". + AnnotationCollectorType = "logging.openshift.io/dev-preview-collector-type" ) diff --git a/internal/constants/constants.go b/internal/constants/constants.go index a6a8bd6c0a..52eb4eca91 100644 --- a/internal/constants/constants.go +++ b/internal/constants/constants.go @@ -44,6 +44,7 @@ const ( TrustedCABundleMountDir = "/etc/pki/ca-trust/extracted/pem/" ElasticsearchName = "elasticsearch" VectorName = "vector" + OTELCollectorName = "otelcol" KibanaName = "kibana" LogfilesmetricexporterName = "logfilesmetricexporter" LogfilesmetricexporterPort = int32(2112) @@ -63,7 +64,8 @@ const ( CollectorServiceAccountName = "logcollector" CollectorTrustedCAName = "collector-trusted-ca-bundle" - VectorImageEnvVar = "RELATED_IMAGE_VECTOR" + VectorImageEnvVar = "RELATED_IMAGE_VECTOR" + OTELCollectorImageEnvVar = "RELATED_IMAGE_OTEL_COLLECTOR" LogfilesmetricImageEnvVar = "RELATED_IMAGE_LOG_FILE_METRIC_EXPORTER" ContainerLogDir = "/var/log/containers" diff --git a/internal/controller/observability/collector.go b/internal/controller/observability/collector.go index 6b3a2ae92f..af2ab45f1f 100644 --- a/internal/controller/observability/collector.go +++ b/internal/controller/observability/collector.go @@ -179,6 +179,11 @@ func GenerateConfig(k8Client client.Client, clf obs.ClusterLogForwarder, resourc tlsProfile, _ := tls.FetchAPIServerTlsProfile(k8Client) op[framework.ClusterTLSProfileSpec] = tls.GetClusterTLSProfileSpec(tlsProfile) EvaluateAnnotationsForEnabledCapabilities(clf.Annotations, op) + + if clf.Annotations[constants.AnnotationCollectorType] == constants.OTELCollectorName { + return generateOTELConfig(clf.Spec) + } + g := forwardergenerator.New() generatedConfig, err := g.GenerateConf(secrets, clf.Spec, clf.Namespace, clf.Name, resourceNames, op) @@ -191,6 +196,40 @@ func GenerateConfig(k8Client client.Client, clf obs.ClusterLogForwarder, resourc return generatedConfig, err } +// generateOTELConfig returns a minimal OTEL collector configuration. +// TODO: Replace with full config generation from CLF spec. +func generateOTELConfig(_ obs.ClusterLogForwarderSpec) (string, error) { + return `receivers: + filelog: + include: + - /var/log/pods/*/*/*.log + exclude: + - /var/log/pods/openshift-logging_collector-*/*/*.log + start_at: end + include_file_path: true + include_file_name: false + operators: + - type: container + id: container-parser + +processors: + batch: + send_batch_size: 8192 + timeout: 2s + +exporters: + debug: + verbosity: basic + +service: + pipelines: + logs: + receivers: [filelog] + processors: [batch] + exporters: [debug] +`, nil +} + // EvaluateAnnotationsForEnabledCapabilities populates generator options with capabilities enabled by the ClusterLogForwarder func EvaluateAnnotationsForEnabledCapabilities(annotations map[string]string, options framework.Options) { if annotations == nil { diff --git a/internal/utils/utils.go b/internal/utils/utils.go index a355990561..07e2d30f57 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -31,6 +31,7 @@ var ( // COMPONENT_IMAGES are keys based on the "container name" + "-{image,version}" var COMPONENT_IMAGES = map[string]string{ constants.VectorName: constants.VectorImageEnvVar, + constants.OTELCollectorName: constants.OTELCollectorImageEnvVar, constants.LogfilesmetricexporterName: constants.LogfilesmetricImageEnvVar, } diff --git a/opentelemetry-collector-migration.md b/opentelemetry-collector-migration.md new file mode 100644 index 0000000000..e0fed58e90 --- /dev/null +++ b/opentelemetry-collector-migration.md @@ -0,0 +1,197 @@ +# OpenTelemetry collector/operator migration + +This document outlines the migration steps of CLO to the OpenTelemetry collector and operator. + +The migration will be done in milestones: +1. Use OpenTelemetry collector in CLO instead of the current vector collector +2. Use OpenTelemetry collector CR in CLO instead of the current vector deployment. +3. Brainstorm the migration path of CLO to pure OpenTelemetry operator. + +--- + +## Milestone 1: Replace Vector with OpenTelemetry Collector + +**Goal:** CLO continues to manage the collector deployment and reconcile the `ClusterLogForwarder` CR, but generates an OpenTelemetry Collector YAML configuration instead of a Vector TOML configuration. The deployed pod runs the OTEL collector binary instead of Vector. + +The `ClusterLogForwarder` CRD API remains unchanged. + +### Architecture Change + +``` +Current: ClusterLogForwarder CR → CLO → Vector TOML config → Vector DaemonSet +Target: ClusterLogForwarder CR → CLO → OTEL Collector YAML config → OTEL Collector DaemonSet +``` + +### Feature Compatibility Analysis + +#### 1. Inputs (Sources) → OTEL Receivers + +| CLO Input | Vector Source | OTEL Receiver | Status | Notes | +|-----------|-------------|---------------|--------|-------| +| `application` (container logs) | `kubernetes_logs` | `filelog` + `k8sattributes` processor | ✅ Supported | `filelog` reads from `/var/log/pods/` or `/var/log/containers/`. `k8sattributes` enriches with pod name, namespace, labels, annotations, UID, node name. Glob include/exclude patterns for namespace filtering are supported. Partial log merge (Docker JSON) handled by `filelog`'s multiline config. | +| `infrastructure` (container logs) | `kubernetes_logs` (filtered) | `filelog` + `k8sattributes` processor | ✅ Supported | Same as application but filtered to `openshift-*`, `kube-*`, `default` namespaces via include glob patterns. | +| `infrastructure` (node/journal) | `journald` | `journalctl` receiver | ✅ Supported | OTEL contrib `journalctl` receiver reads systemd journal. Supports directory config, unit filtering. | +| `audit` (auditd, kube-api, openshift-api, ovn) | `file` (multiple paths) | `filelog` (multiple instances) | ✅ Supported | `filelog` receiver watches specific file paths (`/var/log/audit/audit.log`, `/var/log/kube-apiserver/audit.log`, etc.). Supports `start_at`, `max_log_size`, glob patterns. | +| `receiver` (HTTP) | `http_server` (JSON decoding) | `httpreceiver` (contrib) | ⚠️ Needs verification | CLO's HTTP receiver accepts arbitrary JSON (specifically `kubeAPIAudit` format). The OTEL `httpreceiver` (contrib) or a webhook receiver would work. Alternatively, could use `otlpreceiver` if clients send OTLP. | +| `receiver` (Syslog) | `syslog` (TCP) | `syslog` receiver | ✅ Supported | OTEL contrib `syslog` receiver supports TCP/UDP, RFC3164/5424, TLS. | + +**Input-level tuning:** + +| Feature | Vector | OTEL Equivalent | Status | +|---------|--------|-----------------|--------| +| `MaxRecordsPerSecond` (per-container throttle) | `throttle` transform with `key_field: {{ _internal.file }}` | Elastic's [`ratelimitprocessor`](https://pkg.go.dev/github.com/elastic/opentelemetry-collector-components/processor/ratelimitprocessor); [contrib #35204](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/35204) accepted | ⚠️ Partial — not in official contrib yet | +| `MaxMessageSize` | `max_line_bytes` / `max_read_bytes` | `filelog` receiver `max_log_size` | ✅ Supported | +| `IgnoreOlder` (audit files) | `ignore_older_secs` on file source | `filelog` receiver `start_at: end` + poll interval | ⚠️ Partial | + +#### 2. Filters → OTEL Processors + +| CLO Filter | Vector Transform | OTEL Processor | Status | Notes | +|------------|-----------------|----------------|--------|-------| +| `drop` | `remap` with VRL conditions + `abort` | `filter` processor with OTTL | ✅ Supported | OTTL supports regex matching (`IsMatch`), boolean AND/OR logic, field access. The `filter` processor can drop log records matching conditions. | +| `prune` (field removal) | `remap` with VRL `del()` / field iteration | `transform` processor with OTTL `delete_key` / `keep_keys` | ✅ Supported | OTTL `delete_key`, `delete_matching_keys`, `keep_matching_keys` functions handle both `in` and `notIn` modes. | +| `parse` (JSON parsing) | `remap` with `parse_json()` | `transform` processor with OTTL `ParseJSON` or `json_parser` operator in `filelog` | ✅ Supported | Can parse JSON message body into structured attributes. | +| `openshiftLabels` | `remap` setting `._internal.openshift.labels` | `transform` or `attributes` processor | ✅ Supported | Set resource/log attributes from static values. | +| `detectMultilineException` | `detect_exceptions` transform | No built-in equivalent | ❌ Gap | Vector's `detect_exceptions` transform detects and merges multi-line stack traces (Java, Python, Go, Ruby, etc.) after collection. OTEL's `filelog` receiver has `multiline` config at ingestion, but post-ingestion exception detection across grouped streams is not available as a processor. A custom processor or the `groupby` processor with heuristics would be needed. | +| `kubeAPIAudit` | Complex VRL with policy rules, wildcards, verb matching, field redaction | `transform` processor with OTTL | ⚠️ Complex | The Kube API audit filter implements K8s audit policy logic: rule evaluation with wildcards in users/groups/namespaces/resources, verb matching, `OmitStages`, `OmitResponseCodes`, severity level setting (`None`/`Metadata`/`Request`/`RequestResponse`), field redaction (`requestObject`/`responseObject`). OTTL can handle conditions and field deletion, but the wildcard matching and rule precedence logic is complex. May need a custom processor or very extensive OTTL statements. | + +#### 3. Internal Normalization (VIAQ Data Model) → OTEL Processors + +CLO applies extensive normalization to all logs before forwarding. This is currently implemented as VRL transforms. + +| Feature | Vector VRL | OTEL Equivalent | Status | Notes | +|---------|-----------|-----------------|--------|-------| +| Envelope wrapping (`_internal`) | Wrap all fields in `._internal` | Not needed | ✅ N/A | OTEL's data model (resource attributes, log body, log attributes) provides natural separation. No envelope hack needed. | +| Log type classification | VRL namespace regex matching | `routing` processor or `transform` with OTTL | ✅ Supported | Route by namespace pattern to classify as `application` vs `infrastructure`. | +| Log source tagging | Set `.log_source` based on input | Resource attributes set by receiver pipeline | ✅ Supported | Each receiver pipeline naturally tags the source. | +| Cluster ID injection | `${OPENSHIFT_CLUSTER_ID}` env var | `resource` processor or `transform` | ✅ Supported | Add `openshift.cluster.uid` resource attribute from env var. | +| Hostname injection | `$VECTOR_SELF_NODE_NAME` env var | `resourcedetection` processor or `resource` processor | ✅ Supported | `resourcedetection` processor with `env` detector, or `k8snode` detector. | +| Sequence number | `to_unix_timestamp(now(), nanoseconds)` | Not directly equivalent | ⚠️ Minor | Could use observed timestamp or a custom processor. Sequence is used for ordering. | +| **Log level detection** (5-stage) | logfmt → klog → grok → pattern match → regex | `transform` processor with OTTL | ⚠️ Feasible | OTTL has [`ExtractGrokPatterns`](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/pkg/ottl/ottlfuncs/README.md) (uses Elastic Go-Grok library) which covers stage 3 (grok patterns for Logback, Log4j). Combined with OTTL regex (`IsMatch`) for stages 4-5, most detection is feasible. Stages 1-2 (logfmt, klog) have no dedicated OTTL parser but could be approximated with grok/regex patterns. Overall: achievable with OTTL, though the config will be verbose. | +| Dedot labels | Replace `.` and `/` with `_` in label keys | `transform` processor with OTTL `replace_all_patterns` | ✅ Supported | OTTL string functions can handle key normalization. | +| EventRouter log handling | Parse eventrouter JSON, extract event fields, fix timestamps | `transform` processor with OTTL | ⚠️ Complex | Detecting eventrouter pods, parsing nested JSON event structure, timestamp priority logic (lastTimestamp > firstTimestamp > eventTime > creationTimestamp). Doable with OTTL but complex. | +| Container I/O stream | Extract `stream` → `kubernetes.container_iostream` | `filelog` receiver attributes or `transform` | ✅ Supported | | + +**Key insight:** Many of the VIAQ normalization transforms exist to reshape Vector's internal data model into a standard format. With the OTEL collector, logs are natively in the OTLP data model (resource attributes, scope, log record with body/attributes/severity/timestamp/trace context). This eliminates much of the envelope/reshape logic. However, for non-OTLP outputs (Elasticsearch, Splunk, Syslog, etc.), the OTEL exporters must produce output compatible with what Vector currently emits (the VIAQ format), or consumers must accept the OTLP-based format. + +#### 4. Outputs (Sinks) → OTEL Exporters + +| CLO Output | Vector Sink | OTEL Exporter | Status | Notes | +|------------|------------|---------------|--------|-------| +| `otlp` | `opentelemetry` (HTTP) | `otlp` / `otlphttp` exporter | ✅ Native | This is the OTEL collector's primary export path. Supports gRPC and HTTP. Compression, auth, TLS, batch, retry all built-in. Currently CLO only uses HTTP; OTEL supports both. | +| `elasticsearch` | `elasticsearch` | `elasticsearch` exporter (contrib) | ✅ Supported | Supports bulk API, auth (basic, bearer). Dynamic index routing is the default since [v0.122.0](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/elasticsearchexporter): set `elasticsearch.index` attribute on log records via `transform` processor to route per-record. CLO's index templates map directly. ES 6 support needs verification (may be dropped). | +| `kafka` | `kafka` | `kafka` exporter (contrib) | ✅ Supported | SASL auth (PLAIN, SCRAM-SHA-256/512), TLS, compression (gzip, snappy, lz4, zstd), topic routing, multiple brokers. | +| `loki` | `loki` | `loki` exporter (contrib) | ✅ Supported | Labels, tenant ID, auth (basic, bearer), compression, out-of-order handling. Label key mapping from OTLP attributes is native. | +| `lokiStack` | Multiple `loki` or `otlp` sinks (per tenant) | `loki` exporter with `routing` processor | ✅ Supported | Route by log type (application/infrastructure/audit) to tenant-specific endpoints. LokiStack OTLP data model (`Otel`) maps directly. | +| `splunk` | `splunk_hec_logs` | `splunk_hec` exporter (contrib) | ✅ Supported | HEC token, index, source, sourcetype, host. Supports event format. The complex VRL for source/sourcetype detection would move to `transform` processor + exporter config. | +| `http` (generic JSON/NDJSON) | `http` | No generic HTTP log exporter | ❌ Gap | Vector's HTTP sink sends arbitrary JSON/NDJSON to any HTTP endpoint with configurable method, headers, auth. The OTEL collector has `otlphttp` (OTLP format only) but no generic HTTP JSON exporter. A [JSON log exporter was proposed](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/10836) but not implemented. **Needs a custom exporter.** | +| `syslog` | `socket` (TCP/UDP/TLS) | `syslog` exporter (contrib) | ✅ Supported | RFC3164/5424, TCP/UDP/TLS. Facility, severity, appname, procid, msgid mapping. The complex VRL transforms for syslog field construction would move to `transform` processor config. | +| `cloudwatch` | `aws_cloudwatch_logs` | `awscloudwatchlogs` exporter (contrib) | ✅ Supported | Region, log group (templated), log stream, AWS auth (access key, IAM role, STS). Batch limits (1MB CloudWatch API cap). | +| `s3` | `aws_s3` | `awss3` exporter (contrib) | ✅ Supported | Region, bucket, key prefix, custom endpoint for S3-compatible stores, AWS auth, compression. | +| `googleCloudLogging` | `gcp_stackdriver_logs` | `googlecloud` exporter (contrib) | ✅ Supported | Project/folder/org/billing account ID, log ID, severity mapping, GCP credentials. | +| `azureMonitor` (DEPRECATED) | `azure_monitor_logs` | `azuremonitor` exporter (contrib) | ⚠️ Deprecated | Since this output is deprecated in CLO, lower priority. OTEL contrib has an Azure Monitor exporter but it may target Application Insights rather than Log Analytics. | +| `azureLogsIngestion` | `azure_logs_ingestion` (custom Vector sink) | No dedicated exporter | ❌ Gap | Azure Logs Ingestion API (Data Collection Rules) — no OTEL contrib exporter exists. [Issue #40478](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/40478) is open requesting DCR-based exporter. Needs a custom exporter. Workload identity and client secret auth are required. | + +#### 5. Cross-Cutting Concerns + +| Feature | Vector | OTEL Collector | Status | Notes | +|---------|--------|---------------|--------|-------| +| **TLS** (CA, cert, key, passphrase) | Per-source/sink TLS config | Per-receiver/exporter TLS config | ✅ Supported | OTEL supports `ca_file`, `cert_file`, `key_file`, min TLS version, cipher suites. Key passphrase support needs verification. | +| **OpenShift TLS Security Profiles** | Mapped to Vector TLS min version + ciphers | Map to OTEL TLS `min_version` + `cipher_suites` | ✅ Supported | Same mapping logic, different config format. | +| **Authentication (Bearer token)** | HTTP auth header or file-based token | `bearertokenauth` extension | ✅ Supported | OTEL's auth extensions support token from file (service account). | +| **Authentication (Basic auth)** | Username/password in sink config | `basicauth` extension | ✅ Supported | | +| **Authentication (AWS)** | AWS access key or IAM role (STS) | Built into AWS exporters + `sigv4auth` extension | ✅ Supported | AWS exporters natively support credentials/role chains. | +| **Delivery mode: AtLeastOnce** | Disk buffer (256MB), block when full, retry | `file_storage` extension + persistent sending queue | ✅ Supported | OTEL persistent queue uses `file_storage` extension for disk-backed queuing with retry. | +| **Delivery mode: AtMostOnce** | Memory buffer, drop newest when full | In-memory sending queue, `drop_on_queue_full: true` | ✅ Supported | | +| **Compression** | gzip, zstd, snappy, zlib, lz4 (varies by sink) | gzip, zstd, snappy, zlib (varies by exporter) | ✅ Mostly supported | lz4 (Kafka-only) may need verification. | +| **Batch config** (MaxWrite) | `batch.max_bytes` / `batch.max_events` | `batch` processor or exporter-level `sending_queue` | ✅ Supported | | +| **Retry config** (min/max duration) | `request.retry_initial_backoff_secs`, `request.retry_max_duration_sec` | Exporter `retry_on_failure` with `initial_interval`, `max_interval`, `max_elapsed_time` | ✅ Supported | | +| **Rate limiting (output-level)** | `throttle` transform before sink | No official contrib rate limiter; Elastic's [`ratelimitprocessor`](https://pkg.go.dev/github.com/elastic/opentelemetry-collector-components/processor/ratelimitprocessor) exists | ⚠️ Partial | Elastic's processor supports per-key overrides via client metadata matching. An official contrib rate limit processor is [accepted (#35204)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/35204) but not yet implemented. Per-container-file keying (Vector's `key_field: {{ _internal.file }}`) would need per-attribute keying instead. | +| **Proxy support** | `proxy.http` / `proxy.https` on HTTP sinks | Exporter-level `proxy_url` or env vars (`HTTP_PROXY`) | ✅ Supported | | +| **Health checks / metrics** | Vector API + `internal_metrics` source | OTEL health check extension + `prometheus` exporter or `zpages` extension | ✅ Supported | | + +#### 6. Dynamic Templating + +CLO uses dynamic templates for output field values (index names, topic names, log group names, etc.). These templates reference log record fields with a `{{.path.to.field}}` syntax, resolved at config generation time to Vector's template syntax (`{{ field }}`). + +| Template Usage | Vector | OTEL | Status | +|---------------|--------|------|--------| +| Elasticsearch index name | `{{ field }}` in index template | `elasticsearch.index` attribute on log records | ✅ Supported | Since [v0.122.0](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/elasticsearchexporter), dynamic routing via `elasticsearch.index` attribute is the default. Use `transform` processor to set the attribute per-record based on log fields. | +| Kafka topic name | `{{ field }}` in topic template | Kafka exporter `topic_from_attribute` | ✅ Supported | Kafka exporter can derive topic from log attributes. | +| CloudWatch log group / stream | `{{ field }}` templates | Exporter config | ⚠️ Needs verification | | +| S3 key prefix | `{{ field }}` template | Exporter config or `routing` | ⚠️ Needs verification | | +| Splunk index/source/sourcetype | `{{ field }}` templates | Exporter mapping config | ⚠️ Needs design | | +| Syslog facility/severity/appname | VRL field extraction + syslog codec | `transform` processor + exporter config | ✅ Supported | | + +### Summary of Gaps + +#### Critical Gaps (❌ — no built-in solution, needs custom component) + +1. **Generic HTTP output** — No OTEL exporter sends arbitrary JSON/NDJSON to generic HTTP endpoints. [Proposed but not implemented](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/10836). Needs a custom exporter. + +2. **Azure Logs Ingestion output** — No OTEL exporter for Azure Data Collection Rules (Logs Ingestion API). [Issue #40478](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/40478) is open. Needs a custom exporter. + +3. **Multi-line exception detection** (post-ingestion) — No OTEL processor merges multi-line stack traces after collection the way Vector's `detect_exceptions` transform does (language-specific detection for Java, Python, Go, Ruby, etc.). The `filelog` receiver's `multiline` config works at file ingestion (before CRI log parsing), which could merge lines at the file level but requires CRI-aware regex patterns and doesn't provide the language-specific exception boundary detection. The `exceptions` connector extracts exceptions from trace spans, not from log messages. + +#### Partial Support (⚠️ — exists outside official contrib, or needs significant OTTL) + +4. **Per-key rate limiting (throttle)** — Elastic's [`ratelimitprocessor`](https://pkg.go.dev/github.com/elastic/opentelemetry-collector-components/processor/ratelimitprocessor) supports per-key overrides via client metadata matching. An official contrib rate limit processor is [accepted (#35204)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/35204) but not yet implemented. Vector's per-container-file keying would need adaptation to per-attribute keying. + +5. **Kube API audit policy filter** — The K8s audit policy logic (wildcard matching on users/groups/resources, verb filtering, level assignment, field redaction) is very complex. OTTL can express the conditions but the wildcard-to-regex conversion and rule precedence would produce very verbose config. A dedicated processor would be cleaner. + +6. **Log level detection (5-stage)** — OTTL's [`ExtractGrokPatterns`](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/pkg/ottl/ottlfuncs/README.md) (Elastic Go-Grok library) covers grok-based detection (stage 3). OTTL regex (`IsMatch`) covers stages 4-5. Stages 1-2 (logfmt, klog) lack dedicated parsers but can be approximated with grok/regex. Overall feasible but verbose. + +7. **Dynamic field templating for output destinations** — Varies per exporter. Elasticsearch: ✅ (dynamic index via `elasticsearch.index` attribute, default since v0.122.0). Kafka: ✅ (`topic_from_attribute`). CloudWatch, S3, Splunk: needs verification of per-record attribute-based routing support. May need `routing` processor fan-out for exporters that don't support per-record destination fields. + +8. **VIAQ data model compatibility** — Non-OTLP outputs (Elasticsearch, Splunk, Syslog, HTTP) currently receive logs in the VIAQ data model. With OTEL collector, these exporters serialize from OTLP format. The output schema may differ, potentially breaking existing consumer configurations. This needs careful mapping and potentially a "VIAQ compatibility mode" transform. + +#### Fully Supported (✅) + +- Container log collection (`filelog` + `k8sattributes`) +- Journal/node log collection (`journalctl` receiver) +- Audit file collection (`filelog` receiver) +- Syslog receiver +- Drop filter (`filter` processor) +- Prune filter (`transform` processor) +- Parse filter (`transform` processor) +- OpenShift labels filter (`attributes` processor) +- OTLP output (native) +- Elasticsearch, Kafka, Loki, LokiStack, Splunk, Syslog, CloudWatch, S3, Google Cloud Logging outputs +- TLS configuration +- Authentication (bearer, basic, AWS, SASL) +- Delivery modes (persistent queue for AtLeastOnce) +- Compression, batch, retry configuration +- Log routing by type/source +- Kubernetes metadata enrichment + +### Recommended Approach for Milestone 1 + +#### Phase 1: Core pipeline (OTLP-only output path) +Start with the simplest end-to-end path: +- `filelog` receiver for container + audit logs +- `journalctl` receiver for node logs +- `k8sattributes` processor for Kubernetes metadata +- `transform` processor for log classification (application/infrastructure/audit) +- `resource` processor for cluster ID, hostname +- `otlp`/`otlphttp` exporter + +This validates the basic collection pipeline without needing complex transforms or exotic exporters. + +#### Phase 2: Add supported outputs +Add exporters one by one, starting with the most commonly used: +- Elasticsearch, Kafka, Loki/LokiStack, CloudWatch, Splunk, Syslog, S3, Google Cloud Logging + +#### Phase 3: Filters and normalization +Implement CLO filters as OTEL processors: +- Drop → `filter` processor +- Prune → `transform` processor +- Parse → `transform` processor +- OpenShift labels → `attributes` processor +- Log level detection → custom processor or OTTL regex approximation +- Kube API audit → custom processor or extensive OTTL + +#### Phase 4: Address gaps +- Generic HTTP exporter (custom or community) +- Azure Logs Ingestion exporter (custom) +- Rate limiting processor (custom) +- Multi-line exception detection (custom processor or integration with filelog) +- VIAQ data model compatibility for non-OTLP outputs \ No newline at end of file diff --git a/test/framework/functional/framework.go b/test/framework/functional/framework.go index 991e6d7c14..67141a0916 100644 --- a/test/framework/functional/framework.go +++ b/test/framework/functional/framework.go @@ -36,6 +36,7 @@ import ( "github.com/openshift/cluster-logging-operator/internal/utils" "github.com/openshift/cluster-logging-operator/test/client" commonlog "github.com/openshift/cluster-logging-operator/test/framework/common/log" + frameworkotel "github.com/openshift/cluster-logging-operator/test/framework/functional/otel" frameworkvector "github.com/openshift/cluster-logging-operator/test/framework/functional/vector" "github.com/openshift/cluster-logging-operator/test/helpers/oc" corev1 "k8s.io/api/core/v1" @@ -94,8 +95,11 @@ func NewCollectorFunctionalFrameworkUsing(t *client.Test, fnClose func(), verbos verbosity = i } } - var collectorImpl CollectorFramework = &frameworkvector.VectorCollector{ - Test: t, + var collectorImpl CollectorFramework + if os.Getenv("COLLECTOR_TYPE") == constants.OTELCollectorName { + collectorImpl = &frameworkotel.OTELCollector{Test: t} + } else { + collectorImpl = &frameworkvector.VectorCollector{Test: t} } failureLogger, delayedWriter := commonlog.NewLogger("functional-Framework", verbosity) diff --git a/test/framework/functional/otel/deploy.go b/test/framework/functional/otel/deploy.go new file mode 100644 index 0000000000..affeebcd8a --- /dev/null +++ b/test/framework/functional/otel/deploy.go @@ -0,0 +1,69 @@ +package otel + +import ( + "regexp" + "strings" + + log "github.com/ViaQ/logerr/v2/log/static" + "github.com/openshift/cluster-logging-operator/internal/collector/otel" + "github.com/openshift/cluster-logging-operator/internal/constants" + "github.com/openshift/cluster-logging-operator/internal/runtime" + "github.com/openshift/cluster-logging-operator/internal/utils" + "github.com/openshift/cluster-logging-operator/test/client" + "github.com/openshift/cluster-logging-operator/test/framework/functional/common" +) + +type OTELCollector struct { + *client.Test +} + +func (c *OTELCollector) String() string { + return constants.OTELCollectorName +} + +func (c *OTELCollector) DeployConfigMapForConfig(name, config, clfName, clfYaml string) error { + log.V(2).Info("Creating config configmap for OTEL collector") + configmap := runtime.NewConfigMap(c.NS.Name, name, map[string]string{}) + runtime.NewConfigMapBuilder(configmap). + Add(otel.ConfigFile, config). + Add("clfyaml", clfYaml) + if err := c.Create(configmap); err != nil { + return err + } + return nil +} + +func (c *OTELCollector) BuildCollectorContainer(b *runtime.ContainerBuilder, nodeName string) *runtime.ContainerBuilder { + return b.AddEnvVar("OTEL_LOG_LEVEL", common.AdaptLogLevel()). + AddEnvVarFromFieldRef("POD_IP", "status.podIP"). + AddEnvVar("NODE_NAME", nodeName). + AddEnvVarFromFieldRef("OTEL_RESOURCE_ATTRIBUTES_NODE_NAME", "spec.nodeName"). + AddVolumeMount("config", "/etc/otelcol", "", true). + AddVolumeMount("certs", "/etc/collector/metrics", "", true). + WithCmd([]string{"/otelcol-contrib", "--config=/etc/otelcol/" + otel.ConfigFile}) +} + +func (c *OTELCollector) IsStarted(logs string) bool { + return strings.Contains(logs, "Everything is ready.") +} + +func (c *OTELCollector) Image() string { + return utils.GetComponentImage(constants.OTELCollectorName) +} + +const fakeJournal = ` + filelog/fake_journal: + include: + - /var/log/fakejournal/0.log + start_at: beginning + include_file_path: true + include_file_name: false + operators: + - type: json_parser + id: json-parser +` + +func (c *OTELCollector) ModifyConfig(conf string) string { + re := regexp.MustCompile(`(?msU) journalctl.*?^\n`) + return string(re.ReplaceAll([]byte(conf), []byte(fakeJournal))) +}