From 2821848b0b2d075509972ba0ba8e5fceae96e197 Mon Sep 17 00:00:00 2001 From: Vitalii Parfonov Date: Thu, 13 Aug 2026 13:27:33 +0300 Subject: [PATCH] feat(vector): always raise file descriptor soft limit Always set VECTOR_RAISE_FD_LIMIT=true on the collector so Vector raises its soft limit to the hard limit at startup. This prevents "Too many open files" errors when monitoring large numbers of log files, without requiring user configuration. Signed-off-by: Vitalii Parfonov --- .github/actions/spelling/patterns.txt | 3 ++ docs/features/raise_fd_limit.adoc | 7 ++++ .../operator/api_observability_v1.adoc | 5 +++ internal/collector/collector.go | 1 + internal/collector/collector_test.go | 4 +++ test/framework/functional/vector/deploy.go | 1 + .../misc/disk_buffer_corruption_test.go | 2 +- test/functional/misc/raise_fd_limit_test.go | 34 +++++++++++++++++++ test/helpers/types/types.go | 2 +- 9 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 docs/features/raise_fd_limit.adoc create mode 100644 test/functional/misc/raise_fd_limit_test.go diff --git a/.github/actions/spelling/patterns.txt b/.github/actions/spelling/patterns.txt index cbbdb71ae..af943a451 100644 --- a/.github/actions/spelling/patterns.txt +++ b/.github/actions/spelling/patterns.txt @@ -19,6 +19,9 @@ https?://\S+ # Version strings v\d+\.\d+(\.\d+)? +# AsciiDoc source/listing blocks (---- delimited) +(?s)^----$.*?^----$ + # AsciiDoc directives and macros ^include::.*$ ^ifdef::.*$ diff --git a/docs/features/raise_fd_limit.adoc b/docs/features/raise_fd_limit.adoc new file mode 100644 index 000000000..ebb77e87e --- /dev/null +++ b/docs/features/raise_fd_limit.adoc @@ -0,0 +1,7 @@ +== Vector Collector: Raising File Descriptor Limits + +The operator always sets `VECTOR_RAISE_FD_LIMIT=true` on the collector container so Vector raises its file descriptor soft limit to the hard limit at startup. This prevents "Too many open files" errors when Vector monitors a large number of log files concurrently. + +Systems often default to restrictive file descriptor soft limits (e.g. 1024 on Linux), which can cause Vector to fail when processing many log sources simultaneously. Vector reads `VECTOR_RAISE_FD_LIMIT` natively and raises the soft limit without requiring manual sysadmin intervention. The hard limit remains the safety ceiling set by the OS/container runtime. + +This behavior is not configurable. A collector tuning option may be added in the future if needed. diff --git a/docs/reference/operator/api_observability_v1.adoc b/docs/reference/operator/api_observability_v1.adoc index ac0bb0190..ee48a68d4 100644 --- a/docs/reference/operator/api_observability_v1.adoc +++ b/docs/reference/operator/api_observability_v1.adoc @@ -61,6 +61,7 @@ Type:: object |networkPolicy|object| Define the Network Policy for the Collector |nodeSelector|object| Define nodes for scheduling the pods. |resources|object| The resource requirements for the collector +|terminationGracePeriodSeconds|int| TerminationGracePeriodSeconds defines the termination grace period for collector pods in seconds. If not specified, the default is 10 seconds. |tolerations|array| Define the tolerations the collector pods will accept |====================== @@ -651,6 +652,10 @@ Type:: object Type:: object +=== .spec.collector.terminationGracePeriodSeconds + +Type:: int + === .spec.collector.tolerations[] Type:: array diff --git a/internal/collector/collector.go b/internal/collector/collector.go index 52be2dc2a..519de5296 100644 --- a/internal/collector/collector.go +++ b/internal/collector/collector.go @@ -239,6 +239,7 @@ func (f *Factory) NewCollectorContainer(inputs internalobs.Inputs, outputs inter {Name: "OPENSHIFT_CLUSTER_ID", Value: clusterID}, {Name: "POD_IP", ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "status.podIP"}}}, {Name: "POD_IPS", ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "status.podIPs"}}}, + {Name: "VECTOR_RAISE_FD_LIMIT", Value: "true"}, } collector.Env = append(collector.Env, utils.GetProxyEnvVars()...) diff --git a/internal/collector/collector_test.go b/internal/collector/collector_test.go index b6f531a39..66e55016d 100644 --- a/internal/collector/collector_test.go +++ b/internal/collector/collector_test.go @@ -143,6 +143,10 @@ var _ = Describe("Factory#Daemonset", func() { Expect(collector.Env).To(IncludeEnvVar(v1.EnvVar{Name: "VECTOR_LOG", Value: logLevelDebug})) }) + It("should set VECTOR_RAISE_FD_LIMIT to true", func() { + Expect(collector.Env).To(IncludeEnvVar(v1.EnvVar{Name: "VECTOR_RAISE_FD_LIMIT", Value: "true"})) + }) + Context("the volume mounts", func() { It("should mount all output configmaps", func() { Expect(collector.VolumeMounts).To(IncludeVolumeMount( diff --git a/test/framework/functional/vector/deploy.go b/test/framework/functional/vector/deploy.go index 4c9c05a04..562572f99 100644 --- a/test/framework/functional/vector/deploy.go +++ b/test/framework/functional/vector/deploy.go @@ -38,6 +38,7 @@ func (c *VectorCollector) DeployConfigMapForConfig(name, config, clfName, clfYam func (c *VectorCollector) BuildCollectorContainer(b *runtime.ContainerBuilder, nodeName string) *runtime.ContainerBuilder { return b.AddEnvVar("VECTOR_LOG", common.AdaptLogLevel()). + AddEnvVar("VECTOR_RAISE_FD_LIMIT", "true"). AddEnvVarFromFieldRef("POD_IP", "status.podIP"). AddEnvVar("NODE_NAME", nodeName). AddEnvVar("VECTOR_INTERNAL_LOG_RATE_LIMIT", "0"). diff --git a/test/functional/misc/disk_buffer_corruption_test.go b/test/functional/misc/disk_buffer_corruption_test.go index d3638a603..0205cfa4a 100644 --- a/test/functional/misc/disk_buffer_corruption_test.go +++ b/test/functional/misc/disk_buffer_corruption_test.go @@ -313,7 +313,7 @@ func corruptBufferFile(framework *functional.CollectorFunctionalFramework, datFi const chunkSize = 65536 for i := 0; i < len(corruptedB64); i += chunkSize { - end := min(i + chunkSize, len(corruptedB64)) + end := min(i+chunkSize, len(corruptedB64)) chunk := corruptedB64[i:end] appendCmd := fmt.Sprintf("echo -n '%s' >> '%s'", chunk, stagingFile) _, err = framework.RunCommand(functional.ToolsContainerName, "bash", "-c", appendCmd) diff --git a/test/functional/misc/raise_fd_limit_test.go b/test/functional/misc/raise_fd_limit_test.go new file mode 100644 index 000000000..96ad3441c --- /dev/null +++ b/test/functional/misc/raise_fd_limit_test.go @@ -0,0 +1,34 @@ +package misc + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + obs "github.com/openshift/cluster-logging-operator/api/observability/v1" + "github.com/openshift/cluster-logging-operator/internal/constants" + "github.com/openshift/cluster-logging-operator/test/framework/functional" + testruntime "github.com/openshift/cluster-logging-operator/test/runtime/observability" +) + +var _ = Describe("[Functional][Misc][RaiseFdLimit] Vector raise-fd-limit", func() { + + var framework *functional.CollectorFunctionalFramework + + BeforeEach(func() { + framework = functional.NewCollectorFunctionalFramework() + testruntime.NewClusterLogForwarderBuilder(framework.Forwarder). + FromInput(obs.InputTypeInfrastructure). + ToHttpOutput() + }) + + AfterEach(func() { + framework.Cleanup() + }) + + It("should start with VECTOR_RAISE_FD_LIMIT=true", func() { + Expect(framework.Deploy()).To(BeNil()) + + out, err := framework.RunCommand(constants.CollectorName, "sh", "-c", "echo $VECTOR_RAISE_FD_LIMIT") + Expect(err).To(BeNil()) + Expect(out).To(ContainSubstring("true")) + }) +}) diff --git a/test/helpers/types/types.go b/test/helpers/types/types.go index 89bd54b09..f55e70f76 100644 --- a/test/helpers/types/types.go +++ b/test/helpers/types/types.go @@ -165,7 +165,7 @@ type OpenshiftMeta struct { //+optional Labels map[string]string `json:"labels,omitempty"` - //Sequence is increasing id used in conjunction with the timestamp to estblish a linear timeline + //Sequence is increasing id used in conjunction with the timestamp to establish a linear timeline //of log records. This was added as a workaround for logstores that do not have nano-second precision. Sequence OptionalInt `json:"sequence,omitempty"` }