diff --git a/.github/actions/spelling/patterns.txt b/.github/actions/spelling/patterns.txt index cbbdb71ae..af943a451 100644 --- a/.github/actions/spelling/patterns.txt +++ b/.github/actions/spelling/patterns.txt @@ -19,6 +19,9 @@ https?://\S+ # Version strings v\d+\.\d+(\.\d+)? +# AsciiDoc source/listing blocks (---- delimited) +(?s)^----$.*?^----$ + # AsciiDoc directives and macros ^include::.*$ ^ifdef::.*$ diff --git a/docs/features/raise_fd_limit.adoc b/docs/features/raise_fd_limit.adoc new file mode 100644 index 000000000..ebb77e87e --- /dev/null +++ b/docs/features/raise_fd_limit.adoc @@ -0,0 +1,7 @@ +== Vector Collector: Raising File Descriptor Limits + +The operator always sets `VECTOR_RAISE_FD_LIMIT=true` on the collector container so Vector raises its file descriptor soft limit to the hard limit at startup. This prevents "Too many open files" errors when Vector monitors a large number of log files concurrently. + +Systems often default to restrictive file descriptor soft limits (e.g. 1024 on Linux), which can cause Vector to fail when processing many log sources simultaneously. Vector reads `VECTOR_RAISE_FD_LIMIT` natively and raises the soft limit without requiring manual sysadmin intervention. The hard limit remains the safety ceiling set by the OS/container runtime. + +This behavior is not configurable. A collector tuning option may be added in the future if needed. diff --git a/docs/reference/operator/api_observability_v1.adoc b/docs/reference/operator/api_observability_v1.adoc index ac0bb0190..ee48a68d4 100644 --- a/docs/reference/operator/api_observability_v1.adoc +++ b/docs/reference/operator/api_observability_v1.adoc @@ -61,6 +61,7 @@ Type:: object |networkPolicy|object| Define the Network Policy for the Collector |nodeSelector|object| Define nodes for scheduling the pods. |resources|object| The resource requirements for the collector +|terminationGracePeriodSeconds|int| TerminationGracePeriodSeconds defines the termination grace period for collector pods in seconds. If not specified, the default is 10 seconds. |tolerations|array| Define the tolerations the collector pods will accept |====================== @@ -651,6 +652,10 @@ Type:: object Type:: object +=== .spec.collector.terminationGracePeriodSeconds + +Type:: int + === .spec.collector.tolerations[] Type:: array diff --git a/internal/collector/collector.go b/internal/collector/collector.go index 52be2dc2a..519de5296 100644 --- a/internal/collector/collector.go +++ b/internal/collector/collector.go @@ -239,6 +239,7 @@ func (f *Factory) NewCollectorContainer(inputs internalobs.Inputs, outputs inter {Name: "OPENSHIFT_CLUSTER_ID", Value: clusterID}, {Name: "POD_IP", ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "status.podIP"}}}, {Name: "POD_IPS", ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "status.podIPs"}}}, + {Name: "VECTOR_RAISE_FD_LIMIT", Value: "true"}, } collector.Env = append(collector.Env, utils.GetProxyEnvVars()...) diff --git a/internal/collector/collector_test.go b/internal/collector/collector_test.go index b6f531a39..66e55016d 100644 --- a/internal/collector/collector_test.go +++ b/internal/collector/collector_test.go @@ -143,6 +143,10 @@ var _ = Describe("Factory#Daemonset", func() { Expect(collector.Env).To(IncludeEnvVar(v1.EnvVar{Name: "VECTOR_LOG", Value: logLevelDebug})) }) + It("should set VECTOR_RAISE_FD_LIMIT to true", func() { + Expect(collector.Env).To(IncludeEnvVar(v1.EnvVar{Name: "VECTOR_RAISE_FD_LIMIT", Value: "true"})) + }) + Context("the volume mounts", func() { It("should mount all output configmaps", func() { Expect(collector.VolumeMounts).To(IncludeVolumeMount( diff --git a/test/framework/functional/vector/deploy.go b/test/framework/functional/vector/deploy.go index 4c9c05a04..562572f99 100644 --- a/test/framework/functional/vector/deploy.go +++ b/test/framework/functional/vector/deploy.go @@ -38,6 +38,7 @@ func (c *VectorCollector) DeployConfigMapForConfig(name, config, clfName, clfYam func (c *VectorCollector) BuildCollectorContainer(b *runtime.ContainerBuilder, nodeName string) *runtime.ContainerBuilder { return b.AddEnvVar("VECTOR_LOG", common.AdaptLogLevel()). + AddEnvVar("VECTOR_RAISE_FD_LIMIT", "true"). AddEnvVarFromFieldRef("POD_IP", "status.podIP"). AddEnvVar("NODE_NAME", nodeName). AddEnvVar("VECTOR_INTERNAL_LOG_RATE_LIMIT", "0"). diff --git a/test/functional/misc/disk_buffer_corruption_test.go b/test/functional/misc/disk_buffer_corruption_test.go index d3638a603..0205cfa4a 100644 --- a/test/functional/misc/disk_buffer_corruption_test.go +++ b/test/functional/misc/disk_buffer_corruption_test.go @@ -313,7 +313,7 @@ func corruptBufferFile(framework *functional.CollectorFunctionalFramework, datFi const chunkSize = 65536 for i := 0; i < len(corruptedB64); i += chunkSize { - end := min(i + chunkSize, len(corruptedB64)) + end := min(i+chunkSize, len(corruptedB64)) chunk := corruptedB64[i:end] appendCmd := fmt.Sprintf("echo -n '%s' >> '%s'", chunk, stagingFile) _, err = framework.RunCommand(functional.ToolsContainerName, "bash", "-c", appendCmd) diff --git a/test/functional/misc/raise_fd_limit_test.go b/test/functional/misc/raise_fd_limit_test.go new file mode 100644 index 000000000..96ad3441c --- /dev/null +++ b/test/functional/misc/raise_fd_limit_test.go @@ -0,0 +1,34 @@ +package misc + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + obs "github.com/openshift/cluster-logging-operator/api/observability/v1" + "github.com/openshift/cluster-logging-operator/internal/constants" + "github.com/openshift/cluster-logging-operator/test/framework/functional" + testruntime "github.com/openshift/cluster-logging-operator/test/runtime/observability" +) + +var _ = Describe("[Functional][Misc][RaiseFdLimit] Vector raise-fd-limit", func() { + + var framework *functional.CollectorFunctionalFramework + + BeforeEach(func() { + framework = functional.NewCollectorFunctionalFramework() + testruntime.NewClusterLogForwarderBuilder(framework.Forwarder). + FromInput(obs.InputTypeInfrastructure). + ToHttpOutput() + }) + + AfterEach(func() { + framework.Cleanup() + }) + + It("should start with VECTOR_RAISE_FD_LIMIT=true", func() { + Expect(framework.Deploy()).To(BeNil()) + + out, err := framework.RunCommand(constants.CollectorName, "sh", "-c", "echo $VECTOR_RAISE_FD_LIMIT") + Expect(err).To(BeNil()) + Expect(out).To(ContainSubstring("true")) + }) +}) diff --git a/test/helpers/types/types.go b/test/helpers/types/types.go index 89bd54b09..f55e70f76 100644 --- a/test/helpers/types/types.go +++ b/test/helpers/types/types.go @@ -165,7 +165,7 @@ type OpenshiftMeta struct { //+optional Labels map[string]string `json:"labels,omitempty"` - //Sequence is increasing id used in conjunction with the timestamp to estblish a linear timeline + //Sequence is increasing id used in conjunction with the timestamp to establish a linear timeline //of log records. This was added as a workaround for logstores that do not have nano-second precision. Sequence OptionalInt `json:"sequence,omitempty"` }