From 1f001f1de72bcc40af13d603661c97bbbf073260 Mon Sep 17 00:00:00 2001 From: Jeff McCollum <16550786+jeffmccollum@users.noreply.github.com> Date: Thu, 8 Oct 2026 00:43:53 +0000 Subject: [PATCH 1/2] Size Brainstore object store cache as a percentage of the cache volume Mirrors terraform-aws-braintrust-data-plane#420. When objectStoreCacheFileSize is unset (the new default), each Brainstore pod sets BRAINSTORE_OBJECT_STORE_CACHE_FILE_SIZE at startup to objectStoreCacheFileSizePercent of its cache volume: 90% for readers and fast readers, 75% for writers. The volume size is the filesystem at cacheDir, capped by volume.sizeLimit and volume.size when set. An explicit objectStoreCacheFileSize keeps the existing behavior: Brainstore starts directly and reads the size from its ConfigMap. Co-Authored-By: Claude Opus 5.5 --- braintrust/README.md | 25 +++ .../examples/google-autopilot-cel/values.yaml | 3 - .../examples/google-autopilot/values.yaml | 3 - .../examples/google-standard/values.yaml | 2 - braintrust/templates/_helpers.tpl | 78 ++++++++ .../brainstore-fastreader-configmap.yaml | 2 + .../brainstore-fastreader-deployment.yaml | 7 +- .../brainstore-reader-configmap.yaml | 2 + .../brainstore-reader-deployment.yaml | 7 +- .../brainstore-writer-configmap.yaml | 2 + .../brainstore-writer-deployment.yaml | 7 +- .../brainstore-cache-file-size_test.yaml | 186 ++++++++++++++++++ braintrust/values.yaml | 18 +- 13 files changed, 325 insertions(+), 17 deletions(-) create mode 100644 braintrust/tests/brainstore-cache-file-size_test.yaml diff --git a/braintrust/README.md b/braintrust/README.md index 48bac42..02c5e74 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -289,6 +289,31 @@ Pools use fixed replica counts by default (`api.replicas` and `api.workloadIsolation..replicas`). On GKE, enable `api.autoscaling` to let each pool scale independently instead. +## Brainstore Cache Sizing + +By default, each Brainstore pod sizes its object store cache when it starts, +as a percentage of its cache volume. Readers and fast readers use 90%. Writers +use 75% because they need more free disk. This matches how the AWS Terraform +module sizes the cache from the instance's local disk. + +The cache volume size is the size of the filesystem mounted at `cacheDir`, +capped by `volume.sizeLimit` and `volume.size` when they are set. With a plain +`emptyDir`, that filesystem is the node's disk, so give Brainstore pods +dedicated nodes (see the anti-affinity examples above) or set `volume.size`. + +```yaml +brainstore: + writer: + # Change the percentage + objectStoreCacheFileSizePercent: 70 + reader: + # Or set a fixed size, which disables the percentage + objectStoreCacheFileSize: "800Gi" +``` + +The pod logs the size it picked at startup. If the computed size is under +1Gi, the pod exits with an error asking you to set `objectStoreCacheFileSize`. + ## Brainstore Rollout Controls Brainstore readers, fast readers, and writers have independently configurable diff --git a/braintrust/examples/google-autopilot-cel/values.yaml b/braintrust/examples/google-autopilot-cel/values.yaml index 380184e..ac0184c 100644 --- a/braintrust/examples/google-autopilot-cel/values.yaml +++ b/braintrust/examples/google-autopilot-cel/values.yaml @@ -88,7 +88,6 @@ brainstore: memory: "32Gi" cacheDir: "/mnt/tmp/brainstore" objectStoreCacheMemoryLimit: "1Gi" - objectStoreCacheFileSize: "900Gi" verbose: true securityContext: readOnlyRootFilesystem: true @@ -121,7 +120,6 @@ brainstore: memory: "32Gi" cacheDir: "/mnt/tmp/brainstore" objectStoreCacheMemoryLimit: "1Gi" - objectStoreCacheFileSize: "900Gi" verbose: true securityContext: readOnlyRootFilesystem: true @@ -154,7 +152,6 @@ brainstore: memory: "64Gi" cacheDir: "/mnt/tmp/brainstore" objectStoreCacheMemoryLimit: "1Gi" - objectStoreCacheFileSize: "900Gi" verbose: true securityContext: readOnlyRootFilesystem: true diff --git a/braintrust/examples/google-autopilot/values.yaml b/braintrust/examples/google-autopilot/values.yaml index 9375209..2d5b691 100644 --- a/braintrust/examples/google-autopilot/values.yaml +++ b/braintrust/examples/google-autopilot/values.yaml @@ -96,7 +96,6 @@ brainstore: memory: "32Gi" cacheDir: "/mnt/tmp/brainstore" objectStoreCacheMemoryLimit: "1Gi" - objectStoreCacheFileSize: "900Gi" verbose: true volume: size: "1000Gi" @@ -120,7 +119,6 @@ brainstore: memory: "32Gi" cacheDir: "/mnt/tmp/brainstore" objectStoreCacheMemoryLimit: "1Gi" - objectStoreCacheFileSize: "900Gi" verbose: true volume: size: "1000Gi" @@ -144,7 +142,6 @@ brainstore: memory: "64Gi" cacheDir: "/mnt/tmp/brainstore" objectStoreCacheMemoryLimit: "1Gi" - objectStoreCacheFileSize: "900Gi" verbose: true volume: size: "1000Gi" diff --git a/braintrust/examples/google-standard/values.yaml b/braintrust/examples/google-standard/values.yaml index a1ee0d9..2bf1998 100644 --- a/braintrust/examples/google-standard/values.yaml +++ b/braintrust/examples/google-standard/values.yaml @@ -99,7 +99,6 @@ brainstore: topologyKey: kubernetes.io/hostname cacheDir: "/mnt/tmp/brainstore" objectStoreCacheMemoryLimit: "1Gi" - objectStoreCacheFileSize: "1000Gi" verbose: true volume: size: "200Gi" @@ -136,7 +135,6 @@ brainstore: topologyKey: kubernetes.io/hostname cacheDir: "/mnt/tmp/brainstore" objectStoreCacheMemoryLimit: "1Gi" - objectStoreCacheFileSize: "1000Gi" verbose: true volume: size: "200Gi" diff --git a/braintrust/templates/_helpers.tpl b/braintrust/templates/_helpers.tpl index 228f136..442ac6f 100644 --- a/braintrust/templates/_helpers.tpl +++ b/braintrust/templates/_helpers.tpl @@ -188,3 +188,81 @@ and normal writable-layer/log overhead. {{- end -}} {{- toYaml $resources -}} {{- end -}} + +{{/* +Convert a Kubernetes quantity such as "900Gi" or "1T" to bytes. +*/}} +{{- define "braintrust.quantityToBytes" -}} +{{- $quantity := toString .quantity -}} +{{- $match := regexFindAll "^([0-9]+)(Ki|Mi|Gi|Ti|Pi|k|K|M|G|T|P)?$" $quantity -1 -}} +{{- if not $match -}} +{{- fail (printf "%s must be a whole-number Kubernetes quantity such as 900Gi (got %q)" .path $quantity) -}} +{{- end -}} +{{- $number := regexFind "^[0-9]+" $quantity | int64 -}} +{{- $suffix := trimPrefix (toString $number) $quantity -}} +{{- $multipliers := dict + "" 1 + "Ki" 1024 "Mi" 1048576 "Gi" 1073741824 "Ti" 1099511627776 "Pi" 1125899906842624 + "k" 1000 "K" 1000 "M" 1000000 "G" 1000000000 "T" 1000000000000 "P" 1000000000000000 +-}} +{{- mul $number (index $multipliers $suffix) -}} +{{- end -}} + +{{/* +Render the Brainstore container command. + +When objectStoreCacheFileSize is set, Brainstore starts directly and reads the +size from its ConfigMap. Otherwise the container sets +BRAINSTORE_OBJECT_STORE_CACHE_FILE_SIZE at startup to +objectStoreCacheFileSizePercent of the cache volume, the same way the AWS +Terraform module sizes the cache from the instance's local disk. The cache +volume size is the size of the filesystem mounted at cacheDir, capped by +volume.sizeLimit and volume.size when they are set. + +Template-level percent defaults keep `helm upgrade --reuse-values` working +from chart versions that predate objectStoreCacheFileSizePercent. +*/}} +{{- define "braintrust.brainstoreCommand" -}} +{{- $config := .config -}} +{{- if $config.objectStoreCacheFileSize -}} +command: ["brainstore"] +args: ["web"] +{{- else -}} +{{- $percent := $config.objectStoreCacheFileSizePercent | default .defaultPercent -}} +{{- if not (regexMatch "^[0-9]+$" (toString $percent)) -}} +{{- fail (printf "%s.objectStoreCacheFileSizePercent must be a whole number between 1 and 100" .path) -}} +{{- end -}} +{{- if or (lt (int $percent) 1) (gt (int $percent) 100) -}} +{{- fail (printf "%s.objectStoreCacheFileSizePercent must be a whole number between 1 and 100" .path) -}} +{{- end -}} +{{- $limitBytes := 0 -}} +{{- range $field := list "sizeLimit" "size" -}} +{{- $quantity := index ($config.volume | default dict) $field -}} +{{- if $quantity -}} +{{- $bytes := include "braintrust.quantityToBytes" (dict "quantity" $quantity "path" (printf "%s.volume.%s" $.path $field)) | int64 -}} +{{- if or (eq $limitBytes 0) (lt $bytes $limitBytes) -}} +{{- $limitBytes = $bytes -}} +{{- end -}} +{{- end -}} +{{- end -}} +command: ["/bin/sh", "-c"] +args: + - | + set -eu + percent={{ $percent }} + limit_bytes={{ $limitBytes }} + set -- $(stat -f -c '%b %S' "$BRAINSTORE_CACHE_DIR") + volume_bytes=$(( $1 * $2 )) + if [ "$limit_bytes" -gt 0 ] && [ "$limit_bytes" -lt "$volume_bytes" ]; then + volume_bytes=$limit_bytes + fi + size_gib=$(( volume_bytes / 100 * percent / 1073741824 )) + if [ "$size_gib" -lt 1 ]; then + echo "Brainstore cache volume at $BRAINSTORE_CACHE_DIR is too small ($volume_bytes bytes); set objectStoreCacheFileSize" >&2 + exit 1 + fi + export BRAINSTORE_OBJECT_STORE_CACHE_FILE_SIZE="${size_gib}Gi" + echo "Using ${size_gib}Gi object store cache (${percent}% of $volume_bytes bytes at $BRAINSTORE_CACHE_DIR)" + exec brainstore web +{{- end -}} +{{- end -}} diff --git a/braintrust/templates/brainstore-fastreader-configmap.yaml b/braintrust/templates/brainstore-fastreader-configmap.yaml index e2295f9..a5dac9c 100644 --- a/braintrust/templates/brainstore-fastreader-configmap.yaml +++ b/braintrust/templates/brainstore-fastreader-configmap.yaml @@ -18,7 +18,9 @@ data: BRAINSTORE_PORT: {{ .Values.brainstore.fastreader.service.port | quote }} BRAINSTORE_CACHE_DIR: {{ .Values.brainstore.fastreader.cacheDir | quote }} BRAINSTORE_OBJECT_STORE_CACHE_MEMORY_LIMIT: {{ .Values.brainstore.fastreader.objectStoreCacheMemoryLimit | quote }} + {{- if .Values.brainstore.fastreader.objectStoreCacheFileSize }} BRAINSTORE_OBJECT_STORE_CACHE_FILE_SIZE: {{ .Values.brainstore.fastreader.objectStoreCacheFileSize | quote }} + {{- end }} {{- if eq .Values.cloud "azure" }} # See here for reference: # https://docs.rs/object_store/latest/object_store/azure/struct.MicrosoftAzureBuilder.html diff --git a/braintrust/templates/brainstore-fastreader-deployment.yaml b/braintrust/templates/brainstore-fastreader-deployment.yaml index 6ca1d5e..dc4c2b2 100644 --- a/braintrust/templates/brainstore-fastreader-deployment.yaml +++ b/braintrust/templates/brainstore-fastreader-deployment.yaml @@ -80,8 +80,11 @@ spec: securityContext: {{- toYaml . | nindent 12 }} {{- end }} - command: ["brainstore"] - args: ["web"] + {{- include "braintrust.brainstoreCommand" (dict + "path" "brainstore.fastreader" + "config" .Values.brainstore.fastreader + "defaultPercent" 90 + ) | nindent 10 }} ports: - containerPort: {{ .Values.brainstore.fastreader.service.port }} resources: diff --git a/braintrust/templates/brainstore-reader-configmap.yaml b/braintrust/templates/brainstore-reader-configmap.yaml index adc27e6..6af7881 100644 --- a/braintrust/templates/brainstore-reader-configmap.yaml +++ b/braintrust/templates/brainstore-reader-configmap.yaml @@ -18,7 +18,9 @@ data: BRAINSTORE_PORT: {{ .Values.brainstore.reader.service.port | quote }} BRAINSTORE_CACHE_DIR: {{ .Values.brainstore.reader.cacheDir | quote }} BRAINSTORE_OBJECT_STORE_CACHE_MEMORY_LIMIT: {{ .Values.brainstore.reader.objectStoreCacheMemoryLimit | quote }} + {{- if .Values.brainstore.reader.objectStoreCacheFileSize }} BRAINSTORE_OBJECT_STORE_CACHE_FILE_SIZE: {{ .Values.brainstore.reader.objectStoreCacheFileSize | quote }} + {{- end }} {{- if eq .Values.cloud "azure" }} # See here for reference: # https://docs.rs/object_store/latest/object_store/azure/struct.MicrosoftAzureBuilder.html diff --git a/braintrust/templates/brainstore-reader-deployment.yaml b/braintrust/templates/brainstore-reader-deployment.yaml index f0342a7..1efe307 100644 --- a/braintrust/templates/brainstore-reader-deployment.yaml +++ b/braintrust/templates/brainstore-reader-deployment.yaml @@ -80,8 +80,11 @@ spec: securityContext: {{- toYaml . | nindent 12 }} {{- end }} - command: ["brainstore"] - args: ["web"] + {{- include "braintrust.brainstoreCommand" (dict + "path" "brainstore.reader" + "config" .Values.brainstore.reader + "defaultPercent" 90 + ) | nindent 10 }} ports: - containerPort: {{ .Values.brainstore.reader.service.port }} resources: diff --git a/braintrust/templates/brainstore-writer-configmap.yaml b/braintrust/templates/brainstore-writer-configmap.yaml index 5d4dda5..85013fd 100644 --- a/braintrust/templates/brainstore-writer-configmap.yaml +++ b/braintrust/templates/brainstore-writer-configmap.yaml @@ -18,7 +18,9 @@ data: BRAINSTORE_PORT: {{ .Values.brainstore.writer.service.port | quote }} BRAINSTORE_CACHE_DIR: {{ .Values.brainstore.writer.cacheDir | quote }} BRAINSTORE_OBJECT_STORE_CACHE_MEMORY_LIMIT: {{ .Values.brainstore.writer.objectStoreCacheMemoryLimit | quote }} + {{- if .Values.brainstore.writer.objectStoreCacheFileSize }} BRAINSTORE_OBJECT_STORE_CACHE_FILE_SIZE: {{ .Values.brainstore.writer.objectStoreCacheFileSize | quote }} + {{- end }} {{- if eq .Values.cloud "azure" }} # See here for reference: # https://docs.rs/object_store/latest/object_store/azure/struct.MicrosoftAzureBuilder.html diff --git a/braintrust/templates/brainstore-writer-deployment.yaml b/braintrust/templates/brainstore-writer-deployment.yaml index f42b692..6513655 100644 --- a/braintrust/templates/brainstore-writer-deployment.yaml +++ b/braintrust/templates/brainstore-writer-deployment.yaml @@ -80,8 +80,11 @@ spec: securityContext: {{- toYaml . | nindent 12 }} {{- end }} - command: ["brainstore"] - args: ["web"] + {{- include "braintrust.brainstoreCommand" (dict + "path" "brainstore.writer" + "config" .Values.brainstore.writer + "defaultPercent" 75 + ) | nindent 10 }} ports: - containerPort: {{ .Values.brainstore.writer.service.port }} resources: diff --git a/braintrust/tests/brainstore-cache-file-size_test.yaml b/braintrust/tests/brainstore-cache-file-size_test.yaml new file mode 100644 index 0000000..854a675 --- /dev/null +++ b/braintrust/tests/brainstore-cache-file-size_test.yaml @@ -0,0 +1,186 @@ +suite: test Brainstore object store cache file size +templates: + - brainstore-reader-deployment.yaml + - brainstore-reader-configmap.yaml + - brainstore-fastreader-deployment.yaml + - brainstore-fastreader-configmap.yaml + - brainstore-writer-deployment.yaml + - brainstore-writer-configmap.yaml +release: + namespace: "braintrust" +tests: + - it: should start Brainstore directly when the writer cache size is set + template: brainstore-writer-deployment.yaml + values: + - __fixtures__/base-values.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].command + value: ["brainstore"] + - equal: + path: spec.template.spec.containers[0].args + value: ["web"] + + - it: should keep the writer cache size in the configmap when it is set + template: brainstore-writer-configmap.yaml + values: + - __fixtures__/base-values.yaml + asserts: + - equal: + path: data.BRAINSTORE_OBJECT_STORE_CACHE_FILE_SIZE + value: "50Gi" + + - it: should size the writer cache to 75% of the volume by default + template: brainstore-writer-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + brainstore.writer.objectStoreCacheFileSize: "" + brainstore.writer.objectStoreCacheFileSizePercent: 75 + brainstore.writer.volume.size: "" + asserts: + - equal: + path: spec.template.spec.containers[0].command + value: ["/bin/sh", "-c"] + - matchRegex: + path: spec.template.spec.containers[0].args[0] + pattern: "(?m)^percent=75$" + - matchRegex: + path: spec.template.spec.containers[0].args[0] + pattern: "(?m)^limit_bytes=0$" + - matchRegex: + path: spec.template.spec.containers[0].args[0] + pattern: "(?m)^exec brainstore web$" + + - it: should omit the writer cache size from the configmap when it is computed + template: brainstore-writer-configmap.yaml + values: + - __fixtures__/base-values.yaml + set: + brainstore.writer.objectStoreCacheFileSize: "" + asserts: + - notExists: + path: data.BRAINSTORE_OBJECT_STORE_CACHE_FILE_SIZE + + - it: should default the writer percent to 75 when values predate the percent setting + template: brainstore-writer-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + brainstore.writer.objectStoreCacheFileSize: "" + asserts: + - matchRegex: + path: spec.template.spec.containers[0].args[0] + pattern: "(?m)^percent=75$" + + - it: should default the reader percent to 90 when values predate the percent setting + template: brainstore-reader-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + brainstore.reader.objectStoreCacheFileSize: "" + asserts: + - matchRegex: + path: spec.template.spec.containers[0].args[0] + pattern: "(?m)^percent=90$" + + - it: should default the fast reader percent to 90 when values predate the percent setting + template: brainstore-fastreader-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + brainstore.fastreader.objectStoreCacheFileSize: "" + asserts: + - matchRegex: + path: spec.template.spec.containers[0].args[0] + pattern: "(?m)^percent=90$" + + - it: should omit the reader cache size from the configmap when it is computed + template: brainstore-reader-configmap.yaml + values: + - __fixtures__/base-values.yaml + set: + brainstore.reader.objectStoreCacheFileSize: "" + asserts: + - notExists: + path: data.BRAINSTORE_OBJECT_STORE_CACHE_FILE_SIZE + + - it: should omit the fast reader cache size from the configmap when it is computed + template: brainstore-fastreader-configmap.yaml + values: + - __fixtures__/base-values.yaml + set: + brainstore.fastreader.objectStoreCacheFileSize: "" + asserts: + - notExists: + path: data.BRAINSTORE_OBJECT_STORE_CACHE_FILE_SIZE + + - it: should cap the cache volume at volume.size + template: brainstore-writer-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + brainstore.writer.objectStoreCacheFileSize: "" + brainstore.writer.volume.size: "200Gi" + asserts: + - matchRegex: + path: spec.template.spec.containers[0].args[0] + pattern: "(?m)^limit_bytes=214748364800$" + + - it: should cap the cache volume at the smaller of volume.size and volume.sizeLimit + template: brainstore-writer-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + brainstore.writer.objectStoreCacheFileSize: "" + brainstore.writer.volume.size: "1000Gi" + brainstore.writer.volume.sizeLimit: "900G" + asserts: + - matchRegex: + path: spec.template.spec.containers[0].args[0] + pattern: "(?m)^limit_bytes=900000000000$" + + - it: should use a custom percent + template: brainstore-reader-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + brainstore.reader.objectStoreCacheFileSize: "" + brainstore.reader.objectStoreCacheFileSizePercent: 60 + asserts: + - matchRegex: + path: spec.template.spec.containers[0].args[0] + pattern: "(?m)^percent=60$" + + - it: should reject a percent above 100 + template: brainstore-writer-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + brainstore.writer.objectStoreCacheFileSize: "" + brainstore.writer.objectStoreCacheFileSizePercent: 101 + asserts: + - failedTemplate: + errorMessage: "brainstore.writer.objectStoreCacheFileSizePercent must be a whole number between 1 and 100" + + - it: should reject a non-numeric percent + template: brainstore-writer-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + brainstore.writer.objectStoreCacheFileSize: "" + brainstore.writer.objectStoreCacheFileSizePercent: "75%" + asserts: + - failedTemplate: + errorMessage: "brainstore.writer.objectStoreCacheFileSizePercent must be a whole number between 1 and 100" + + - it: should reject a volume size it cannot parse + template: brainstore-writer-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + brainstore.writer.objectStoreCacheFileSize: "" + brainstore.writer.volume.size: "1.5Ti" + asserts: + - failedTemplate: + errorMessage: "brainstore.writer.volume.size must be a whole-number Kubernetes quantity such as 900Gi (got \"1.5Ti\")" diff --git a/braintrust/values.yaml b/braintrust/values.yaml index d51977b..586dd48 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -446,7 +446,11 @@ brainstore: memory: "32Gi" cacheDir: "/mnt/tmp/brainstore" objectStoreCacheMemoryLimit: "1Gi" - objectStoreCacheFileSize: "1000Gi" + # Optional. Fixed object store cache size (e.g. "500Gi"). When empty, the cache is + # sized at pod startup to objectStoreCacheFileSizePercent of the cache volume + # (the filesystem at cacheDir, capped by volume.sizeLimit and volume.size). + objectStoreCacheFileSize: "" + objectStoreCacheFileSizePercent: 90 verbose: true # Optional: Volume configuration for cache storage # When not set, uses default emptyDir: {} (backward compatible) @@ -504,7 +508,11 @@ brainstore: memory: "32Gi" cacheDir: "/mnt/tmp/brainstore" objectStoreCacheMemoryLimit: "1Gi" - objectStoreCacheFileSize: "1000Gi" + # Optional. Fixed object store cache size (e.g. "500Gi"). When empty, the cache is + # sized at pod startup to objectStoreCacheFileSizePercent of the cache volume + # (the filesystem at cacheDir, capped by volume.sizeLimit and volume.size). + objectStoreCacheFileSize: "" + objectStoreCacheFileSizePercent: 90 verbose: true # Optional: Volume configuration for cache storage # When not set, uses default emptyDir: {} (backward compatible) @@ -562,7 +570,11 @@ brainstore: memory: "64Gi" cacheDir: "/mnt/tmp/brainstore" objectStoreCacheMemoryLimit: "1Gi" - objectStoreCacheFileSize: "1000Gi" + # Optional. Fixed object store cache size (e.g. "500Gi"). When empty, the cache is + # sized at pod startup to objectStoreCacheFileSizePercent of the cache volume + # (the filesystem at cacheDir, capped by volume.sizeLimit and volume.size). + objectStoreCacheFileSize: "" + objectStoreCacheFileSizePercent: 75 verbose: true # Optional: Volume configuration for cache storage # When not set, uses default emptyDir: {} (backward compatible) From 0e2fcca6f0696db5fd50d1a83c3d39d210dc8414 Mon Sep 17 00:00:00 2001 From: Jeff McCollum <16550786+jeffmccollum@users.noreply.github.com> Date: Thu, 8 Oct 2026 00:46:58 +0000 Subject: [PATCH 2/2] Cap computed Brainstore cache size at the ephemeral storage budget On EKS the documented storage budget is ephemeralStorage.request, so the computed cache size now also respects ephemeralStorage.request and ephemeralStorage.limit. Document per-platform behavior in the README. Co-Authored-By: Claude Opus 5.5 --- braintrust/README.md | 19 ++++++- braintrust/templates/_helpers.tpl | 13 +++-- .../brainstore-cache-file-size_test.yaml | 57 +++++++++++++++++++ braintrust/values.yaml | 9 ++- 4 files changed, 87 insertions(+), 11 deletions(-) diff --git a/braintrust/README.md b/braintrust/README.md index 02c5e74..bb4be07 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -297,9 +297,22 @@ use 75% because they need more free disk. This matches how the AWS Terraform module sizes the cache from the instance's local disk. The cache volume size is the size of the filesystem mounted at `cacheDir`, -capped by `volume.sizeLimit` and `volume.size` when they are set. With a plain -`emptyDir`, that filesystem is the node's disk, so give Brainstore pods -dedicated nodes (see the anti-affinity examples above) or set `volume.size`. +capped by the smallest of `volume.sizeLimit`, `volume.size`, +`ephemeralStorage.request` and `ephemeralStorage.limit` that is set. This works +the same way on every cloud: + +| Platform | Cache volume | What sets the size | +| --- | --- | --- | +| GKE Autopilot | `emptyDir` on the node's local SSD | `volume.size` (also the ephemeral-storage request) or `ephemeralStorage.request` | +| GKE Standard | `emptyDir` on the node pool's disk | The node disk, capped by any size set above | +| EKS (managed nodes, Karpenter, Auto Mode) | `emptyDir` on the node's kubelet disk | `ephemeralStorage.request` (see [AWS EKS Local Storage](#aws-eks-local-storage)) | +| AKS with Azure Container Storage | Ephemeral volume of `volume.size` | `volume.size` | +| AKS without Azure Container Storage | `emptyDir` on the node's disk | The node disk, capped by any size set above | + +An `emptyDir` lives on a node disk that container images, logs and other pods +also use. When no size is set, the percentage applies to that whole disk, so +either give Brainstore pods dedicated nodes (see the anti-affinity examples +above) or set one of the sizes above. ```yaml brainstore: diff --git a/braintrust/templates/_helpers.tpl b/braintrust/templates/_helpers.tpl index 442ac6f..b2251dd 100644 --- a/braintrust/templates/_helpers.tpl +++ b/braintrust/templates/_helpers.tpl @@ -216,8 +216,10 @@ size from its ConfigMap. Otherwise the container sets BRAINSTORE_OBJECT_STORE_CACHE_FILE_SIZE at startup to objectStoreCacheFileSizePercent of the cache volume, the same way the AWS Terraform module sizes the cache from the instance's local disk. The cache -volume size is the size of the filesystem mounted at cacheDir, capped by -volume.sizeLimit and volume.size when they are set. +volume size is the size of the filesystem mounted at cacheDir, capped by the +smallest of volume.sizeLimit, volume.size, ephemeralStorage.request and +ephemeralStorage.limit that is set. The cap matters when the cache is an +emptyDir on a node disk shared with images, logs and other pods. Template-level percent defaults keep `helm upgrade --reuse-values` working from chart versions that predate objectStoreCacheFileSizePercent. @@ -236,10 +238,11 @@ args: ["web"] {{- fail (printf "%s.objectStoreCacheFileSizePercent must be a whole number between 1 and 100" .path) -}} {{- end -}} {{- $limitBytes := 0 -}} -{{- range $field := list "sizeLimit" "size" -}} -{{- $quantity := index ($config.volume | default dict) $field -}} +{{- range $field := list "volume.sizeLimit" "volume.size" "ephemeralStorage.request" "ephemeralStorage.limit" -}} +{{- $parts := splitList "." $field -}} +{{- $quantity := index ((index $config (first $parts)) | default dict) (last $parts) -}} {{- if $quantity -}} -{{- $bytes := include "braintrust.quantityToBytes" (dict "quantity" $quantity "path" (printf "%s.volume.%s" $.path $field)) | int64 -}} +{{- $bytes := include "braintrust.quantityToBytes" (dict "quantity" $quantity "path" (printf "%s.%s" $.path $field)) | int64 -}} {{- if or (eq $limitBytes 0) (lt $bytes $limitBytes) -}} {{- $limitBytes = $bytes -}} {{- end -}} diff --git a/braintrust/tests/brainstore-cache-file-size_test.yaml b/braintrust/tests/brainstore-cache-file-size_test.yaml index 854a675..7b8de38 100644 --- a/braintrust/tests/brainstore-cache-file-size_test.yaml +++ b/braintrust/tests/brainstore-cache-file-size_test.yaml @@ -184,3 +184,60 @@ tests: asserts: - failedTemplate: errorMessage: "brainstore.writer.volume.size must be a whole-number Kubernetes quantity such as 900Gi (got \"1.5Ti\")" + + - it: should cap the cache volume at ephemeralStorage.request + template: brainstore-writer-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: aws + brainstore.writer.objectStoreCacheFileSize: "" + brainstore.writer.volume.size: "" + brainstore.writer.ephemeralStorage.request: "1700Gi" + asserts: + - matchRegex: + path: spec.template.spec.containers[0].args[0] + pattern: "(?m)^limit_bytes=1825361100800$" + + - it: should cap the cache volume at ephemeralStorage.limit when it is the smallest + template: brainstore-writer-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: aws + brainstore.writer.objectStoreCacheFileSize: "" + brainstore.writer.volume.size: "" + brainstore.writer.ephemeralStorage.request: "1700Gi" + brainstore.writer.ephemeralStorage.limit: "1500Gi" + asserts: + - matchRegex: + path: spec.template.spec.containers[0].args[0] + pattern: "(?m)^limit_bytes=1610612736000$" + + - it: should reject an ephemeral storage size it cannot parse + template: brainstore-writer-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + brainstore.writer.objectStoreCacheFileSize: "" + brainstore.writer.ephemeralStorage.request: "lots" + asserts: + - failedTemplate: + errorMessage: "brainstore.writer.ephemeralStorage.request must be a whole-number Kubernetes quantity such as 900Gi (got \"lots\")" + + - it: should size the cache on Azure Container Storage volumes + template: brainstore-writer-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + cloud: azure + azure.enableAzureContainerStorageDriver: true + brainstore.writer.objectStoreCacheFileSize: "" + brainstore.writer.volume.size: "200Gi" + asserts: + - matchRegex: + path: spec.template.spec.containers[0].args[0] + pattern: "(?m)^limit_bytes=214748364800$" + - equal: + path: spec.template.spec.volumes[0].ephemeral.volumeClaimTemplate.spec.resources.requests.storage + value: "200Gi" diff --git a/braintrust/values.yaml b/braintrust/values.yaml index 586dd48..b1c0923 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -448,7 +448,8 @@ brainstore: objectStoreCacheMemoryLimit: "1Gi" # Optional. Fixed object store cache size (e.g. "500Gi"). When empty, the cache is # sized at pod startup to objectStoreCacheFileSizePercent of the cache volume - # (the filesystem at cacheDir, capped by volume.sizeLimit and volume.size). + # (the filesystem at cacheDir, capped by the smallest of volume.sizeLimit, volume.size, + # ephemeralStorage.request and ephemeralStorage.limit that is set). objectStoreCacheFileSize: "" objectStoreCacheFileSizePercent: 90 verbose: true @@ -510,7 +511,8 @@ brainstore: objectStoreCacheMemoryLimit: "1Gi" # Optional. Fixed object store cache size (e.g. "500Gi"). When empty, the cache is # sized at pod startup to objectStoreCacheFileSizePercent of the cache volume - # (the filesystem at cacheDir, capped by volume.sizeLimit and volume.size). + # (the filesystem at cacheDir, capped by the smallest of volume.sizeLimit, volume.size, + # ephemeralStorage.request and ephemeralStorage.limit that is set). objectStoreCacheFileSize: "" objectStoreCacheFileSizePercent: 90 verbose: true @@ -572,7 +574,8 @@ brainstore: objectStoreCacheMemoryLimit: "1Gi" # Optional. Fixed object store cache size (e.g. "500Gi"). When empty, the cache is # sized at pod startup to objectStoreCacheFileSizePercent of the cache volume - # (the filesystem at cacheDir, capped by volume.sizeLimit and volume.size). + # (the filesystem at cacheDir, capped by the smallest of volume.sizeLimit, volume.size, + # ephemeralStorage.request and ephemeralStorage.limit that is set). objectStoreCacheFileSize: "" objectStoreCacheFileSizePercent: 75 verbose: true