improvement(setup): complete knowledge and update flows (#6521)

Author: TheodoreSpeaks

README.md

@@ -105,6 +105,7 @@ Manage your install with `bun run sim`:
 
 ```bash
 bun run sim start | stop | restart   # bring your install up / down / cycle
+bun run sim update                   # pull/rebuild and apply Compose images
 bun run sim status                    # what's installed and healthy
 bun run sim logs                      # follow logs
 bun run sim doctor                    # diagnose configuration problems

apps/docs/content/docs/en/platform/self-hosting/docker.mdx

@@ -149,8 +149,8 @@ docker compose -f docker-compose.prod.yml logs migrations
 # Is the scheduler firing?
 docker compose -f docker-compose.prod.yml logs -f cron
 
-# Upgrade: bump SIM_VERSION in .env, then
-docker compose -f docker-compose.prod.yml pull && docker compose -f docker-compose.prod.yml up -d
+# Upgrade: bump SIM_VERSION in .env when pinned, then
+bun run sim update
 ```
 
 <FAQ items={[
@@ -159,4 +159,3 @@ docker compose -f docker-compose.prod.yml pull && docker compose -f docker-compo
   { question: "How do I back up and restore the database?", answer: "Back up with: docker compose -f docker-compose.prod.yml exec db pg_dump -U postgres simstudio > backup.sql. Restore with: docker compose -f docker-compose.prod.yml exec -T db psql -U postgres simstudio < backup.sql. The database data is persisted in a Docker volume named postgres_data."},
   { question: "Can I customize the PostgreSQL credentials?", answer: "Yes. The docker-compose.prod.yml uses environment variable defaults: POSTGRES_USER (default: postgres), POSTGRES_PASSWORD (default: postgres), POSTGRES_DB (default: simstudio), and POSTGRES_PORT (default: 5432). Set these in your .env file to override them." },
 ]} />
-

apps/docs/content/docs/en/platform/self-hosting/upgrades.mdx

@@ -142,11 +142,14 @@ kubectl logs -n simstudio deploy/sim-app -c migrations --tail=100
   <Tab value="Docker Compose">
 
 ```bash
-docker compose -f docker-compose.prod.yml pull
-docker compose -f docker-compose.prod.yml up -d
+bun run sim update
 docker compose -f docker-compose.prod.yml logs migrations
 ```
 
+`bun run sim update` pulls the versions configured by `SIM_VERSION` (or `latest` when it is
+unset), recreates the changed services, and keeps data volumes. It is equivalent to running
+`docker compose pull` followed by `docker compose up -d`.
+
 There is a short window where the app is unavailable while containers restart. Compose has no rolling-update mechanism — plan a maintenance window, or run Kubernetes if you need zero-downtime upgrades.
 
   </Tab>

scripts/setup/capability-config.test.ts

@@ -10,6 +10,7 @@ import {
   defineCapabilitySetup,
   EMAIL_SETUP,
   getOAuthClientSetupFields,
+  KNOWLEDGE_EMBEDDINGS_SETUP,
   STORAGE_SETUP,
 } from './capability-config.ts'
 import { getCapabilitySetupOptions } from './capability-setup.ts'
@@ -76,6 +77,12 @@ describe('capability setup configuration', () => {
     )
   })
 
+  it('offers OpenAI first for fresh knowledge embedding setup', () => {
+    expect(
+      getCapabilitySetupOptions(KNOWLEDGE_EMBEDDINGS_SETUP).map((option) => option.id)
+    ).toEqual(['openai', 'azure-openai', 'openrouter'])
+  })
+
   it('maps every OAuth runtime field to a CLI input mode in runtime order', () => {
     for (const id of Object.keys(OAUTH_CLIENT_CAPABILITIES) as Array<
       keyof typeof OAUTH_CLIENT_CAPABILITIES

scripts/setup/capability-config.ts

@@ -823,7 +823,7 @@ export const KNOWLEDGE_EMBEDDINGS_SETUP = defineCapabilitySetup(KNOWLEDGE_EMBEDD
       ],
     },
   },
-  optionOrder: ['azure-openai', 'openai', 'openrouter'],
+  optionOrder: ['openai', 'azure-openai', 'openrouter'],
 })
 
 export const CAPABILITY_SETUPS = [

scripts/setup/capability-setup.ts

@@ -49,6 +49,8 @@ interface PromptState {
   values: Record<string, string>
 }
 
+const SKIP_OPTION_ID = '__skip-capability-setup__'
+
 /** Stages a capability transition into a larger setup run without losing prompt context. */
 export function stageCapabilitySetupTransition(
   currentValues: Map<string, string>,
@@ -176,11 +178,11 @@ export function getCapabilitySetupOptions(
   })
 }
 
-/** Resolves the setup option representing the effective current configuration. */
+/** Resolves the setup option representing the effective current configuration, if one exists. */
 export function resolveCurrentCapabilitySetupOptionId(
   setup: CapabilitySetupDefinition,
   values: EnvCapabilityValues
-): string {
+): string | undefined {
   const options = getCapabilitySetupOptions(setup)
   const explicitAction = options.find(
     (option) =>
@@ -214,9 +216,10 @@ export function resolveCurrentCapabilitySetupOptionId(
 
   const firstAction = options.find((option) => option.kind === 'action')
   if (firstAction) return firstAction.id
-  throw new Error(
-    `Capability ${setup.definition.id} has no setup option for its current configuration`
-  )
+  if (options.length === 0) {
+    throw new Error(`Capability ${setup.definition.id} has no setup options`)
+  }
+  return undefined
 }
 
 /** Applies selector and activation inference to the CLI-entered values. */
@@ -473,6 +476,21 @@ async function renderPrompts(prompts: readonly SetupPrompt[], state: PromptState
   }
 }
 
+async function promptSelectedCapabilitySetup(
+  setup: CapabilitySetupDefinition,
+  selected: ResolvedSetupOption,
+  currentValues: ReadonlyMap<string, string>
+): Promise<EnvCapabilitySetupTransition> {
+  const state: PromptState = {
+    setup,
+    optionId: selected.id,
+    currentValues,
+    values: {},
+  }
+  await renderPrompts(selected.prompts, state)
+  return buildCapabilitySetupTransition(setup, selected.id, state.values, currentValues)
+}
+
 /** Renders a CLI-owned capability setup and returns its validated environment transition. */
 export async function promptCapabilitySetup(
   setup: CapabilitySetupDefinition,
@@ -497,12 +515,44 @@ export async function promptCapabilitySetup(
     )
   }
 
-  const state: PromptState = {
-    setup,
-    optionId: selected.id,
-    currentValues,
-    values: {},
+  return promptSelectedCapabilitySetup(setup, selected, currentValues)
+}
+
+/** Offers capability providers while allowing the user to leave configuration unchanged. */
+export async function promptOptionalCapabilitySetup(
+  setup: CapabilitySetupDefinition,
+  currentValues: ReadonlyMap<string, string>,
+  context: CapabilitySetupContext,
+  skipHint: string
+): Promise<EnvCapabilitySetupTransition | null> {
+  const options = getCapabilitySetupOptions(setup)
+  if (options.some((option) => option.id === SKIP_OPTION_ID)) {
+    throw new Error(`Capability ${setup.definition.id} uses reserved option ${SKIP_OPTION_ID}`)
   }
-  await renderPrompts(selected.prompts, state)
-  return buildCapabilitySetupTransition(setup, selected.id, state.values, currentValues)
+  const currentOptionId = resolveCurrentCapabilitySetupOptionId(setup, currentValues)
+  const selectedOptionId = await p.select({
+    message: setup.message,
+    options: [
+      ...options.map((option) => ({
+        value: option.id,
+        label: option.label,
+        hint: markCurrentlyUsed(resolveHint(option.hint, context), option.id === currentOptionId),
+      })),
+      {
+        value: SKIP_OPTION_ID,
+        label: 'Not now',
+        hint: currentOptionId ? 'leave the current configuration unchanged' : skipHint,
+      },
+    ],
+    initialValue: currentOptionId ?? options[0]?.id,
+  })
+  if (selectedOptionId === SKIP_OPTION_ID) return null
+
+  const selected = options.find((option) => option.id === selectedOptionId)
+  if (!selected) {
+    throw new Error(
+      `Capability ${setup.definition.id} returned unknown setup option ${selectedOptionId}`
+    )
+  }
+  return promptSelectedCapabilitySetup(setup, selected, currentValues)
 }

scripts/setup/index.ts

@@ -18,6 +18,7 @@ const USAGE = `Usage:
   bun run sim setup <feature>                  configure one feature
   bun run sim doctor [--fix] [--json]          check your setup
   bun run sim start | stop | restart           bring your install up / down / cycle
+  bun run sim update                           pull/rebuild and apply Compose images
   bun run sim status                           what's installed and healthy
   bun run sim logs                             follow logs
   bun run sim down                             remove containers (data kept)

scripts/setup/lifecycle.test.ts

@@ -0,0 +1,14 @@
+import { describe, expect, it } from 'bun:test'
+import { getComposeUpdateMode, isLifecycleCommand } from './lifecycle.ts'
+
+describe('setup lifecycle', () => {
+  it('recognizes update as a lifecycle command', () => {
+    expect(isLifecycleCommand('update')).toBe(true)
+  })
+
+  it('pulls published installs and rebuilds source installs', () => {
+    expect(getComposeUpdateMode('/repo/docker-compose.prod.yml')).toBe('pull')
+    expect(getComposeUpdateMode('/repo/docker-compose.local.yml')).toBe('build')
+    expect(() => getComposeUpdateMode('/repo/compose.yml')).toThrow(/Unsupported Sim Compose file/)
+  })
+})

scripts/setup/lifecycle.ts

@@ -20,6 +20,7 @@ export const LIFECYCLE_COMMANDS = [
   'start',
   'stop',
   'restart',
+  'update',
   'status',
   'logs',
   'down',
@@ -325,6 +326,50 @@ function restart(install: Install): void {
   p.note(k8sReachHints(install.context), 'Kubernetes is managed with kubectl')
 }
 
+export type ComposeUpdateMode = 'pull' | 'build'
+
+/** Resolves how a setup-managed Compose install obtains its next image. */
+export function getComposeUpdateMode(file: string): ComposeUpdateMode {
+  const name = path.basename(file)
+  if (name === 'docker-compose.prod.yml') return 'pull'
+  if (name === 'docker-compose.local.yml') return 'build'
+  throw new Error(`Unsupported Sim Compose file: ${file}`)
+}
+
+function update(install: Install): void {
+  if (install.kind === 'dev') {
+    throw new SetupError('sim update is only available for Docker Compose installs.', [
+      'update the source checkout with git, run bun install, then restart bun run dev:full',
+    ])
+  }
+  if (install.kind === 'k8s') {
+    throw new SetupError('sim update does not upgrade Kubernetes releases.', [
+      'upgrade the release with helm after reviewing the chart and release notes',
+    ])
+  }
+
+  const mode = getComposeUpdateMode(install.file)
+  const spin = p.spinner()
+  if (mode === 'pull') {
+    spin.start('Pulling configured Sim images…')
+    dockerRun(composeArgs(install, 'pull'), 'docker compose pull failed', install.dir)
+  } else {
+    spin.start('Rebuilding Sim images with current base images…')
+    dockerRun(composeArgs(install, 'build', '--pull'), 'docker compose build failed', install.dir)
+  }
+  spin.message('Applying updated images and running migrations…')
+  dockerRun(composeArgs(install, 'up', '-d'), 'docker compose up failed', install.dir)
+  spin.stop('Sim updated (data volumes kept)')
+  p.note(
+    [
+      `version: ${theme.command(`SIM_VERSION in ${path.join(install.dir, '.env')}`)} (latest when unset)`,
+      `check:   ${theme.command('bun run sim status')}`,
+      `logs:    ${theme.command('bun run sim logs')}`,
+    ].join('\n'),
+    'Update complete'
+  )
+}
+
 function showLogs(install: Install): void {
   if (install.kind === 'compose') {
     dockerInherit(composeArgs(install, 'logs', '-f', '--tail', '100'), install.dir)
@@ -488,6 +533,8 @@ export async function runLifecycle(command: LifecycleCommand): Promise<void> {
       return stop(install)
     case 'restart':
       return restart(install)
+    case 'update':
+      return update(install)
     case 'logs':
       return showLogs(install)
     case 'down':

scripts/setup/modes/compose.ts

@@ -13,6 +13,7 @@ import {
   collectSecrets,
   mothershipOverride,
   promptCopilotKey,
+  promptKnowledgeEmbeddings,
   promptLlmKeys,
   promptSecurity,
   promptSignInProviders,
@@ -117,9 +118,13 @@ export async function runComposeMode(detection: Detection, quick: boolean): Prom
   if (copilotKey) values.COPILOT_API_KEY = copilotKey
   Object.assign(values, chatFlagValues(copilotKey))
   Object.assign(values, await promptLlmKeys(detection, !quick))
+  const stagedVars = new Map(root.vars)
+  for (const [key, value] of Object.entries(values)) stagedVars.set(key, value)
+  const embeddings = await promptKnowledgeEmbeddings(stagedVars, { containerized: true })
+  if (embeddings) {
+    stageCapabilitySetupTransition(stagedVars, values, remove, embeddings)
+  }
   if (!quick) {
-    const stagedVars = new Map(root.vars)
-    for (const [key, value] of Object.entries(values)) stagedVars.set(key, value)
     const storage = await promptCapabilitySetup(STORAGE_SETUP, stagedVars, {
       containerized: true,
     })