diff --git a/crates/fctl-agent/src/claude.rs b/crates/fctl-agent/src/claude.rs index dfa3b123..10fc7dd8 100644 --- a/crates/fctl-agent/src/claude.rs +++ b/crates/fctl-agent/src/claude.rs @@ -33,6 +33,76 @@ pub const FORBIDDEN_ENV: &[&str] = &[ "CLAUDE_CODE_OAUTH_REFRESH_TOKEN", ]; +/// The oldest Claude Code fctl works with. 2.1.101 is the first that asks +/// its host for a fresh OAuth access token (the `oauth_token_refresh` +/// control request), which sessions on a full Claude login need once +/// theirs expires; Remote Control's `remote_control` request came in +/// 2.1.63. +pub const MIN_VERSION: Version = Version(2, 1, 101); + +/// A Claude Code version: major, minor, patch. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct Version(pub u32, pub u32, pub u32); + +impl Version { + /// The first `X.Y.Z` in `claude --version`'s output, `2.1.302 (Claude + /// Code)`; a pre-release suffix doesn't count. + pub fn parse(text: &str) -> Option { + text.split_whitespace().find_map(|word| { + let word = word.trim_start_matches('v'); + let mut parts = word.splitn(3, '.'); + let major = parts.next()?.parse().ok()?; + let minor = parts.next()?.parse().ok()?; + let patch = parts.next()?; + let digits = patch.chars().take_while(char::is_ascii_digit).count(); + let patch = patch[..digits].parse().ok()?; + Some(Self(major, minor, patch)) + }) + } +} + +impl std::fmt::Display for Version { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}.{}.{}", self.0, self.1, self.2) + } +} + +/// Asks `bin --version` which Claude Code the sandbox has. `Err` says why +/// sessions can't run with it: it is missing, or older than +/// [`MIN_VERSION`]. `Ok(None)` when it doesn't say. +pub async fn check_version(bin: &str) -> std::result::Result, String> { + let run = Command::new(bin) + .arg("--version") + .stdin(Stdio::null()) + .kill_on_drop(true) + .output(); + let out = match tokio::time::timeout(Duration::from_secs(30), run).await { + Ok(Ok(out)) => out, + Ok(Err(e)) if e.kind() == std::io::ErrorKind::NotFound => { + return Err(format!( + "This sandbox has no Claude Code (`{bin}` is not on the PATH): install it in the image, or build the image from ghcr.io/hyperstructure/sandbox, which has it" + )); + } + Ok(Err(e)) => { + tracing::warn!("running {bin} --version: {e}"); + return Ok(None); + } + Err(_) => { + tracing::warn!("{bin} --version did not answer within 30 seconds"); + return Ok(None); + } + }; + let Some(version) = Version::parse(&String::from_utf8_lossy(&out.stdout)) else { + return Ok(None); + }; + if version < MIN_VERSION { + return Err(format!( + "This sandbox has Claude Code {version}, and fctl needs {MIN_VERSION} or later: update it in the image (ghcr.io/hyperstructure/sandbox has the newest)" + )); + } + Ok(Some(version)) +} + pub enum ClaudeOut { /// A stream-json message. Message(Value), @@ -107,6 +177,52 @@ pub fn user_settings(project: &ProjectSettings, co_authors: &[GitIdentity]) -> V settings } +/// Claude's user settings: the image's own `~/.claude/settings.json`, with +/// fctl's ([`user_settings`]) over it. fctl decides the permission mode, +/// `attribution`, `disableClaudeAiConnectors` and the auto-updater, and +/// drops what could move Claude off the subscription or out of bypass mode: +/// `apiKeyHelper`, `ANTHROPIC_*` and provider variables in `env` (as +/// [`project_settings_are_unsafe`] finds them in a repository), and +/// `permissions.disableBypassPermissionsMode`. Everything else the image +/// ships stays: plugins and marketplaces, hooks, permission rules, other +/// variables. A sandbox that resumes merges into what it has. +pub fn merge_user_settings(image: &Value, ours: &Value) -> Value { + let mut merged = match image { + Value::Object(o) => o.clone(), + _ => serde_json::Map::new(), + }; + for key in ["apiKeyHelper", "attribution", "disableClaudeAiConnectors"] { + merged.remove(key); + } + let mut env = take_object(&mut merged, "env"); + env.retain(|k, _| !k.starts_with("ANTHROPIC_") && !FORBIDDEN_ENV.contains(&k.as_str())); + let mut permissions = take_object(&mut merged, "permissions"); + permissions.remove("disableBypassPermissionsMode"); + for (key, value) in ours.as_object().into_iter().flatten() { + match (key.as_str(), value) { + ("env", Value::Object(o)) => env.extend(o.clone()), + ("permissions", Value::Object(o)) => permissions.extend(o.clone()), + _ => { + merged.insert(key.clone(), value.clone()); + } + } + } + merged.insert("env".into(), Value::Object(env)); + merged.insert("permissions".into(), Value::Object(permissions)); + Value::Object(merged) +} + +/// Removes `key` from `map`: its object, or an empty one. +fn take_object( + map: &mut serde_json::Map, + key: &str, +) -> serde_json::Map { + match map.remove(key) { + Some(Value::Object(o)) => o, + _ => serde_json::Map::new(), + } +} + /// Claude's trailer in commits that credit people too. Claude Code's own /// names the model, which isn't known before Claude starts; Claude Code /// writes this one itself when it can't tell the model's version. @@ -470,6 +586,65 @@ mod tests { assert!(s.get("apiKeyHelper").is_none()); } + #[test] + fn the_images_settings_stay_under_fctls() { + let image = json!({ + "enabledPlugins": { "rust@acme": true }, + "extraKnownMarketplaces": { "acme": { "source": { "source": "github", "repo": "acme/plugins" } } }, + "hooks": { "PostToolUse": [] }, + "permissions": { + "allow": ["Bash(cargo:*)"], + "defaultMode": "plan", + "disableBypassPermissionsMode": "disable" + }, + "env": { + "RUST_LOG": "info", + "ANTHROPIC_API_KEY": "sk-ant-x", + "ANTHROPIC_BASE_URL": "https://proxy", + "CLAUDE_CODE_USE_BEDROCK": "1", + "DISABLE_AUTOUPDATER": "0" + }, + "apiKeyHelper": "echo sk-ant-x", + "attribution": { "commit": "Made by acme" }, + "disableClaudeAiConnectors": true, + "model": "sonnet" + }); + let s = merge_user_settings(&image, &user_settings(&project(true), &[])); + assert_eq!(s["enabledPlugins"], image["enabledPlugins"]); + assert_eq!(s["extraKnownMarketplaces"], image["extraKnownMarketplaces"]); + assert_eq!(s["hooks"], image["hooks"]); + assert_eq!(s["model"], "sonnet"); + assert_eq!(s["permissions"]["allow"], json!(["Bash(cargo:*)"])); + assert_eq!(s["permissions"]["defaultMode"], "bypassPermissions"); + assert!( + s["permissions"] + .get("disableBypassPermissionsMode") + .is_none() + ); + assert_eq!( + s["env"], + json!({ "RUST_LOG": "info", "DISABLE_AUTOUPDATER": "1" }) + ); + assert!(s.get("apiKeyHelper").is_none()); + // The project's settings decide credits and connectors. + assert_eq!(s["attribution"], json!({ "pr": "" })); + assert!(s.get("disableClaudeAiConnectors").is_none()); + // A resume merges into what the last run wrote: the same again. + let again = merge_user_settings(&s, &user_settings(&project(true), &[])); + assert_eq!(again, s); + // A turned-off setting comes back on once the project says so. + let off = merge_user_settings(&s, &user_settings(&project(false), &[])); + assert_eq!(off["attribution"]["sessionUrl"], false); + let on = merge_user_settings(&off, &user_settings(&project(true), &[])); + assert!(on["attribution"].get("sessionUrl").is_none()); + // No settings, or ones that aren't an object: fctl's alone. + for nothing in [Value::Null, json!([1])] { + let ours = user_settings(&project(true), &[]); + let s = merge_user_settings(¬hing, &ours); + assert_eq!(s, ours); + } + } + #[test] fn user_settings_turn_connectors_off() { assert!( @@ -605,6 +780,54 @@ mod tests { Ok(()) } + #[test] + fn reads_claude_codes_version() { + assert_eq!( + Version::parse("2.1.302 (Claude Code)\n"), + Some(Version(2, 1, 302)) + ); + assert_eq!( + Version::parse("2.1.999 (fake-claude)"), + Some(Version(2, 1, 999)) + ); + assert_eq!( + Version::parse("claude v2.1.101-rc.1"), + Some(Version(2, 1, 101)) + ); + assert_eq!(Version::parse("Claude Code"), None); + assert!(Version(2, 1, 100) < MIN_VERSION && MIN_VERSION < Version(2, 2, 0)); + assert_eq!(MIN_VERSION.to_string(), "2.1.101"); + } + + #[tokio::test] + async fn sessions_need_a_recent_claude_code() -> anyhow::Result<()> { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir()?; + let claude = |version: &str| -> anyhow::Result { + let bin = dir.path().join(format!("claude-{version}")); + std::fs::write(&bin, format!("#!/bin/sh\necho '{version} (Claude Code)'\n"))?; + std::fs::set_permissions(&bin, std::fs::Permissions::from_mode(0o755))?; + Ok(bin.display().to_string()) + }; + assert_eq!( + check_version(&claude("2.1.302")?).await, + Ok(Some(Version(2, 1, 302))) + ); + assert_eq!( + check_version(&claude("2.1.90")?).await, + Err("This sandbox has Claude Code 2.1.90, and fctl needs 2.1.101 or later: update it in the image (ghcr.io/hyperstructure/sandbox has the newest)".into()) + ); + let missing = check_version("fctl-test-no-claude").await; + assert!( + missing + .as_ref() + .is_err_and(|e| e.starts_with("This sandbox has no Claude Code")), + "{missing:?}" + ); + assert_eq!(check_version(&claude("unknown")?).await, Ok(None)); + Ok(()) + } + #[test] fn truncates_on_char_boundaries() { assert_eq!(truncate("héllo", 2), "h…"); diff --git a/crates/fctl-agent/src/supervisor.rs b/crates/fctl-agent/src/supervisor.rs index 8a3522fd..f2037fa3 100644 --- a/crates/fctl-agent/src/supervisor.rs +++ b/crates/fctl-agent/src/supervisor.rs @@ -110,6 +110,8 @@ pub struct Supervisor { pending_files: Vec<(Attachment, String)>, /// Said once that the sandbox's fctl cannot rename the session. told_no_fctl: bool, + /// The sandbox's Claude Code, once checked. + claude_version: Option, /// `fctl publish` calls waiting for the host's answer, by request id. publishing: HashMap, Instant)>, /// Publishes the caller stopped waiting for: if the host publishes one @@ -173,6 +175,7 @@ impl Supervisor { last_git: None, pending_files: Vec::new(), told_no_fctl: false, + claude_version: None, publishing: HashMap::new(), late_publishes: VecDeque::new(), }, @@ -235,7 +238,7 @@ impl Supervisor { configured: self.config.is_some(), status: self.state.status, agent_seq: self.journal.last_seq(), - claude_version: None, + claude_version: self.claude_version.map(|v| v.to_string()), features: vec![FEATURE_ATTACHMENTS.to_owned()], }) } @@ -768,10 +771,24 @@ impl Supervisor { cfg["oauthAccount"] = raw; } std::fs::write(&path, serde_json::to_string_pretty(&cfg)?)?; - std::fs::write( - claude_dir.join("settings.json"), - serde_json::to_string_pretty(&claude::user_settings(project, co_authors))?, - )?; + // The image's settings stay, with fctl's over them. + let settings_path = claude_dir.join("settings.json"); + let image = match std::fs::read_to_string(&settings_path) { + Ok(text) => serde_json::from_str(&text).unwrap_or_else(|e| { + let kept = claude_dir.join("settings.json.invalid"); + tracing::warn!( + "{} is not JSON ({e}); keeping it as {}", + settings_path.display(), + kept.display() + ); + let _ = std::fs::rename(&settings_path, &kept); + Value::Null + }), + Err(_) => Value::Null, + }; + let settings = + claude::merge_user_settings(&image, &claude::user_settings(project, co_authors)); + std::fs::write(&settings_path, serde_json::to_string_pretty(&settings)?)?; Ok(()) } @@ -781,6 +798,14 @@ impl Supervisor { }; let spec = &config.spec; let workdir = config.workdir.clone(); + let bin = config.claude_bin.clone().unwrap_or_else(|| "claude".into()); + if self.claude_version.is_none() { + match claude::check_version(&bin).await { + Ok(Some(version)) => self.claude_version = Some(version), + Ok(None) => tracing::warn!("{bin} --version said no version; starting it anyway"), + Err(why) => anyhow::bail!(why), + } + } self.write_claude_config( &paths.home, &workdir, @@ -855,7 +880,6 @@ impl Supervisor { || claude::conversation_exists(&paths.home, &spec.claude_session_id)); let fresh_resume = wants_resume && !resume; let system = prompt::system_prompt(spec, &workdir, &paths, config.sandbox); - let bin = config.claude_bin.clone().unwrap_or_else(|| "claude".into()); let launch = LaunchSpec { bin: &bin, cwd: &paths.workdir, diff --git a/crates/fctl-client/src/http.rs b/crates/fctl-client/src/http.rs index 406c8deb..32bb6be3 100644 --- a/crates/fctl-client/src/http.rs +++ b/crates/fctl-client/src/http.rs @@ -95,29 +95,127 @@ impl HttpClient { body: Option>, content_type: Option<&str>, ) -> Result<(StatusCode, http::HeaderMap, Bytes)> { - let mut req = Request::builder().method(method).uri(url); - if let Some(h) = req.headers_mut() { - if let Ok(v) = HeaderValue::from_str(&crate::user_agent()) { - h.insert(http::header::USER_AGENT, v); + let req = request(method, url, token, body, content_type, &[])?; + self.finish(url, req).await + } + + /// A GET with `headers` besides the usual ones, which they replace (an + /// `Accept` of their own, say). + pub async fn get_with( + &self, + url: &str, + token: Option<&str>, + headers: &[(&str, &str)], + ) -> Result<(StatusCode, http::HeaderMap, Bytes)> { + let req = request(Method::GET, url, token, None, None, headers)?; + self.finish(url, req).await + } + + /// Downloads `url` into the file at `path`, following up to 5 + /// redirects, within `limit`. The token and `headers` go to `url`'s + /// origin only. Fails once more than `max_bytes` came. Returns the size. + pub async fn download_to( + &self, + url: &str, + token: Option<&str>, + headers: &[(&str, &str)], + path: &std::path::Path, + max_bytes: u64, + limit: Duration, + ) -> Result { + tokio::time::timeout( + limit, + self.download_to_inner(url, token, headers, path, max_bytes), + ) + .await + .map_err(|_| Error::Timeout(limit))? + } + + async fn download_to_inner( + &self, + url: &str, + token: Option<&str>, + headers: &[(&str, &str)], + path: &std::path::Path, + max_bytes: u64, + ) -> Result { + use tokio::io::AsyncWriteExt; + let origin = |u: &str| url::Url::parse(u).ok().map(|u| u.origin()); + let first = origin(url); + let mut current = url.to_owned(); + for _ in 0..6 { + let same_origin = first.is_some() && origin(¤t) == first; + let (token, headers) = if same_origin { + (token, headers) + } else { + (None, &[][..]) + }; + let req = request(Method::GET, ¤t, token, None, None, headers)?; + // Signed download links carry a token in their query. + let shown = without_query(¤t); + let network = |message: String| Error::Network { + url: shown.clone(), + message, + }; + let res = self + .client + .request(req) + .await + .map_err(|e| network(error_chain(&e)))?; + let status = res.status(); + if status.is_redirection() { + let Some(loc) = res + .headers() + .get(http::header::LOCATION) + .and_then(|v| v.to_str().ok()) + else { + return Err(Error::Decode("redirect without a location".into())); + }; + current = url::Url::parse(¤t) + .and_then(|base| base.join(loc)) + .map(|u| u.to_string()) + .map_err(|e| Error::Decode(e.to_string()))?; + continue; } - h.insert( - http::header::ACCEPT, - HeaderValue::from_static("application/json"), - ); - if let Some(t) = token - && let Ok(v) = HeaderValue::from_str(&format!("Bearer {t}")) - { - h.insert(http::header::AUTHORIZATION, v); + let mut body = res.into_body(); + if !status.is_success() { + let bytes = body + .collect() + .await + .map(|b| b.to_bytes()) + .unwrap_or_default(); + return Err(Error::Status { + status: status.as_u16(), + body: String::from_utf8_lossy(&bytes).chars().take(500).collect(), + }); } - if let Some(ct) = content_type - && let Ok(v) = HeaderValue::from_str(ct) - { - h.insert(http::header::CONTENT_TYPE, v); + let io = |e: std::io::Error| Error::Other(format!("writing {}: {e}", path.display())); + let mut file = tokio::fs::File::create(path).await.map_err(io)?; + let mut size: u64 = 0; + while let Some(frame) = body.frame().await { + let frame = frame.map_err(|e| network(e.to_string()))?; + if let Ok(data) = frame.into_data() { + size += data.len() as u64; + if size > max_bytes { + return Err(Error::Other(format!( + "{shown} is larger than {} MiB", + max_bytes / 1024 / 1024 + ))); + } + file.write_all(&data).await.map_err(io)?; + } } + file.flush().await.map_err(io)?; + return Ok(size); } - let req = req - .body(Full::new(Bytes::from(body.unwrap_or_default()))) - .map_err(|e| Error::Other(e.to_string()))?; + Err(Error::Other("too many redirects".into())) + } + + async fn finish( + &self, + url: &str, + req: Request>, + ) -> Result<(StatusCode, http::HeaderMap, Bytes)> { let fut = async { let res = self.client.request(req).await.map_err(|e| Error::Network { url: url.to_owned(), @@ -195,6 +293,53 @@ impl HttpClient { } } +/// `url` without its query or fragment, for messages. +fn without_query(url: &str) -> String { + url.split(['?', '#']).next().unwrap_or_default().to_owned() +} + +/// A request with fctl's user agent, JSON accepted, the token, and +/// `headers`, which replace those. +fn request( + method: Method, + url: &str, + token: Option<&str>, + body: Option>, + content_type: Option<&str>, + headers: &[(&str, &str)], +) -> Result>> { + let mut req = Request::builder().method(method).uri(url); + if let Some(h) = req.headers_mut() { + if let Ok(v) = HeaderValue::from_str(&crate::user_agent()) { + h.insert(http::header::USER_AGENT, v); + } + h.insert( + http::header::ACCEPT, + HeaderValue::from_static("application/json"), + ); + if let Some(t) = token + && let Ok(v) = HeaderValue::from_str(&format!("Bearer {t}")) + { + h.insert(http::header::AUTHORIZATION, v); + } + if let Some(ct) = content_type + && let Ok(v) = HeaderValue::from_str(ct) + { + h.insert(http::header::CONTENT_TYPE, v); + } + for (name, value) in headers { + if let (Ok(name), Ok(value)) = ( + http::header::HeaderName::from_bytes(name.as_bytes()), + HeaderValue::from_str(value), + ) { + h.insert(name, value); + } + } + } + req.body(Full::new(Bytes::from(body.unwrap_or_default()))) + .map_err(|e| Error::Other(e.to_string())) +} + fn error_chain(e: &dyn std::error::Error) -> String { let mut s = e.to_string(); let mut cur = e.source(); diff --git a/crates/fctl-control/src/api/projects.rs b/crates/fctl-control/src/api/projects.rs index dd357b9c..0c129c4e 100644 --- a/crates/fctl-control/src/api/projects.rs +++ b/crates/fctl-control/src/api/projects.rs @@ -47,6 +47,16 @@ fn clean_settings(mut s: ProjectSettings) -> AppResult { Some(validate::image(img)?) }; } + if let Some(b) = &s.image_build { + s.image_build = Some(validate::image_build(b)?); + } + if s.image.is_some() && s.image_build.is_some() { + // On the image's field, which forms from before builds show too. + return Err(validation( + "image", + "Choose an image or a build from the repository, not both", + )); + } if let Some(m) = s.model.as_deref() { s.model = if m.trim().is_empty() { None @@ -113,6 +123,14 @@ fn clean_settings(mut s: ProjectSettings) -> AppResult { Ok(s) } +/// A build comes from the project's repository. +fn needs_repo() -> AppError { + validation( + "settings.image_build", + "Link a repository first: the image is built from its files", + ) +} + async fn slug_taken(ctx: &Ctx, slug: &str) -> AppResult { let c: Option = q( &ctx.db, @@ -207,6 +225,9 @@ pub async fn create(ctx: &mut Ctx, req: &mut Request) -> AppResult { repo = Some(parsed); } } + if settings.image_build.is_some() && repo.is_none() { + return Err(needs_repo()); + } let id = fctl_domain::ids::project_id(ctx.now); batch( &ctx.db, @@ -299,6 +320,9 @@ pub async fn update(ctx: &mut Ctx, req: &mut Request, id: &str) -> AppResult row.settings(), }; + if settings.image_build.is_some() && (owner.is_none() || repo_name.is_none()) { + return Err(needs_repo()); + } let archived_at = match body.archived { Some(true) => row.archived_at.or(Some(ctx.now)), Some(false) => None, diff --git a/crates/fctl-control/src/do_fleet.rs b/crates/fctl-control/src/do_fleet.rs index 3fe8afb8..17759ad8 100644 --- a/crates/fctl-control/src/do_fleet.rs +++ b/crates/fctl-control/src/do_fleet.rs @@ -12,6 +12,7 @@ use crate::internal::{ LiveRunners, LoginCode, LoginStart, NotifySession, Schedule, Stop, }; use crate::spec; +use fctl_domain::image::{HostImages, ImageWant}; use fctl_domain::schedule::{Home, Slot, Wait}; use fctl_domain::size::Want; use fctl_proto::api::{ClaudeLoginJob, ClaudeLoginState, FeedServer}; @@ -125,6 +126,9 @@ struct Candidate { labels: Vec, sandbox: Option, draining: bool, + image: Option, + allowed_images: Option>, + allowed_builds: Option>, } impl Candidate { @@ -138,6 +142,11 @@ impl Candidate { labels: &self.labels, sandbox: self.sandbox, draining: self.draining, + images: HostImages { + own: self.image.as_deref(), + allowed: self.allowed_images.as_deref(), + builds: self.allowed_builds.as_deref(), + }, } } } @@ -1100,7 +1109,7 @@ impl FleetObject { .await; continue; } - let want = wants.get(&item.session_id).copied().unwrap_or_default(); + let want = wants.get(&item.session_id).cloned().unwrap_or_default(); // A resume goes back to the runner that has its sandbox, where // Claude continues the same conversation, and waits a while for // it when it is busy. Not after it declined the session. @@ -1110,7 +1119,8 @@ impl FleetObject { self.home_of(&item.session_id) }; let wait = now - (item.enqueued_at as i64) < HOME_WAIT_MS; - let (runner_id, runner_name) = match self.pick_runner(want, home.as_deref(), wait) { + let picked = self.pick_runner(want.size, &want.image, home.as_deref(), wait); + let (runner_id, runner_name) = match picked { Ok(r) => r, Err(wait) => { self.note_waiting(&item, wait.reason()).await; @@ -1245,6 +1255,9 @@ impl FleetObject { labels: hello.labels, sandbox: hello.sandbox, draining: r.draining_since.is_some(), + image: hello.image, + allowed_images: hello.allowed_images, + allowed_builds: hello.allowed_builds, }) }) .collect() @@ -1303,13 +1316,14 @@ impl FleetObject { .any(|reports| reports.iter().any(|r| r.session_id.as_str() == session_id)) } - /// The least loaded connected runner with a free slot that gives a - /// sandbox what the session wants, or why there is none. `home`, the - /// runner that has the session's sandbox, comes first, and with `wait` - /// the session waits for it while it is busy. + /// The least loaded connected runner with a free slot that runs the + /// session's image and gives its sandbox what it wants, or why there is + /// none. `home`, the runner that has the session's sandbox, comes first, + /// and with `wait` the session waits for it while it is busy. fn pick_runner( &self, want: Want, + image: &ImageWant, home: Option<&str>, wait: bool, ) -> std::result::Result<(String, String), Wait> { @@ -1322,7 +1336,7 @@ impl FleetObject { name: &c.name, wait, }); - let chosen = fctl_domain::schedule::pick(&slots, &[], want, home)?; + let chosen = fctl_domain::schedule::pick(&slots, &[], want, image, home)?; let name = candidates .iter() .find(|c| c.id == chosen.id) diff --git a/crates/fctl-control/src/model.rs b/crates/fctl-control/src/model.rs index 95f4d21e..a316fd8f 100644 --- a/crates/fctl-control/src/model.rs +++ b/crates/fctl-control/src/model.rs @@ -488,6 +488,9 @@ impl RunnerRow { memory_available_mb: free.memory_mb, disk_available_mb: free.disk_mb, sandbox: hello.as_ref().and_then(|h| h.sandbox), + image: hello.as_ref().and_then(|h| h.image.clone()), + allowed_images: hello.as_ref().and_then(|h| h.allowed_images.clone()), + allowed_builds: hello.as_ref().and_then(|h| h.allowed_builds.clone()), labels: hello.as_ref().map(|h| h.labels.clone()).unwrap_or_default(), last_seen_at: self.last_seen_at, drain: live.drains.get(&self.id).cloned(), diff --git a/crates/fctl-control/src/spec.rs b/crates/fctl-control/src/spec.rs index bb53c34b..c55fe432 100644 --- a/crates/fctl-control/src/spec.rs +++ b/crates/fctl-control/src/spec.rs @@ -9,6 +9,7 @@ use crate::model::{ CredentialRow, CredentialSecret, PROJECT_SELECT, ProjectRow, REPO_SELECT, RepoRow, SESSION_SELECT, SessionRow, }; +use fctl_domain::image::ImageWant; use fctl_domain::size::Want; use fctl_proto::runner::{ ClaudeCredential, ClaudeCredentialKind, GitIdentity, ProjectSpec, SessionCredentials, @@ -57,17 +58,32 @@ pub async fn grants(db: &D1Database, project_id: &str) -> AppResult AppResult> { +/// What a queued session asks of a runner. +#[derive(Clone, Debug, Default)] +pub struct SessionWants { + /// CPUs and memory. + pub size: Want, + /// The image it starts from. + pub image: ImageWant, +} + +/// The CPUs, memory and image each of these sessions asks for (its +/// project's sandbox settings), in one query. +pub async fn session_wants( + db: &D1Database, + ids: &[&str], +) -> AppResult> { #[derive(serde::Deserialize)] struct Row { id: String, settings: String, + repo_owner: Option, + repo_name: Option, } let rows: Vec = q( db, - "SELECT s.id AS id, p.settings AS settings FROM sessions s JOIN projects p ON p.id = s.project_id \ + "SELECT s.id AS id, p.settings AS settings, p.repo_owner AS repo_owner, p.repo_name AS repo_name \ + FROM sessions s JOIN projects p ON p.id = s.project_id \ WHERE s.id IN (SELECT value FROM json_each(?))", &[s(&serde_json::to_string(ids)?)], )? @@ -77,7 +93,15 @@ pub async fn session_wants(db: &D1Database, ids: &[&str]) -> AppResult Some(RepoRef { owner, name }), + _ => None, + }; + let wants = SessionWants { + size: Want::of(&settings), + image: ImageWant::of(&settings, repo.as_ref()), + }; + (r.id, wants) }) .collect()) } diff --git a/crates/fctl-domain/src/image.rs b/crates/fctl-domain/src/image.rs new file mode 100644 index 00000000..9641b967 --- /dev/null +++ b/crates/fctl-domain/src/image.rs @@ -0,0 +1,676 @@ +//! Sandbox images: references as registries read them, the images and +//! builds a host allows, and what a session's project asks for. + +use fctl_proto::session::{ProjectSettings, RepoRef}; + +/// The name the default image had before it was published, which hosts and +/// projects from then may still have. +pub const OLD_DEFAULT_IMAGE: &str = "fctl-agent:latest"; + +/// Whether an image setting, a host's or a project's, means the host's +/// default: empty, or the name the default had before it was published. +pub fn means_default(image: &str) -> bool { + let image = image.trim(); + image.is_empty() || image == OLD_DEFAULT_IMAGE +} + +/// An image reference as Docker reads it: `ubuntu` is +/// `docker.io/library/ubuntu`, at the tag `latest`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ImageRef { + /// The registry, lowercase, and the repository in it: + /// `ghcr.io/acme/agent`. + pub name: String, + /// Absent when the reference names none: `latest`, unless it names a + /// digest. + pub tag: Option, + /// `sha256:…`. + pub digest: Option, +} + +impl ImageRef { + /// Parses a reference by Docker's grammar, refusing what Docker refuses + /// (a repository with uppercase letters, a digest of the wrong length). + pub fn parse(reference: &str) -> Option { + let s = reference.trim(); + if s.is_empty() || s.len() > 255 || s.chars().any(|c| c.is_whitespace() || c.is_control()) { + return None; + } + let (rest, digest) = match s.split_once('@') { + Some((rest, digest)) => (rest, Some(digest)), + None => (s, None), + }; + if let Some(d) = digest + && !valid_digest(d) + { + return None; + } + // A tag follows the last colon after the last slash; one before it + // separates a registry's port. + let last = rest.rfind('/').map_or(0, |i| i + 1); + let (path, tag) = match rest[last..].rfind(':') { + Some(i) => (&rest[..last + i], Some(&rest[last + i + 1..])), + None => (rest, None), + }; + if let Some(t) = tag + && !valid_tag(t) + { + return None; + } + let name = full_name(path)?; + Some(Self { + name, + tag: tag.map(str::to_owned), + digest: digest.map(str::to_owned), + }) + } + + /// The tag a pull without a digest gets. + fn effective_tag(&self) -> Option<&str> { + match (&self.tag, &self.digest) { + (Some(t), _) => Some(t), + (None, None) => Some("latest"), + (None, Some(_)) => None, + } + } + + /// Whether both are the same image: the same repository, tag and digest. + pub fn same(&self, other: &Self) -> bool { + self.name == other.name + && self.effective_tag() == other.effective_tag() + && self.digest == other.digest + } +} + +/// An algorithm and its hex digits, in lowercase: 64 for `sha256`, 96 for +/// `sha384`, 128 for `sha512`, at least 32 for others. +fn valid_digest(d: &str) -> bool { + let Some((algorithm, hex)) = d.split_once(':') else { + return false; + }; + let algorithm_ok = algorithm.split(['+', '.', '_', '-']).all(|part| { + part.starts_with(|c: char| c.is_ascii_alphabetic()) + && part.bytes().all(|b| b.is_ascii_alphanumeric()) + }); + let length_ok = match algorithm { + "sha256" => hex.len() == 64, + "sha384" => hex.len() == 96, + "sha512" => hex.len() == 128, + _ => hex.len() >= 32, + }; + algorithm_ok + && length_ok + && hex + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} + +fn valid_tag(t: &str) -> bool { + !t.is_empty() + && t.len() <= 128 + && !t.starts_with(['.', '-']) + && t.bytes() + .all(|b| b.is_ascii_alphanumeric() || b"_.-".contains(&b)) +} + +/// Whether Docker reads the first part of a name as a registry: it has a dot +/// or a colon, is `localhost`, or has an uppercase letter. +fn is_registry(first: &str) -> bool { + first.contains(['.', ':']) + || first == "localhost" + || first.chars().any(|c| c.is_ascii_uppercase()) +} + +/// A registry's host and port, lowercase; `index.docker.io` is Docker Hub's +/// `docker.io`. `None` when it isn't a host name or an IP address. +fn registry(part: &str) -> Option { + let (host, port) = match part.rsplit_once(':') { + Some((host, port)) if !host.ends_with(':') => (host, Some(port)), + _ => (part, None), + }; + if port.is_some_and(|p| p.is_empty() || !p.bytes().all(|b| b.is_ascii_digit())) { + return None; + } + let host_ok = match host.strip_prefix('[').and_then(|h| h.strip_suffix(']')) { + Some(v6) => !v6.is_empty() && v6.bytes().all(|b| b.is_ascii_hexdigit() || b == b':'), + None => host.split('.').all(|label| { + !label.is_empty() + && label + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-') + && !label.starts_with('-') + && !label.ends_with('-') + }), + }; + if !host_ok { + return None; + } + let full = match port { + Some(p) => format!("{}:{p}", host.to_ascii_lowercase()), + None => host.to_ascii_lowercase(), + }; + Some(if full == "index.docker.io" { + "docker.io".to_owned() + } else { + full + }) +} + +/// A part of a repository's path as Docker allows it: lowercase letters and +/// digits, separated by one `.`, one or two `_`, or `-`s. +fn valid_component(c: &str) -> bool { + let b = c.as_bytes(); + let alnum = |x: u8| x.is_ascii_lowercase() || x.is_ascii_digit(); + let (Some(&first), Some(&last)) = (b.first(), b.last()) else { + return false; + }; + if !alnum(first) || !alnum(last) { + return false; + } + let mut i = 0; + while i < b.len() { + if alnum(b[i]) { + i += 1; + continue; + } + match b[i] { + b'.' => i += 1, + b'_' => i += if b.get(i + 1) == Some(&b'_') { 2 } else { 1 }, + b'-' => { + while b.get(i) == Some(&b'-') { + i += 1; + } + } + _ => return false, + } + if !b.get(i).is_some_and(|&x| alnum(x)) { + return false; + } + } + true +} + +/// `registry/path` for a repository as written: Docker Hub's when the first +/// part names no registry, and its official images under `library/`. +fn full_name(path: &str) -> Option { + let (registry, repo) = match path.split_once('/') { + Some((first, rest)) if is_registry(first) => (registry(first)?, rest), + _ => ("docker.io".to_owned(), path), + }; + if !repo.split('/').all(valid_component) { + return None; + } + let repo = if registry == "docker.io" && !repo.contains('/') { + format!("library/{repo}") + } else { + repo.to_owned() + }; + Some(format!("{registry}/{repo}")) +} + +/// Why an entry of `allowed_images` matches nothing, or `None` when it is +/// one: `*`, a prefix that ends in `/*` (`ghcr.io/acme/*`), or a reference, +/// with or without a tag, pinned to a digest or not. +pub fn check_image_pattern(pattern: &str) -> Option { + let p = pattern.trim(); + if p == "*" { + return None; + } + if let Some(prefix) = p.strip_suffix("/*") { + return full_prefix(prefix) + .is_none() + .then(|| format!("`{p}` is not a registry or a path in one")); + } + if p.ends_with('*') { + return Some(format!("`{p}`: a prefix ends in /*, like ghcr.io/acme/*")); + } + ImageRef::parse(p) + .is_none() + .then(|| format!("`{p}` is not an image reference")) +} + +/// What a prefix entry covers: `ghcr.io/acme` from `ghcr.io/acme`, Docker +/// Hub's `docker.io/acme` from `acme`, `docker.io/library` from +/// `docker.io/library`, and a registry alone (`ghcr.io`). +fn full_prefix(prefix: &str) -> Option { + let prefix = prefix.trim().trim_end_matches('/'); + if prefix.is_empty() { + return None; + } + let (first, rest) = match prefix.split_once('/') { + Some((first, rest)) => (first, Some(rest)), + None => (prefix, None), + }; + let (registry, path) = if is_registry(first) { + (registry(first)?, rest) + } else { + ("docker.io".to_owned(), Some(prefix)) + }; + match path { + None => Some(registry), + Some(path) => path + .split('/') + .all(valid_component) + .then(|| format!("{registry}/{path}")), + } +} + +/// Whether `pattern`, an entry of a host's `allowed_images`, allows `image`. +/// +/// - `*`: any image. +/// - `ghcr.io/acme/*`: any image in that registry path. +/// - `ghcr.io/acme/agent`: any tag or digest of that repository. +/// - `ghcr.io/acme/agent:1`: that tag, not pinned to a digest: a pull by +/// digest ignores the tag, so the host can't tell that it is that tag's. +/// - `ghcr.io/acme/agent@sha256:…`: only that digest, which the project's +/// reference must name too: a tag can move. +pub fn pattern_allows(pattern: &str, image: &ImageRef) -> bool { + let p = pattern.trim(); + if p == "*" { + return true; + } + if let Some(prefix) = p.strip_suffix("/*") { + return full_prefix(prefix).is_some_and(|prefix| { + image + .name + .strip_prefix(&prefix) + .is_some_and(|rest| rest.starts_with('/')) + }); + } + let Some(p) = ImageRef::parse(p) else { + return false; + }; + if p.name != image.name { + return false; + } + match (&p.digest, &p.tag) { + (Some(digest), _) => image.digest.as_ref() == Some(digest), + (None, Some(tag)) => image.digest.is_none() && image.effective_tag() == Some(tag.as_str()), + (None, None) => true, + } +} + +/// Why an entry of `allowed_builds` matches no repository, or `None` when it +/// is one: `owner/name`, `owner/*`, or `*`. +pub fn check_build_pattern(pattern: &str) -> Option { + let p = pattern.trim(); + if p == "*" { + return None; + } + crate::repos::parse(p) + .err() + .map(|_| format!("`{p}` is not owner/name or owner/*")) +} + +/// Whether `pattern`, an entry of a host's `allowed_builds`, allows builds +/// from `repo`'s files. GitHub ignores case. +pub fn build_pattern_allows(pattern: &str, repo: &RepoRef) -> bool { + let p = pattern.trim(); + if p == "*" { + return true; + } + match crate::repos::parse(p) { + Ok((owner, None)) => owner.eq_ignore_ascii_case(&repo.owner), + Ok((owner, Some(name))) => { + owner.eq_ignore_ascii_case(&repo.owner) && name.eq_ignore_ascii_case(&repo.name) + } + Err(_) => false, + } +} + +/// What a session's sandbox starts from, as its project's settings say. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub enum ImageWant { + /// The host's own image. + #[default] + Default, + /// An image from a registry. + Image(String), + /// An image built from a file in this repository. + Build(RepoRef), +} + +impl ImageWant { + /// What sessions of a project with these settings, and this primary + /// repository, start from. A build without a repository has nothing to + /// build from, and the control plane refuses one. + pub fn of(settings: &ProjectSettings, repo: Option<&RepoRef>) -> Self { + if settings.image_build.is_some() + && let Some(repo) = repo + { + return Self::Build(repo.clone()); + } + match settings.image.as_deref() { + Some(image) if !means_default(image) => Self::Image(image.trim().to_owned()), + _ => Self::Default, + } + } +} + +/// The images a host runs and the repositories it builds images from, as +/// its hello says. +#[derive(Clone, Copy, Debug, Default)] +pub struct HostImages<'a> { + /// The image it gives sessions whose project sets none. It runs that + /// one whatever `allowed` says. + pub own: Option<&'a str>, + /// `allowed_images`. `None`, as from hosts from before it: any image. + pub allowed: Option<&'a [String]>, + /// `allowed_builds`, sent only by hosts that can build. `None`, as from + /// hosts from before it: none. + pub builds: Option<&'a [String]>, +} + +impl HostImages<'_> { + /// Whether the host runs sessions whose project chose `image`. + pub fn runs(&self, image: &str) -> bool { + if means_default(image) { + return true; + } + let Some(allowed) = self.allowed else { + return true; + }; + let Some(image) = ImageRef::parse(image) else { + return false; + }; + let own = self + .own + .and_then(ImageRef::parse) + .is_some_and(|own| own.same(&image)); + own || allowed.iter().any(|p| pattern_allows(p, &image)) + } + + /// Whether the host builds images from `repo`'s files. + pub fn builds_from(&self, repo: &RepoRef) -> bool { + self.builds + .is_some_and(|b| b.iter().any(|p| build_pattern_allows(p, repo))) + } + + /// Whether the host can give a session what it wants to start from. + pub fn takes(&self, want: &ImageWant) -> bool { + match want { + ImageWant::Default => true, + ImageWant::Image(image) => self.runs(image), + ImageWant::Build(repo) => self.builds_from(repo), + } + } +} + +/// Which images a host runs for projects that choose their own, in words: +/// `runs any image a project chooses`, `runs only its own image`, `runs its +/// own image and ghcr.io/acme/*`. +pub fn describe_allowed(allowed: Option<&[String]>) -> String { + match allowed { + None => "runs any image a project chooses".into(), + Some(list) if list.iter().any(|p| p.trim() == "*") => { + "runs any image a project chooses".into() + } + Some([]) => "runs only its own image".into(), + Some(list) => format!("runs its own image and {}", list.join(", ")), + } +} + +/// Which repositories a host builds images from, in words: `builds no +/// images`, `builds images from acme/*`, `builds images from any +/// repository`. +pub fn describe_builds(builds: Option<&[String]>) -> String { + match builds { + None | Some([]) => "builds no images".into(), + Some(list) if list.iter().any(|p| p.trim() == "*") => { + "builds images from any repository".into() + } + Some(list) => format!("builds images from {}", list.join(", ")), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use fctl_proto::session::ImageBuild; + + fn r(s: &str) -> ImageRef { + ImageRef::parse(s).unwrap_or_else(|| ImageRef { + name: format!("unparsable {s}"), + tag: None, + digest: None, + }) + } + + const DIGEST: &str = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + + #[test] + fn reads_references_as_docker_does() { + assert_eq!( + ImageRef::parse("ubuntu"), + Some(ImageRef { + name: "docker.io/library/ubuntu".into(), + tag: None, + digest: None + }) + ); + assert_eq!(r("acme/tools:2").name, "docker.io/acme/tools"); + assert_eq!(r("acme/tools:2").tag.as_deref(), Some("2")); + assert_eq!( + r("index.docker.io/library/ubuntu").name, + "docker.io/library/ubuntu" + ); + let pinned = r(&format!("GHCR.io/acme/agent:1@{DIGEST}")); + assert_eq!(pinned.name, "ghcr.io/acme/agent"); + assert_eq!(pinned.tag.as_deref(), Some("1")); + assert_eq!(pinned.digest.as_deref(), Some(DIGEST)); + // A port is not a tag. + let local = r("localhost:5000/agent"); + assert_eq!(local.name, "localhost:5000/agent"); + assert_eq!(local.tag, None); + assert_eq!(r("registry.acme.dev:5000/x/y:3").tag.as_deref(), Some("3")); + assert_eq!(r("[::1]:5000/agent").name, "[::1]:5000/agent"); + // Uppercase makes the first part a registry, as Docker reads it. + assert_eq!(r("Acme/agent").name, "acme/agent"); + // Separators as Docker allows them. + assert_eq!( + r("acme/my__tool---x.y").name, + "docker.io/acme/my__tool---x.y" + ); + for bad in [ + "", + "two words", + "acme/agent:", + "acme/agent@sha256:short", + "acme//agent", + "ghcr.io/acme/-agent", + "acme/agent:-1", + "ghcr.io/Acme/agent", + "acme/a..b", + "acme/a._b", + "acme/a___b", + "-x.com/agent", + "ghcr.io:port/agent", + "acme/agent@sha256:0123456789abcdef0123456789abcdef", + "acme/agent@sha256:0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF", + ] { + assert_eq!(ImageRef::parse(bad), None, "{bad:?}"); + } + } + + #[test] + fn patterns_allow_registries_repositories_tags_and_digests() { + let agent = r("ghcr.io/acme/agent:1"); + assert!(pattern_allows("*", &agent)); + assert!(pattern_allows("ghcr.io/acme/*", &agent)); + assert!(pattern_allows("ghcr.io/*", &agent)); + assert!(pattern_allows("GHCR.IO/acme/*", &agent)); + assert!(!pattern_allows("ghcr.io/acm/*", &agent)); + assert!(!pattern_allows("ghcr.io/acme/agent/*", &agent)); + assert!(pattern_allows("ghcr.io/acme/agent", &agent)); + assert!(pattern_allows("ghcr.io/acme/agent:1", &agent)); + assert!(!pattern_allows("ghcr.io/acme/agent:2", &agent)); + // A tag allows itself pinned to any digest, a digest only itself. + let pinned = r(&format!("ghcr.io/acme/agent:1@{DIGEST}")); + assert!(!pattern_allows("ghcr.io/acme/agent:1", &pinned)); + assert!(pattern_allows( + &format!("ghcr.io/acme/agent:1@{DIGEST}"), + &pinned + )); + assert!(pattern_allows( + &format!("ghcr.io/acme/agent@{DIGEST}"), + &pinned + )); + assert!(!pattern_allows( + &format!("ghcr.io/acme/agent@{DIGEST}"), + &agent + )); + let by_digest = r(&format!("ghcr.io/acme/agent@{DIGEST}")); + assert!(!pattern_allows("ghcr.io/acme/agent:latest", &by_digest)); + assert!(pattern_allows("ghcr.io/acme/agent", &by_digest)); + // No tag is `latest`. + assert!(pattern_allows( + "ghcr.io/acme/agent:latest", + &r("ghcr.io/acme/agent") + )); + // Docker Hub names. + assert!(pattern_allows( + "ubuntu", + &r("docker.io/library/ubuntu:24.04") + )); + assert!(pattern_allows("acme/*", &r("acme/tools"))); + assert!(pattern_allows("docker.io/*", &r("ubuntu"))); + assert!(pattern_allows("index.docker.io/*", &r("ubuntu"))); + assert!(pattern_allows("docker.io/library/*", &r("ubuntu:24.04"))); + assert!(pattern_allows("docker.io/acme/*", &r("acme/tools"))); + assert!(!pattern_allows("docker.io/acme/*", &r("ubuntu"))); + // `Acme/agent` is the registry Acme's, not Docker Hub's acme. + assert!(!pattern_allows("acme/*", &r("Acme/agent"))); + assert!(!pattern_allows("acme/*", &r("ghcr.io/acme/tools"))); + // Not patterns. + assert!(!pattern_allows("ghcr.io/acme*", &agent)); + assert!(!pattern_allows("", &agent)); + } + + #[test] + fn checks_patterns() { + for good in [ + "*", + "ghcr.io/*", + "ghcr.io/acme/*", + "acme/*", + "docker.io/library/*", + "ubuntu", + "ghcr.io/a/b:1", + ] { + assert_eq!(check_image_pattern(good), None, "{good}"); + } + assert!(check_image_pattern("ghcr.io/acme*").is_some_and(|e| e.contains("/*"))); + assert!(check_image_pattern("two words").is_some()); + assert!(check_image_pattern("ghcr.io/Acme/*").is_some()); + for good in ["*", "acme/*", "acme/app", "https://github.com/acme/app"] { + assert_eq!(check_build_pattern(good), None, "{good}"); + } + assert!(check_build_pattern("acme").is_some()); + } + + #[test] + fn builds_are_allowed_by_repository() { + let app = RepoRef { + owner: "Acme".into(), + name: "App".into(), + }; + assert!(build_pattern_allows("*", &app)); + assert!(build_pattern_allows("acme/*", &app)); + assert!(build_pattern_allows("acme/app", &app)); + assert!(build_pattern_allows("github.com/acme/app", &app)); + assert!(!build_pattern_allows("acme/web", &app)); + assert!(!build_pattern_allows("other/*", &app)); + } + + #[test] + fn hosts_run_their_own_image_and_the_ones_they_allow() { + let list = vec!["ghcr.io/acme/*".to_owned()]; + let host = HostImages { + own: Some("ghcr.io/hyperstructure/sandbox:0.5.0"), + allowed: Some(&list), + builds: None, + }; + assert!(host.runs("")); + assert!(host.runs("fctl-agent:latest")); + assert!(host.runs("ghcr.io/acme/agent:1")); + assert!(host.runs("ghcr.io/hyperstructure/sandbox:0.5.0")); + assert!(!host.runs("ghcr.io/hyperstructure/sandbox:0.4.0")); + assert!(!host.runs("docker.io/evil/miner")); + assert!(!host.runs("not an image")); + // Without a list, any image: hosts from before `allowed_images`. + let any = HostImages { + own: None, + allowed: None, + builds: None, + }; + assert!(any.runs("docker.io/evil/miner")); + // An empty list: only its own. + let none: Vec = vec![]; + let own_only = HostImages { + allowed: Some(&none), + ..host + }; + assert!(own_only.runs("ghcr.io/hyperstructure/sandbox:0.5.0")); + assert!(!own_only.runs("ghcr.io/acme/agent:1")); + } + + #[test] + fn says_what_hosts_allow() { + let list = |l: &[&str]| l.iter().map(|s| (*s).to_owned()).collect::>(); + assert_eq!(describe_allowed(None), "runs any image a project chooses"); + assert_eq!(describe_allowed(Some(&[])), "runs only its own image"); + assert_eq!( + describe_allowed(Some(&list(&["ghcr.io/acme/*", "ubuntu"]))), + "runs its own image and ghcr.io/acme/*, ubuntu" + ); + assert_eq!( + describe_allowed(Some(&list(&["*"]))), + "runs any image a project chooses" + ); + assert_eq!(describe_builds(None), "builds no images"); + assert_eq!( + describe_builds(Some(&list(&["acme/*"]))), + "builds images from acme/*" + ); + assert_eq!( + describe_builds(Some(&list(&["*"]))), + "builds images from any repository" + ); + } + + #[test] + fn sessions_want_their_projects_image_or_build() { + let repo = RepoRef { + owner: "acme".into(), + name: "app".into(), + }; + let mut s = ProjectSettings::default(); + assert_eq!(ImageWant::of(&s, Some(&repo)), ImageWant::Default); + s.image = Some("fctl-agent:latest".into()); + assert_eq!(ImageWant::of(&s, Some(&repo)), ImageWant::Default); + s.image = Some(" ghcr.io/acme/agent:1 ".into()); + assert_eq!( + ImageWant::of(&s, Some(&repo)), + ImageWant::Image("ghcr.io/acme/agent:1".into()) + ); + s.image = None; + s.image_build = Some(ImageBuild { + file: "Dockerfile".into(), + context: None, + }); + assert_eq!( + ImageWant::of(&s, Some(&repo)), + ImageWant::Build(repo.clone()) + ); + assert_eq!(ImageWant::of(&s, None), ImageWant::Default); + + let builders = vec!["acme/*".to_owned()]; + let host = HostImages { + own: None, + allowed: None, + builds: Some(&builders), + }; + assert!(host.takes(&ImageWant::Build(repo.clone()))); + assert!(!HostImages::default().takes(&ImageWant::Build(repo))); + } +} diff --git a/crates/fctl-domain/src/lib.rs b/crates/fctl-domain/src/lib.rs index 8724ab59..6504eca2 100644 --- a/crates/fctl-domain/src/lib.rs +++ b/crates/fctl-domain/src/lib.rs @@ -8,6 +8,7 @@ pub mod allowlist; pub mod events; pub mod files; pub mod ids; +pub mod image; pub mod journal; pub mod naming; pub mod notify; diff --git a/crates/fctl-domain/src/schedule.rs b/crates/fctl-domain/src/schedule.rs index 59215ce6..63c1a5eb 100644 --- a/crates/fctl-domain/src/schedule.rs +++ b/crates/fctl-domain/src/schedule.rs @@ -1,5 +1,6 @@ //! Choosing a runner for a session. +use crate::image::{HostImages, ImageWant}; use crate::size::{self, Want}; use fctl_proto::runner::{SandboxLimits, SandboxSize}; @@ -17,6 +18,8 @@ pub struct Slot<'a> { /// It drains: no new session starts on it, but the sessions whose /// sandboxes it has still resume there. pub draining: bool, + /// The images it runs and builds. + pub images: HostImages<'a>, } /// The runner that keeps a session's sandbox from an earlier run: there, @@ -52,6 +55,10 @@ pub enum Wait { }, /// Runners give its CPUs and runners give its memory, but none gives both. NoneGivesBoth { cpus: u32, memory_mb: u32 }, + /// No online runner runs the image its project chose. + Image { image: String }, + /// No online runner builds images from its project's repository. + Build { repo: String }, } impl Wait { @@ -83,6 +90,22 @@ impl Wait { size::cpus(*cpus), size::memory((*memory_mb).into()) ), + Self::Image { image } => format!("No runner allows this project's image, {image}"), + Self::Build { repo } => format!("No runner builds this project's image from {repo}"), + } + } + + /// Why no runner takes a session that wants `image`. + fn image(image: &ImageWant) -> Self { + match image { + ImageWant::Build(repo) => Self::Build { + repo: repo.full_name(), + }, + ImageWant::Image(image) => Self::Image { + image: image.clone(), + }, + // Every runner runs its own image. + ImageWant::Default => Self::NoRunner, } } } @@ -98,8 +121,8 @@ fn parts(cpus: Option, memory_mb: Option, memory_suffix: &str) -> Stri } /// The least loaded online runner with a free slot that has every required -/// label and gives a sandbox the size the session wants. Ties go to the -/// runner listed first. +/// label, runs or builds the image the session wants, and gives a sandbox +/// the size it wants. Ties go to the runner listed first. /// /// A session's `home` comes first when it is online and could take it, also /// while it drains: with a free slot it gets the session, and without one @@ -109,6 +132,7 @@ pub fn pick<'a>( slots: &'a [Slot<'a>], required_labels: &[String], want: Want, + image: &ImageWant, home: Option>, ) -> Result<&'a Slot<'a>, Wait> { let online: Vec<&Slot> = slots @@ -119,6 +143,13 @@ pub fn pick<'a>( if online.is_empty() { return Err(Wait::NoRunner); } + let online: Vec<&Slot> = online + .into_iter() + .filter(|s| s.images.takes(image)) + .collect(); + if online.is_empty() { + return Err(Wait::image(image)); + } if let Some(home) = home && let Some(slot) = online .iter() @@ -191,6 +222,7 @@ mod tests { labels, sandbox: None, draining: false, + images: HostImages::default(), } } @@ -228,14 +260,20 @@ mod tests { slot("d", 1, 1, &none), ]; let any = Want::default(); - assert_eq!(pick(&slots, &[], any, None).map(|s| s.id), Ok("b")); - assert_eq!(pick(&slots, &gpu, any, None).map(|s| s.id), Ok("b")); assert_eq!( - pick(&slots[2..3], &[], any, None).map(|s| s.id), + pick(&slots, &[], any, &ImageWant::Default, None).map(|s| s.id), + Ok("b") + ); + assert_eq!( + pick(&slots, &gpu, any, &ImageWant::Default, None).map(|s| s.id), + Ok("b") + ); + assert_eq!( + pick(&slots[2..3], &[], any, &ImageWant::Default, None).map(|s| s.id), Err(Wait::NoRunner) ); assert_eq!( - pick(&slots[3..], &[], any, None).map(|s| s.id), + pick(&slots[3..], &[], any, &ImageWant::Default, None).map(|s| s.id), Err(Wait::Busy) ); } @@ -255,18 +293,21 @@ mod tests { ]; // The idle runner first, while the session fits there. assert_eq!( - pick(&slots, &[], memory(8192), None).map(|s| s.id), + pick(&slots, &[], memory(8192), &ImageWant::Default, None).map(|s| s.id), Ok("docker-vm") ); assert_eq!( - pick(&slots, &[], memory(16_384), None).map(|s| s.id), + pick(&slots, &[], memory(16_384), &ImageWant::Default, None).map(|s| s.id), Ok("studio") ); let many = Want { cpus: Some(12), memory_mb: None, }; - assert_eq!(pick(&slots, &[], many, None).map(|s| s.id), Ok("studio")); + assert_eq!( + pick(&slots, &[], many, &ImageWant::Default, None).map(|s| s.id), + Ok("studio") + ); // The only runner that fits is full: it waits for that one. let full = [ slots[0].clone(), @@ -276,7 +317,7 @@ mod tests { }, ]; assert_eq!( - pick(&full, &[], memory(16_384), None).map(|s| s.id), + pick(&full, &[], memory(16_384), &ImageWant::Default, None).map(|s| s.id), Err(Wait::Busy) ); } @@ -286,7 +327,7 @@ mod tests { let none: Vec = vec![]; let slots = [slot("old", 4, 0, &none)]; assert_eq!( - pick(&slots, &[], memory(262_144), None).map(|s| s.id), + pick(&slots, &[], memory(262_144), &ImageWant::Default, None).map(|s| s.id), Ok("old") ); } @@ -304,7 +345,11 @@ mod tests { ..slot("b", 4, 0, &none) }, ]; - let reason = |want: Want| pick(&slots, &[], want, None).err().map(|w| w.reason()); + let reason = |want: Want| { + pick(&slots, &[], want, &ImageWant::Default, None) + .err() + .map(|w| w.reason()) + }; assert_eq!( reason(memory(131_072)).as_deref(), Some("No runner can give this session 128 GiB of memory (the most is 64 GiB)") @@ -349,7 +394,7 @@ mod tests { let any = Want::default(); // Busier, but it has the sandbox. assert_eq!( - pick(&slots, &[], any, home("studio", true)).map(|s| s.id), + pick(&slots, &[], any, &ImageWant::Default, home("studio", true)).map(|s| s.id), Ok("studio") ); // Full: the session waits for it, or starts elsewhere once it no @@ -362,13 +407,13 @@ mod tests { }, ]; assert_eq!( - pick(&full, &[], any, home("studio", true)).map(|s| s.id), + pick(&full, &[], any, &ImageWant::Default, home("studio", true)).map(|s| s.id), Err(Wait::Home { name: "studio".into() }) ); assert_eq!( - pick(&full, &[], any, home("studio", false)).map(|s| s.id), + pick(&full, &[], any, &ImageWant::Default, home("studio", false)).map(|s| s.id), Ok("idle") ); assert_eq!( @@ -397,17 +442,38 @@ mod tests { ]; // Offline: it doesn't wait. assert_eq!( - pick(&slots, &[], Want::default(), home("asleep", true)).map(|s| s.id), + pick( + &slots, + &[], + Want::default(), + &ImageWant::Default, + home("asleep", true) + ) + .map(|s| s.id), Ok("idle") ); // Too small now (the project asks for more): it doesn't wait either. assert_eq!( - pick(&slots, &[], memory(16_384), home("small", true)).map(|s| s.id), + pick( + &slots, + &[], + memory(16_384), + &ImageWant::Default, + home("small", true) + ) + .map(|s| s.id), Ok("idle") ); // Gone. assert_eq!( - pick(&slots, &[], Want::default(), home("removed", true)).map(|s| s.id), + pick( + &slots, + &[], + Want::default(), + &ImageWant::Default, + home("removed", true) + ) + .map(|s| s.id), Ok("idle") ); } @@ -424,10 +490,13 @@ mod tests { ]; let any = Want::default(); // Emptier, but draining: new sessions go elsewhere. - assert_eq!(pick(&slots, &[], any, None).map(|s| s.id), Ok("laptop")); + assert_eq!( + pick(&slots, &[], any, &ImageWant::Default, None).map(|s| s.id), + Ok("laptop") + ); // A resume goes back to its sandbox there. assert_eq!( - pick(&slots, &[], any, home("studio", true)).map(|s| s.id), + pick(&slots, &[], any, &ImageWant::Default, home("studio", true)).map(|s| s.id), Ok("studio") ); // The other one is full: a new session waits, and says why when @@ -439,10 +508,13 @@ mod tests { ..slots[1].clone() }, ]; - assert_eq!(pick(&full, &[], any, None).map(|s| s.id), Err(Wait::Busy)); + assert_eq!( + pick(&full, &[], any, &ImageWant::Default, None).map(|s| s.id), + Err(Wait::Busy) + ); let draining = [slots[0].clone()]; assert_eq!( - pick(&draining, &[], any, None).map(|s| s.id), + pick(&draining, &[], any, &ImageWant::Default, None).map(|s| s.id), Err(Wait::Draining) ); assert_eq!( @@ -450,4 +522,95 @@ mod tests { "Waiting for a runner: the ones online are draining" ); } + + #[test] + fn sessions_go_to_runners_that_allow_their_image() { + use fctl_proto::session::RepoRef; + let none: Vec = vec![]; + let acme = vec!["ghcr.io/acme/*".to_owned()]; + let only_own: Vec = vec![]; + let builds = vec!["acme/*".to_owned()]; + let slots = [ + Slot { + images: HostImages { + own: Some("ghcr.io/hyperstructure/sandbox:0.5.0"), + allowed: Some(&only_own), + builds: None, + }, + ..slot("strict", 4, 0, &none) + }, + Slot { + images: HostImages { + own: None, + allowed: Some(&acme), + builds: Some(&builds), + }, + ..slot("acme", 4, 3, &none) + }, + ]; + let any = Want::default(); + let image = |i: &str| ImageWant::Image(i.into()); + // The default image runs everywhere: the emptier runner. + assert_eq!( + pick(&slots, &[], any, &ImageWant::Default, None).map(|s| s.id), + Ok("strict") + ); + // A project's image only where its owner allows it. + assert_eq!( + pick(&slots, &[], any, &image("ghcr.io/acme/agent:1"), None).map(|s| s.id), + Ok("acme") + ); + let wait = pick(&slots, &[], any, &image("docker.io/evil/miner"), None); + assert_eq!( + wait.clone().map(|s| s.id), + Err(Wait::Image { + image: "docker.io/evil/miner".into() + }) + ); + assert_eq!( + wait.err().map(|w| w.reason()).as_deref(), + Some("No runner allows this project's image, docker.io/evil/miner") + ); + // A build only where its owner builds from the repository. + let app = ImageWant::Build(RepoRef { + owner: "acme".into(), + name: "app".into(), + }); + assert_eq!(pick(&slots, &[], any, &app, None).map(|s| s.id), Ok("acme")); + let other = ImageWant::Build(RepoRef { + owner: "other".into(), + name: "app".into(), + }); + assert_eq!( + pick(&slots, &[], any, &other, None) + .err() + .map(|w| w.reason()) + .as_deref(), + Some("No runner builds this project's image from other/app") + ); + // A resume doesn't go back to a runner that no longer allows it. + assert_eq!( + pick( + &slots, + &[], + any, + &image("ghcr.io/acme/agent:1"), + home("strict", true) + ) + .map(|s| s.id), + Ok("acme") + ); + // Older hosts send no list and run any image, but build none. + let old = [slot("old", 4, 0, &none)]; + assert_eq!( + pick(&old, &[], any, &image("docker.io/evil/miner"), None).map(|s| s.id), + Ok("old") + ); + assert_eq!( + pick(&old, &[], any, &app, None).map(|s| s.id), + Err(Wait::Build { + repo: "acme/app".into() + }) + ); + } } diff --git a/crates/fctl-domain/src/settings.rs b/crates/fctl-domain/src/settings.rs index 2ac2323b..86cc49aa 100644 --- a/crates/fctl-domain/src/settings.rs +++ b/crates/fctl-domain/src/settings.rs @@ -40,9 +40,14 @@ pub fn session_env(settings: &ProjectSettings) -> BTreeMap { /// opened before a deploy, an older CLI) would reset it to its default /// without having shown it: for `claude_ai_connectors`, `credit_people` and /// `credit_claude`, on; for `publish_screenshots`, off; for -/// `screenshot_days`, 30 days (send `null` to choose the default). `sent` -/// is the update's `settings` object as it arrived. +/// `screenshot_days`, 30 days (send `null` to choose the default). An +/// `image_build` it leaves out stays unless it chooses an image instead +/// (send `null` for none). `sent` is the update's `settings` object as it +/// arrived. pub fn keep_unsent(settings: &mut ProjectSettings, stored: &ProjectSettings, sent: &Value) { + if sent.get("image_build").is_none() && settings.image.is_none() { + settings.image_build = stored.image_build.clone(); + } if sent.get("claude_ai_connectors").is_none() { settings.claude_ai_connectors = stored.claude_ai_connectors; } @@ -159,6 +164,35 @@ mod tests { assert_eq!(s.screenshot_days, None); } + #[test] + fn updates_without_the_build_keep_it_unless_they_choose_an_image() { + let stored = ProjectSettings { + image_build: Some(fctl_proto::session::ImageBuild { + file: "Dockerfile".into(), + context: None, + }), + ..Default::default() + }; + // An older client, which doesn't know builds. + let mut s = ProjectSettings::default(); + keep_unsent(&mut s, &stored, &json!({ "credit_claude": true })); + assert_eq!(s.image_build, stored.image_build); + // One that chooses an image replaces the build with it. + let mut s = ProjectSettings { + image: Some("ghcr.io/acme/agent:1".into()), + ..Default::default() + }; + keep_unsent(&mut s, &stored, &json!({ "image": "ghcr.io/acme/agent:1" })); + assert_eq!(s.image_build, None); + // `null` stops building. + let mut s = ProjectSettings::default(); + keep_unsent(&mut s, &stored, &json!({ "image_build": null })); + assert_eq!(s.image_build, None); + // Settings as this version sends them always say. + let sent = serde_json::to_value(ProjectSettings::default()).unwrap_or_default(); + assert_eq!(sent.get("image_build"), Some(&Value::Null)); + } + #[test] fn idle_sessions_stop_after_ten_minutes_unless_the_project_says_otherwise() { assert_eq!(idle_timeout_secs(&ProjectSettings::default()), 600); diff --git a/crates/fctl-domain/src/validate.rs b/crates/fctl-domain/src/validate.rs index ea8033a2..ce66f369 100644 --- a/crates/fctl-domain/src/validate.rs +++ b/crates/fctl-domain/src/validate.rs @@ -2,6 +2,7 @@ //! [`ApiError`] naming the field, so handlers can return it as is. use crate::webpush; +use fctl_proto::session::ImageBuild; use fctl_proto::{ApiError, AttachmentId}; pub type Result = std::result::Result; @@ -395,12 +396,65 @@ pub fn image(input: &str) -> Result { || !s .bytes() .all(|b| b.is_ascii_alphanumeric() || b"-._/:@".contains(&b)) + || crate::image::ImageRef::parse(s).is_none() { return Err(ApiError::validation("image", "Not a valid image reference")); } Ok(s.to_owned()) } +/// A sandbox image built from the project's repository: a file in it, and +/// for a Dockerfile, the directory it builds in (`None` for the root). +pub fn image_build(build: &ImageBuild) -> Result { + const FIELD: &str = "settings.image_build"; + let file = repo_path(&build.file) + .filter(|f| !f.is_empty()) + .ok_or_else(|| { + ApiError::validation( + FIELD, + "Name a Dockerfile or a devcontainer.json by its path in the repository", + ) + })?; + let context = match build.context.as_deref() { + None => None, + Some(c) => Some(repo_path(c).ok_or_else(|| { + ApiError::validation( + FIELD, + "Name the build's directory by its path in the repository", + ) + })?), + } + .filter(|c| !c.is_empty()); + let build = ImageBuild { file, context }; + if build.is_devcontainer() && build.context.is_some() { + return Err(ApiError::validation( + FIELD, + "A devcontainer.json says where it builds; leave the directory empty", + )); + } + Ok(build) +} + +/// A relative path in a repository, without `./` at its start or a slash at +/// its end: `""` for the root. `None` when it leaves the repository or isn't +/// a path. +fn repo_path(input: &str) -> Option { + let mut s = input.trim(); + while let Some(rest) = s.strip_prefix("./") { + s = rest; + } + let s = s.trim_end_matches('/'); + if s == "." || s.is_empty() { + return Some(String::new()); + } + let ok = s.len() <= 255 + && !s.starts_with('/') + && !s.chars().any(|c| c.is_control() || c == '\\') + && s.split('/') + .all(|part| !part.is_empty() && part != "." && part != ".."); + ok.then(|| s.to_owned()) +} + /// A push subscription's endpoint: an https URL on a named host, at the /// browser's push service. With `local` (development), plain http and local /// hosts too. @@ -671,6 +725,65 @@ mod tests { } } + #[test] + fn image_builds() { + let build = |file: &str, context: Option<&str>| { + image_build(&ImageBuild { + file: file.into(), + context: context.map(str::to_owned), + }) + }; + let ok = build(" ./docker/Dockerfile ", Some("./")).ok(); + assert_eq!( + ok, + Some(ImageBuild { + file: "docker/Dockerfile".into(), + context: None + }) + ); + assert_eq!( + build("Dockerfile", Some("app/")) + .ok() + .and_then(|b| b.context), + Some("app".into()) + ); + assert!(build(".devcontainer/devcontainer.json", None).is_ok()); + assert!(build(".devcontainer/devcontainer.json", Some("..")).is_err()); + assert!(build(".devcontainer.json", Some("app")).is_err()); + for bad in [ + "", + ".", + "/etc/passwd", + "../Dockerfile", + "a/../../b", + "a//b", + "a\\b", + ] { + assert!(build(bad, None).is_err(), "{bad:?}"); + } + assert!(build("Dockerfile", Some("../other")).is_err()); + } + + #[test] + fn images() { + for ok in [ + "ubuntu", + "ghcr.io/acme/agent:1", + "localhost:5000/a@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + ] { + assert!(image(ok).is_ok(), "{ok}"); + } + for bad in [ + "", + "a b", + "acme/agent:", + "acme//agent", + "ghcr.io/acme/agent@latest", + ] { + assert!(image(bad).is_err(), "{bad}"); + } + } + #[test] fn branches() { for ok in ["main", "fctl/12-add-login", "feature/x.y"] { diff --git a/crates/fctl-host/src/build.rs b/crates/fctl-host/src/build.rs new file mode 100644 index 00000000..76a873e5 --- /dev/null +++ b/crates/fctl-host/src/build.rs @@ -0,0 +1,1570 @@ +//! Sandbox images built from a project's repository: a Dockerfile, or a dev +//! container configuration, as the project's default branch has it. A host +//! builds one when its owner allows builds from the repository +//! (`allowed_builds` in host.toml), keeps it, and builds it again once the +//! files it comes from change. One build runs at a time, within +//! `build_timeout_minutes`, and an image larger than `max_build_mb` is +//! removed. + +use crate::session::HostCtx; +use anyhow::{Context, Result, bail}; +use fctl_proto::runner::SessionSpec; +use fctl_proto::session::{ImageBuild, RepoRef}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use std::collections::BTreeMap; +use std::path::{Component, Path, PathBuf}; +use std::time::Duration; + +/// Changes when fctl builds differently, so that hosts build again. +const RECIPE: &str = "fctl image build 1"; + +/// The largest repository archive a host downloads to build from. +const MAX_ARCHIVE_BYTES: u64 = 2 * 1024 * 1024 * 1024; + +/// How long downloading the archive may take. +const ARCHIVE_TIMEOUT: Duration = Duration::from_secs(15 * 60); + +/// The largest Dockerfile or dev container configuration a host reads. +const MAX_FILE_BYTES: usize = 1024 * 1024; + +/// What to build, as the setting and the file it names say. Paths are in +/// the repository, `""` for its root. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Recipe { + /// Build `dockerfile` in `context`. + Build { + dockerfile: String, + context: String, + args: BTreeMap, + target: Option, + }, + /// Start from this image: a dev container configuration that names one + /// and builds nothing. + Image(String), +} + +/// The recipe for `setting`, given the text of the file it names, and what +/// fctl leaves out of a dev container configuration, for the log. +pub fn recipe(setting: &ImageBuild, file: &[u8]) -> Result<(Recipe, Vec)> { + if !setting.is_devcontainer() { + let recipe = Recipe::Build { + dockerfile: setting.file.clone(), + context: setting.context.clone().unwrap_or_default(), + args: BTreeMap::new(), + target: None, + }; + return Ok((recipe, vec![])); + } + let text = String::from_utf8_lossy(file); + let config: Value = serde_json::from_str(&strip_jsonc(&text)) + .with_context(|| format!("Reading {} (JSON with comments)", setting.file))?; + devcontainer(&setting.file, &config) +} + +/// The recipe in a dev container configuration at `path`: its `image`, or +/// its `build` (`dockerfile`, `context`, `args`, `target`; `dockerFile` and +/// `context` at the top level in older ones), with paths relative to it. +fn devcontainer(path: &str, config: &Value) -> Result<(Recipe, Vec)> { + if config.get("dockerComposeFile").is_some() { + bail!( + "{path} uses Docker Compose, which fctl doesn't build: name an image or a Dockerfile in it, or a Dockerfile in the project's settings" + ); + } + let dir = match path.rsplit_once('/') { + Some((dir, _)) => dir, + None => "", + }; + let build = config.get("build"); + let field = |name: &str| { + build + .and_then(|b| b.get(name)) + .or_else(|| config.get(name)) + .and_then(Value::as_str) + .filter(|s| !s.trim().is_empty()) + }; + let dockerfile = field("dockerfile").or_else(|| field("dockerFile")); + let mut notes = Vec::new(); + let ignored: Vec<&str> = [ + "features", + "onCreateCommand", + "updateContentCommand", + "postCreateCommand", + "postStartCommand", + "postAttachCommand", + "containerEnv", + "mounts", + "runArgs", + ] + .into_iter() + .filter(|k| config.get(*k).is_some_and(|v| !is_empty(v))) + .collect(); + if !ignored.is_empty() { + notes.push(format!( + "fctl leaves out {} from {path}: install what they would in the Dockerfile, or in the project's setup script", + ignored.join(", ") + )); + } + if let Some(user) = ["remoteUser", "containerUser"] + .iter() + .find_map(|k| config.get(*k).and_then(Value::as_str)) + { + notes.push(format!( + "Sessions run as the image's own user, not {user} from {path}: end the Dockerfile with `USER {user}` to run as {user}" + )); + } + let Some(dockerfile) = dockerfile else { + return match config.get("image").and_then(Value::as_str) { + Some(image) if fctl_domain::image::ImageRef::parse(image).is_some() => { + Ok((Recipe::Image(image.trim().to_owned()), notes)) + } + Some(image) if !image.trim().is_empty() => { + bail!("{path} names an image that isn't a valid reference: {image}") + } + _ => bail!("{path} names neither an image nor a Dockerfile"), + }; + }; + let context = field("context").unwrap_or("."); + let args = build + .and_then(|b| b.get("args")) + .and_then(Value::as_object) + .map(|args| { + args.iter() + .filter_map(|(k, v)| v.as_str().map(|v| (k.clone(), v.to_owned()))) + .collect() + }) + .unwrap_or_default(); + let target = build + .and_then(|b| b.get("target")) + .and_then(Value::as_str) + .filter(|t| !t.trim().is_empty()) + .map(str::to_owned); + let recipe = Recipe::Build { + dockerfile: join(dir, dockerfile) + .with_context(|| format!("The Dockerfile {path} names"))?, + context: join(dir, context).with_context(|| format!("The directory {path} names"))?, + args, + target, + }; + Ok((recipe, notes)) +} + +fn is_empty(v: &Value) -> bool { + match v { + Value::Null => true, + Value::Object(o) => o.is_empty(), + Value::Array(a) => a.is_empty(), + Value::String(s) => s.trim().is_empty(), + _ => false, + } +} + +/// `rel` from the directory `dir`, both in the repository: `""` for its +/// root. Fails for a path that leaves it. +fn join(dir: &str, rel: &str) -> Result { + let mut parts: Vec<&str> = if rel.starts_with('/') { + vec![] + } else { + dir.split('/').filter(|p| !p.is_empty()).collect() + }; + for part in rel.split('/') { + match part { + "" | "." => {} + ".." => { + if parts.pop().is_none() { + bail!("`{rel}` is outside the repository"); + } + } + p => parts.push(p), + } + } + Ok(parts.join("/")) +} + +/// JSON with comments, as dev container configurations are, made plain: +/// without `//` and `/* */` comments, or commas before a closing bracket. +/// One pass: a comma waits for the next character that isn't space or a +/// comment, which says whether it stays. +pub fn strip_jsonc(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut chars = text.chars().peekable(); + let mut in_string = false; + let mut comma: Option = None; + while let Some(c) = chars.next() { + if in_string { + out.push(c); + match c { + '\\' => { + if let Some(next) = chars.next() { + out.push(next); + } + } + '"' => in_string = false, + _ => {} + } + continue; + } + match (c, chars.peek()) { + ('/', Some('/')) => { + for c in chars.by_ref() { + if c == '\n' { + out.push('\n'); + break; + } + } + } + ('/', Some('*')) => { + chars.next(); + let mut last = ' '; + for c in chars.by_ref() { + if last == '*' && c == '/' { + break; + } + last = c; + } + out.push(' '); + } + (c, _) if c.is_whitespace() => out.push(c), + (c, _) => { + if let Some(at) = comma.take() + && (c == '}' || c == ']') + { + out.remove(at); + } + match c { + '"' => in_string = true, + ',' => comma = Some(out.len()), + _ => {} + } + out.push(c); + } + } + } + out +} + +/// What a build's tag comes from: the recipe, the text of the file the +/// setting names, the Dockerfile's text, and the git tree of the context, +/// which changes with any file in it. A host builds again when one changes. +pub fn key(recipe: &Recipe, file: &[u8], dockerfile: &[u8], context_tree: &str) -> String { + let mut h = Sha256::new(); + let mut part = |bytes: &[u8]| { + h.update((bytes.len() as u64).to_be_bytes()); + h.update(bytes); + }; + part(RECIPE.as_bytes()); + part(file); + part(dockerfile); + part(context_tree.as_bytes()); + if let Recipe::Build { + dockerfile, + context, + args, + target, + } = recipe + { + part(dockerfile.as_bytes()); + part(context.as_bytes()); + for (k, v) in args { + part(k.as_bytes()); + part(v.as_bytes()); + } + part(target.as_deref().unwrap_or_default().as_bytes()); + } + hex::encode(h.finalize())[..12].to_owned() +} + +/// The name of a project's builds on this host: `fctl-build-prj_…`. +pub fn repository(project_id: &str) -> String { + let id: String = project_id + .to_ascii_lowercase() + .chars() + .map(|c| { + if c.is_ascii_alphanumeric() || c == '_' { + c + } else { + '-' + } + }) + .collect(); + format!("fctl-build-{}", id.trim_matches(['-', '_'])) +} + +/// GitHub didn't answer, or said to come back later (5xx, 429, or out of +/// requests): then sessions may start from the last build. +#[derive(Debug)] +pub struct Unreachable(String); + +impl std::fmt::Display for Unreachable { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +impl std::error::Error for Unreachable {} + +/// The GitHub API, with a session's token. +pub struct Github { + http: fctl_client::HttpClient, + base: String, + token: String, +} + +impl Github { + pub fn new(base: &str, token: &str) -> Self { + Self { + http: fctl_client::HttpClient::new(Duration::from_secs(60)), + base: base.trim_end_matches('/').to_owned(), + token: token.to_owned(), + } + } + + fn url(&self, repo: &RepoRef, rest: &str) -> String { + format!( + "{}/repos/{}/{}/{rest}", + self.base, + encode(&repo.owner), + encode(&repo.name) + ) + } + + /// GET `url`; `None` for 404, [`Unreachable`] when GitHub can't answer + /// now. + async fn get(&self, url: &str, accept: &str) -> Result>> { + let headers = [("accept", accept), ("x-github-api-version", "2022-11-28")]; + let (status, headers, body) = self + .http + .get_with(url, Some(&self.token), &headers) + .await + .map_err(|e| Unreachable(format!("Asking GitHub ({url}): {e}")))?; + if status.as_u16() == 404 { + return Ok(None); + } + let out_of_requests = status.as_u16() == 403 + && headers + .get("x-ratelimit-remaining") + .is_some_and(|v| v.as_bytes() == b"0"); + if status.is_server_error() || status.as_u16() == 429 || out_of_requests { + return Err(Unreachable(format!("GitHub answered {status} for {url}")).into()); + } + if !status.is_success() { + let text: String = String::from_utf8_lossy(&body).chars().take(300).collect(); + bail!("GitHub answered {status} for {url}: {text}"); + } + Ok(Some(body.to_vec())) + } + + /// The commit at the tip of `branch`. + pub async fn commit(&self, repo: &RepoRef, branch: &str) -> Result { + let url = self.url(repo, &format!("commits/{}", encode_path(branch))); + let body = self + .get(&url, "application/vnd.github.sha") + .await? + .with_context(|| format!("{} has no branch {branch}", repo.full_name()))?; + let sha = String::from_utf8_lossy(&body).trim().to_owned(); + if sha.len() < 40 || !sha.bytes().all(|b| b.is_ascii_hexdigit()) { + bail!("GitHub sent no commit for {branch}"); + } + Ok(sha) + } + + /// The file at `path` in `commit`, or `None`. + pub async fn file(&self, repo: &RepoRef, commit: &str, path: &str) -> Result>> { + let url = self.url( + repo, + &format!("contents/{}?ref={commit}", encode_path(path)), + ); + self.get(&url, "application/vnd.github.raw").await + } + + /// The git tree of the directory `dir` in `commit` (`""`: the root), or + /// `None` when there is no such directory. + pub async fn tree(&self, repo: &RepoRef, commit: &str, dir: &str) -> Result> { + let url = self.url(repo, &format!("git/trees/{commit}:{}", encode_path(dir))); + let Some(body) = self.get(&url, "application/vnd.github+json").await? else { + return Ok(None); + }; + let v: Value = serde_json::from_slice(&body).context("Reading GitHub's tree")?; + Ok(v.get("sha").and_then(Value::as_str).map(str::to_owned)) + } + + /// Downloads the repository as it is in `commit`, a `.tar.gz`, to `dest`. + pub async fn archive(&self, repo: &RepoRef, commit: &str, dest: &Path) -> Result<()> { + let url = self.url(repo, &format!("tarball/{commit}")); + let headers = [("x-github-api-version", "2022-11-28")]; + self.http + .download_to( + &url, + Some(&self.token), + &headers, + dest, + MAX_ARCHIVE_BYTES, + ARCHIVE_TIMEOUT, + ) + .await + .with_context(|| format!("Downloading {} at {commit}", repo.full_name()))?; + Ok(()) + } +} + +/// Percent-encodes one part of a URL's path. +fn encode(part: &str) -> String { + let mut out = String::new(); + for b in part.bytes() { + if b.is_ascii_alphanumeric() || b"-._~".contains(&b) { + out.push(b as char); + } else { + out.push_str(&format!("%{b:02X}")); + } + } + out +} + +/// Percent-encodes a path, keeping its slashes. +fn encode_path(path: &str) -> String { + path.split('/').map(encode).collect::>().join("/") +} + +/// Unpacks the parts of a GitHub archive (`.tar.gz`, one directory at the +/// top) that a build reads, the files under `context`, `dockerfile` and its +/// `.dockerignore`, into `dest`, as they are in the repository. Only files, +/// directories and symbolic links; nothing outside `dest`, and nothing +/// through a link. +pub fn unpack(archive: &Path, dest: &Path, context: &str, dockerfile: &str) -> Result<()> { + let file = + std::fs::File::open(archive).with_context(|| format!("Opening {}", archive.display()))?; + let mut tar = tar::Archive::new(flate2::read::GzDecoder::new(file)); + tar.set_preserve_permissions(true); + let context = Path::new(context); + let ignore = PathBuf::from(format!("{dockerfile}.dockerignore")); + let dockerfile = Path::new(dockerfile); + std::fs::create_dir_all(dest)?; + let root = dest.canonicalize()?; + for entry in tar.entries().context("Reading the repository's archive")? { + let mut entry = entry.context("Reading the repository's archive")?; + let kind = entry.header().entry_type(); + if !(kind.is_file() || kind.is_dir() || kind.is_symlink()) { + continue; + } + let path = entry.path()?.into_owned(); + // The archive's own directory, `owner-repo-sha/`, comes first. + let mut parts = path.components(); + parts.next(); + let rel: PathBuf = parts.collect(); + if rel.as_os_str().is_empty() + || !rel.components().all(|c| matches!(c, Component::Normal(_))) + { + continue; + } + let wanted = rel.starts_with(context) + || rel == dockerfile + || rel == ignore + || (kind.is_dir() && (context.starts_with(&rel) || dockerfile.starts_with(&rel))); + if !wanted || through_link(dest, &rel) { + continue; + } + let out = dest.join(&rel); + if let Some(parent) = out.parent() { + std::fs::create_dir_all(parent)?; + if !parent.canonicalize()?.starts_with(&root) { + continue; + } + } + entry + .unpack(&out) + .with_context(|| format!("Unpacking {}", rel.display()))?; + } + Ok(()) +} + +/// Whether a directory on the way to `rel` under `dest` is a symbolic link. +fn through_link(dest: &Path, rel: &Path) -> bool { + let mut at = dest.to_path_buf(); + for part in rel.parent().into_iter().flat_map(Path::components) { + at.push(part); + match std::fs::symlink_metadata(&at) { + Ok(m) if m.file_type().is_symlink() => return true, + Ok(_) => {} + // The rest isn't there yet: unpacking makes real directories. + Err(_) => return false, + } + } + false +} + +/// A directory that goes when dropped, also when a stop calls off the build +/// that uses it. +struct Scratch(PathBuf); + +impl Drop for Scratch { + fn drop(&mut self) { + let dir = std::mem::take(&mut self.0); + // Removing a large archive takes a moment; not on the runtime. + std::thread::spawn(move || { + let _ = std::fs::remove_dir_all(dir); + }); + } +} + +/// The last build of each project on this host, in `builds.json` in its +/// data directory. +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct Builds { + pub projects: BTreeMap, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct Built { + /// The image. + pub tag: String, + /// The commit it was built from. + pub commit: String, + /// The setting it was built for: a changed setting doesn't start from it. + #[serde(default)] + pub setting: Option, +} + +impl Builds { + fn path(data_dir: &Path) -> PathBuf { + data_dir.join("builds.json") + } + + pub fn load(data_dir: &Path) -> Self { + std::fs::read_to_string(Self::path(data_dir)) + .ok() + .and_then(|s| serde_json::from_str(&s).ok()) + .unwrap_or_default() + } + + pub fn save(&self, data_dir: &Path) -> Result<()> { + std::fs::create_dir_all(data_dir)?; + let path = Self::path(data_dir); + let tmp = path.with_extension("tmp"); + std::fs::write(&tmp, serde_json::to_string_pretty(self)?)?; + std::fs::rename(&tmp, &path)?; + Ok(()) + } +} + +/// What a project's build resolves to at the tip of its default branch. +enum Resolved { + /// The image to start from, built or not. + Ready(String), + /// Build `tag` from `commit`, with the Dockerfile GitHub sent. + Build { + tag: String, + commit: String, + recipe: Recipe, + dockerfile: Vec, + }, +} + +/// The image a session of a project that builds its own starts from: this +/// host's build of the files on the project's default branch, built now +/// when they changed since. While GitHub can't be reached, the last build +/// for the same setting. +pub async fn image( + ctx: &HostCtx, + spec: &SessionSpec, + log: &(dyn Fn(String) + Send + Sync), +) -> Result { + let project = &spec.project; + let setting = project + .settings + .image_build + .as_ref() + .context("The project builds no image")?; + let repo = project + .repo + .as_ref() + .context("The project has no repository to build its image from")?; + let token = spec + .credentials + .github + .as_ref() + .or(spec.credentials.github_read.as_ref()) + .map(|c| c.token.as_str()) + .with_context(|| { + format!( + "The session has no GitHub token to read {}'s files with", + repo.full_name() + ) + })?; + let gh = Github::new(&ctx.cfg.github_api, token); + let data_dir = ctx.cfg.data_dir(); + let id = project.id.as_str(); + let name = repository(id); + let resolved = match resolve(&gh, repo, &project.default_branch, setting, &name, log).await { + Ok(r) => r, + Err(e) if e.downcast_ref::().is_some() => { + // GitHub is down: the last build of the same setting will do. + let last = Builds::load(&data_dir).projects.get(id).cloned(); + match last { + Some(last) + if last.setting.as_ref() == Some(setting) + && ctx.driver.has_image(&last.tag).await => + { + log(format!( + "Could not check {} for changes to its image ({e:#}); starting from the last build, {}", + repo.full_name(), + last.tag + )); + return Ok(last.tag); + } + _ => return Err(e), + } + } + Err(e) => return Err(e), + }; + let (tag, commit, recipe, dockerfile) = match resolved { + Resolved::Ready(image) => return Ok(image), + Resolved::Build { + tag, + commit, + recipe, + dockerfile, + } => (tag, commit, recipe, dockerfile), + }; + if !ctx.driver.has_image(&tag).await { + // Sessions that need the same build wait for the first, and find + // its image. + let same = ctx.build_lock(&tag); + let _same = same.lock().await; + if !ctx.driver.has_image(&tag).await { + let job = Job { + repo, + commit: &commit, + recipe: &recipe, + tag: &tag, + dockerfile: &dockerfile, + setting, + }; + build(ctx, &gh, &job, log).await?; + } + } + let previous = { + let _record = ctx + .build_record + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let mut builds = Builds::load(&data_dir); + let previous = builds.projects.insert( + id.to_owned(), + Built { + tag: tag.clone(), + commit, + setting: Some(setting.clone()), + }, + ); + if let Err(e) = builds.save(&data_dir) { + tracing::warn!("saving builds.json: {e:#}"); + } + previous + }; + // The project's earlier build goes, unless a sandbox still uses it. + if let Some(old) = previous.filter(|old| old.tag != tag) { + match ctx.driver.remove_image(&old.tag).await { + Ok(()) => tracing::info!("removed {}, an earlier build", old.tag), + Err(e) => tracing::info!("keeping {}: {e:#}", old.tag), + } + } + Ok(tag) +} + +/// Reads the setting's file at the tip of `branch` and works out what to +/// build, or the image to start from. +async fn resolve( + gh: &Github, + repo: &RepoRef, + branch: &str, + setting: &ImageBuild, + name: &str, + log: &(dyn Fn(String) + Send + Sync), +) -> Result { + let commit = gh.commit(repo, branch).await?; + let missing = |path: &str| { + format!( + "{path} isn't in {} on {branch} (Settings → Sandbox → Image)", + repo.full_name() + ) + }; + let file = gh + .file(repo, &commit, &setting.file) + .await? + .with_context(|| missing(&setting.file))?; + too_large(&setting.file, &file)?; + let (recipe, notes) = recipe(setting, &file)?; + for note in notes { + log(note); + } + let (dockerfile, context) = match &recipe { + Recipe::Image(image) => return Ok(Resolved::Ready(image.clone())), + Recipe::Build { + dockerfile, + context, + .. + } => (dockerfile, context), + }; + let dockerfile_text = if *dockerfile == setting.file { + file.clone() + } else { + gh.file(repo, &commit, dockerfile) + .await? + .with_context(|| missing(dockerfile))? + }; + too_large(dockerfile, &dockerfile_text)?; + let tree = gh.tree(repo, &commit, context).await?.with_context(|| { + format!( + "{} has no directory {} on {branch} to build in", + repo.full_name(), + if context.is_empty() { "/" } else { context } + ) + })?; + let tag = format!("{name}:{}", key(&recipe, &file, &dockerfile_text, &tree)); + Ok(Resolved::Build { + tag, + commit, + recipe, + dockerfile: dockerfile_text, + }) +} + +/// Refuses a Dockerfile or configuration larger than [`MAX_FILE_BYTES`]. +fn too_large(path: &str, text: &[u8]) -> Result<()> { + if text.len() > MAX_FILE_BYTES { + bail!( + "{path} is larger than {} MiB: name a Dockerfile or a devcontainer.json", + MAX_FILE_BYTES / 1024 / 1024 + ); + } + Ok(()) +} + +/// One build: what it comes from and the image it makes. +struct Job<'a> { + repo: &'a RepoRef, + commit: &'a str, + recipe: &'a Recipe, + tag: &'a str, + /// The Dockerfile's text, as GitHub sent it. + dockerfile: &'a [u8], + setting: &'a ImageBuild, +} + +/// Downloads the repository at the job's commit and builds its image, +/// once no other build runs on this host. The Dockerfile is the text GitHub +/// sent, written beside the repository: never a file the repository links +/// to on the host. +async fn build( + ctx: &HostCtx, + gh: &Github, + job: &Job<'_>, + log: &(dyn Fn(String) + Send + Sync), +) -> Result<()> { + let Recipe::Build { + dockerfile, + context, + args, + target, + } = job.recipe + else { + return Ok(()); + }; + let short = &job.commit[..job.commit.len().min(7)]; + log(format!( + "Building the sandbox image from {} in {} at {short}", + job.setting.file, + job.repo.full_name() + )); + let dir = ctx.cfg.data_dir().join("builds").join(format!( + "{}-{}", + job.tag.replace([':', '/'], "-"), + std::process::id() + )); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir)?; + let _scratch = Scratch(dir.clone()); + // Download first: a token that waits behind other builds could expire. + let archive = dir.join("repo.tar.gz"); + gh.archive(job.repo, job.commit, &archive).await?; + let src = dir.join("src"); + let (archive2, src2, context2, dockerfile2) = ( + archive.clone(), + src.clone(), + context.clone(), + dockerfile.clone(), + ); + tokio::task::spawn_blocking(move || unpack(&archive2, &src2, &context2, &dockerfile2)) + .await + .context("Unpacking the repository")??; + let _ = std::fs::remove_file(&archive); + let file = dir.join("Dockerfile"); + std::fs::write(&file, job.dockerfile)?; + let ignore = src.join(format!("{dockerfile}.dockerignore")); + if std::fs::symlink_metadata(&ignore).is_ok_and(|m| m.is_file()) { + std::fs::copy(&ignore, dir.join("Dockerfile.dockerignore"))?; + } + let turn = match ctx.builds.try_lock() { + Ok(turn) => turn, + Err(_) => { + log("Waiting for another image build on this host".into()); + ctx.builds.lock().await + } + }; + let limit = Duration::from_secs(u64::from(ctx.cfg.build_timeout_minutes.max(1)) * 60); + let spec = crate::driver::BuildJob { + tag: job.tag, + dockerfile: &file, + context: &src.join(context), + args, + target: target.as_deref(), + limit, + }; + ctx.driver.build(&spec, log).await?; + drop(turn); + let max = u64::from(ctx.cfg.max_build_mb) * 1024 * 1024; + if let Some(size) = ctx.driver.image_size(job.tag).await + && size > max + { + let _ = ctx.driver.remove_image(job.tag).await; + bail!( + "The image is {}, more than the {} this host keeps (max_build_mb in host.toml)", + fctl_domain::size::memory(size / 1024 / 1024), + fctl_domain::size::memory(u64::from(ctx.cfg.max_build_mb)) + ); + } + log(format!("Built {}", job.tag)); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn setting(file: &str, context: Option<&str>) -> ImageBuild { + ImageBuild { + file: file.into(), + context: context.map(str::to_owned), + } + } + + fn build(dockerfile: &str, context: &str) -> Recipe { + Recipe::Build { + dockerfile: dockerfile.into(), + context: context.into(), + args: BTreeMap::new(), + target: None, + } + } + + #[test] + fn a_dockerfile_builds_in_its_context() -> Result<()> { + let (r, notes) = recipe(&setting("docker/Dockerfile", None), b"FROM x")?; + assert_eq!(r, build("docker/Dockerfile", "")); + assert!(notes.is_empty()); + let (r, _) = recipe(&setting("Dockerfile", Some("app")), b"FROM x")?; + assert_eq!(r, build("Dockerfile", "app")); + Ok(()) + } + + #[test] + fn reads_json_with_comments() -> Result<()> { + let text = r#"{ + // The name. + "name": "app // not a comment", /* a block + comment */ + "build": { + "dockerfile": "Dockerfile", + "args": { "A": "1", }, + }, +}"#; + let v: Value = serde_json::from_str(&strip_jsonc(text))?; + assert_eq!(v["name"], "app // not a comment"); + assert_eq!(v["build"]["args"]["A"], "1"); + assert_eq!(strip_jsonc(r#"{"a": "x\"," , }"#), r#"{"a": "x\"," }"#); + assert_eq!(strip_jsonc("[1, // two\n]"), "[1 \n]"); + Ok(()) + } + + #[test] + fn reads_large_json_in_one_pass() -> Result<()> { + // 200 000 commas: quadratic work would take minutes. + let items = vec!["1"; 200_000].join(", "); + let text = format!("{{ \"a\": [{items}, ], // done\n }}"); + let started = std::time::Instant::now(); + let v: Value = serde_json::from_str(&strip_jsonc(&text))?; + assert!(started.elapsed() < Duration::from_secs(5)); + assert_eq!(v["a"].as_array().map(Vec::len), Some(200_000)); + assert!(too_large("x.json", &vec![b' '; MAX_FILE_BYTES + 1]).is_err()); + assert!(too_large("x.json", text.as_bytes()).is_ok()); + Ok(()) + } + + #[test] + fn dev_containers_build_relative_to_their_configuration() -> Result<()> { + let config = json!({ + "build": { "dockerfile": "Dockerfile", "context": "..", "args": { "V": "3" }, "target": "dev" }, + "features": { "ghcr.io/devcontainers/features/node:1": {} }, + "remoteUser": "vscode", + "postCreateCommand": "npm ci" + }); + let (r, notes) = devcontainer(".devcontainer/devcontainer.json", &config)?; + assert_eq!( + r, + Recipe::Build { + dockerfile: ".devcontainer/Dockerfile".into(), + context: "".into(), + args: BTreeMap::from([("V".to_owned(), "3".to_owned())]), + target: Some("dev".into()), + } + ); + assert_eq!(notes.len(), 2, "{notes:?}"); + assert!( + notes[0].contains("features, postCreateCommand"), + "{notes:?}" + ); + assert!(notes[1].contains("USER vscode"), "{notes:?}"); + // The older form, and the default context: the configuration's directory. + let (r, _) = devcontainer( + ".devcontainer/devcontainer.json", + &json!({ "dockerFile": "../docker/Dockerfile" }), + )?; + assert_eq!(r, build("docker/Dockerfile", ".devcontainer")); + // An image, built from nothing. + let (r, notes) = devcontainer( + ".devcontainer.json", + &json!({ "image": "mcr.microsoft.com/devcontainers/rust:1", "features": {} }), + )?; + assert_eq!( + r, + Recipe::Image("mcr.microsoft.com/devcontainers/rust:1".into()) + ); + assert!(notes.is_empty()); + // What fctl can't build. + for config in [ + json!({ "dockerComposeFile": "compose.yml", "service": "app" }), + json!({ "name": "nothing" }), + json!({ "build": { "dockerfile": "../../Dockerfile" } }), + json!({ "image": "-x ubuntu" }), + ] { + assert!( + devcontainer(".devcontainer/devcontainer.json", &config).is_err(), + "{config}" + ); + } + Ok(()) + } + + #[test] + fn builds_again_when_what_they_come_from_changes() { + let r = build("Dockerfile", ""); + let k = key(&r, b"FROM a", b"FROM a", "tree1"); + assert_eq!(k.len(), 12); + assert_eq!(k, key(&r, b"FROM a", b"FROM a", "tree1")); + assert_ne!(k, key(&r, b"FROM a", b"FROM a", "tree2")); + assert_ne!(k, key(&r, b"FROM b", b"FROM a", "tree1")); + assert_ne!( + k, + key(&build("Dockerfile", "app"), b"FROM a", b"FROM a", "tree1") + ); + assert_eq!( + repository("prj_01M57YPG1ATK9ARAXTZG43G51J"), + "fctl-build-prj_01m57ypg1atk9araxtzg43g51j" + ); + } + + #[test] + fn joins_paths_in_the_repository() -> Result<()> { + assert_eq!( + join(".devcontainer", "Dockerfile")?, + ".devcontainer/Dockerfile" + ); + assert_eq!(join(".devcontainer", "..")?, ""); + assert_eq!(join(".devcontainer", "./x/../y")?, ".devcontainer/y"); + assert_eq!(join("", ".")?, ""); + assert!(join(".devcontainer", "../..").is_err()); + Ok(()) + } + + /// A GitHub archive of `files`, under `acme-app-1234567/`, with a + /// symbolic link and a hard link. + fn archive(dir: &Path, files: &[(&str, &str)]) -> Result { + let path = dir.join("repo.tar.gz"); + let gz = flate2::write::GzEncoder::new( + std::fs::File::create(&path)?, + flate2::Compression::fast(), + ); + let mut tar = tar::Builder::new(gz); + for (name, text) in files { + let mut h = tar::Header::new_gnu(); + h.set_mode(0o755); + h.set_size(text.len() as u64); + tar.append_data(&mut h, format!("acme-app-1234567/{name}"), text.as_bytes())?; + } + let mut link = tar::Header::new_gnu(); + link.set_entry_type(tar::EntryType::Link); + link.set_size(0); + tar.append_link(&mut link, "acme-app-1234567/app/hard", "/etc/passwd")?; + let mut sym = tar::Header::new_gnu(); + sym.set_entry_type(tar::EntryType::Symlink); + sym.set_size(0); + tar.append_link(&mut sym, "acme-app-1234567/app/link", "run.sh")?; + tar.into_inner()?.finish()?; + Ok(path) + } + + #[test] + fn unpacks_what_the_build_reads() -> Result<()> { + let tmp = tempfile::tempdir()?; + let path = archive( + tmp.path(), + &[ + ("README.md", "# app\n"), + ("docker/Dockerfile", "FROM x\n"), + ("app/run.sh", "#!/bin/sh\n"), + ("app/src/main.rs", "fn main() {}\n"), + ("web/index.html", "

\n"), + ], + )?; + let dest = tmp.path().join("src"); + unpack(&path, &dest, "app", "docker/Dockerfile")?; + assert!(dest.join("docker/Dockerfile").is_file()); + assert!(dest.join("app/src/main.rs").is_file()); + assert!(!dest.join("web").exists()); + assert!(!dest.join("README.md").exists()); + // Hard links could point anywhere; symbolic links stay links. + assert!(!dest.join("app/hard").exists()); + assert!(std::fs::symlink_metadata(dest.join("app/link"))?.is_symlink()); + // The root context takes everything. + let all = tmp.path().join("all"); + unpack(&path, &all, "", "docker/Dockerfile")?; + assert!(all.join("README.md").is_file()); + assert!(all.join("web/index.html").is_file()); + Ok(()) + } + + #[test] + fn unpacks_nothing_through_a_link() -> Result<()> { + let tmp = tempfile::tempdir()?; + let outside = tmp.path().join("outside"); + std::fs::create_dir(&outside)?; + let path = tmp.path().join("repo.tar.gz"); + let gz = flate2::write::GzEncoder::new( + std::fs::File::create(&path)?, + flate2::Compression::fast(), + ); + let mut tar = tar::Builder::new(gz); + let mut sym = tar::Header::new_gnu(); + sym.set_entry_type(tar::EntryType::Symlink); + sym.set_size(0); + tar.append_link(&mut sym, "acme-app-1234567/app/out", &outside)?; + let mut file = tar::Header::new_gnu(); + file.set_mode(0o644); + file.set_size(2); + tar.append_data(&mut file, "acme-app-1234567/app/out/sub/x", &b"x\n"[..])?; + tar.into_inner()?.finish()?; + let dest = tmp.path().join("src"); + unpack(&path, &dest, "app", "Dockerfile")?; + assert!(std::fs::symlink_metadata(dest.join("app/out"))?.is_symlink()); + assert!(!outside.join("sub").exists()); + Ok(()) + } + + #[test] + fn remembers_the_last_build_of_each_project() -> Result<()> { + let tmp = tempfile::tempdir()?; + assert_eq!(Builds::load(tmp.path()), Builds::default()); + let mut b = Builds::default(); + b.projects.insert( + "prj_1".into(), + Built { + tag: "fctl-build-prj_1:abc".into(), + commit: "f".repeat(40), + setting: Some(setting("Dockerfile", None)), + }, + ); + b.save(tmp.path())?; + assert_eq!(Builds::load(tmp.path()), b); + Ok(()) + } + + use crate::driver::Driver; + use crate::driver::container::Flavor; + use crate::driver::container::fake::FakeCli; + use crate::session::HostCtx; + use std::sync::{Arc, Mutex, PoisonError}; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + /// A stand-in for GitHub with one repository, acme/app, and the files on + /// its main branch, which tests change. Archives come from a second + /// address, as GitHub's come from codeload.github.com. + struct FakeGithub { + url: String, + repo: Arc>, + } + + #[derive(Default)] + struct FakeRepo { + files: BTreeMap, + /// Paths the archive holds as symbolic links instead, to these + /// targets, while the API still sends the file's text. + links: BTreeMap, + commits: u32, + down: bool, + /// Each request: the path, and whether it carried the token. + requests: Vec<(String, bool)>, + } + + impl FakeRepo { + fn commit(&self) -> String { + format!("{:040x}", self.commits + 1) + } + + /// A tree hash of the files under `dir`. + fn tree(&self, dir: &str) -> Option { + let prefix = if dir.is_empty() { + String::new() + } else { + format!("{dir}/") + }; + let mut h = Sha256::new(); + let mut any = false; + for (path, text) in &self.files { + if let Some(rest) = path.strip_prefix(&prefix) { + any = true; + h.update(rest.as_bytes()); + h.update(text.as_bytes()); + } + } + any.then(|| hex::encode(h.finalize())) + } + + fn archive(&self) -> Result> { + let gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); + let mut tar = tar::Builder::new(gz); + let top = format!("acme-app-{}", &self.commit()[..7]); + for (path, text) in &self.files { + if let Some(target) = self.links.get(path) { + let mut h = tar::Header::new_gnu(); + h.set_entry_type(tar::EntryType::Symlink); + h.set_size(0); + tar.append_link(&mut h, format!("{top}/{path}"), target)?; + continue; + } + let mut h = tar::Header::new_gnu(); + h.set_mode(0o644); + h.set_size(text.len() as u64); + tar.append_data(&mut h, format!("{top}/{path}"), text.as_bytes())?; + } + Ok(tar.into_inner()?.finish()?) + } + } + + impl FakeGithub { + async fn start(files: &[(&str, &str)]) -> Result { + let api = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let codeload = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let url = format!("http://{}", api.local_addr()?); + let codeload_url = format!("http://{}", codeload.local_addr()?); + let repo = Arc::new(Mutex::new(FakeRepo { + files: files + .iter() + .map(|(p, t)| ((*p).to_owned(), (*t).to_owned())) + .collect(), + ..Default::default() + })); + for listener in [api, codeload] { + let (repo, codeload_url) = (repo.clone(), codeload_url.clone()); + tokio::spawn(async move { + while let Ok((mut sock, _)) = listener.accept().await { + let (repo, codeload_url) = (repo.clone(), codeload_url.clone()); + tokio::spawn(async move { + let _ = serve(&mut sock, &repo, &codeload_url).await; + }); + } + }); + } + Ok(Self { url, repo }) + } + + fn change(&self, f: impl FnOnce(&mut FakeRepo)) { + let mut repo = self.repo.lock().unwrap_or_else(PoisonError::into_inner); + f(&mut repo); + repo.commits += 1; + } + + fn requests(&self) -> Vec<(String, bool)> { + let repo = self.repo.lock().unwrap_or_else(PoisonError::into_inner); + repo.requests.clone() + } + } + + /// Answers one request, then closes the connection. + async fn serve( + sock: &mut tokio::net::TcpStream, + repo: &Mutex, + codeload: &str, + ) -> Result<()> { + let mut head = Vec::new(); + let mut buf = [0u8; 4096]; + while !head.windows(4).any(|w| w == b"\r\n\r\n") { + let n = sock.read(&mut buf).await?; + if n == 0 { + return Ok(()); + } + head.extend_from_slice(&buf[..n]); + } + let head = String::from_utf8_lossy(&head).into_owned(); + let target = head + .split_whitespace() + .nth(1) + .unwrap_or_default() + .to_owned(); + let authorized = head + .to_ascii_lowercase() + .contains("authorization: bearer gh-token"); + let (status, location, body) = { + let mut r = repo.lock().unwrap_or_else(PoisonError::into_inner); + r.requests.push((target.clone(), authorized)); + route(&r, &target, codeload)? + }; + let mut response = format!( + "HTTP/1.1 {status}\r\ncontent-length: {}\r\nconnection: close\r\n", + body.len() + ); + if let Some(l) = location { + response.push_str(&format!("location: {l}\r\n")); + } + response.push_str("\r\n"); + sock.write_all(response.as_bytes()).await?; + sock.write_all(&body).await?; + sock.shutdown().await?; + Ok(()) + } + + fn route( + r: &FakeRepo, + target: &str, + codeload: &str, + ) -> Result<(&'static str, Option, Vec)> { + let not_found = Ok(("404 Not Found", None, b"{}".to_vec())); + if r.down { + return Ok(("503 Service Unavailable", None, b"{}".to_vec())); + } + let commit = r.commit(); + let (path, query) = target.split_once('?').unwrap_or((target, "")); + if path == "/repos/acme/app/commits/main" { + return Ok(("200 OK", None, commit.into_bytes())); + } + if let Some(file) = path.strip_prefix("/repos/acme/app/contents/") { + if query != format!("ref={commit}") { + return not_found; + } + return match r.files.get(file) { + Some(text) => Ok(("200 OK", None, text.clone().into_bytes())), + None => not_found, + }; + } + if let Some(rest) = path.strip_prefix(&format!("/repos/acme/app/git/trees/{commit}:")) { + return match r.tree(rest) { + Some(sha) => Ok(( + "200 OK", + None, + json!({ "sha": sha }).to_string().into_bytes(), + )), + None => not_found, + }; + } + if path == format!("/repos/acme/app/tarball/{commit}") { + let to = format!("{codeload}/acme/app/legacy.tar.gz/{commit}"); + return Ok(("302 Found", Some(to), vec![])); + } + if path == format!("/acme/app/legacy.tar.gz/{commit}") { + return Ok(("200 OK", None, r.archive()?)); + } + not_found + } + + fn spec(file: &str, context: Option<&str>) -> Result { + Ok(serde_json::from_value(json!({ + "session_id": "ses_test", + "number": 1, + "title": "Test", + "project": { + "id": "prj_Test1", + "slug": "app", + "name": "App", + "repo": { "owner": "acme", "name": "app" }, + "default_branch": "main", + "settings": { "image_build": { "file": file, "context": context } } + }, + "branch": "fctl/1-test", + "base_branch": "main", + "author": { "id": "usr_test", "name": "Ada" }, + "claude_session_id": "00000000-0000-0000-0000-000000000000", + "resume": false, + "secrets": {}, + "credentials": { + "claude": { "id": "crd_test", "kind": "setup_token", "access_token": "x", "scopes": [] }, + "github": { "token": "gh-token", "expires_at": 0 } + }, + "git_author": { "name": "fctl", "email": "fctl@example.com" }, + "co_authors": [], + "web_url": "https://fctl.test/s/ses_test", + "idle_timeout_secs": 600 + }))?) + } + + struct Host { + ctx: HostCtx, + cli: FakeCli, + _data: tempfile::TempDir, + log: Mutex>, + } + + impl Host { + fn new(github: &FakeGithub) -> Result { + let cli = FakeCli::new(&[], &[])?; + let data = tempfile::tempdir()?; + let cfg = crate::HostConfig { + data_dir: Some(data.path().to_owned()), + github_api: github.url.clone(), + ..Default::default() + }; + let driver = Driver::Container(cli.driver(Flavor::Docker)); + Ok(Self { + ctx: HostCtx::new(cfg, driver, "fctl_rt_test".into()), + cli, + _data: data, + log: Mutex::new(Vec::new()), + }) + } + + async fn image(&self, spec: &SessionSpec) -> Result { + let log = |line: String| { + self.log + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push(line) + }; + image(&self.ctx, spec, &log).await + } + + fn builds(&self) -> Vec { + let calls = self.cli.calls(); + calls + .into_iter() + .filter(|c| c.starts_with("build ")) + .collect() + } + + fn has(&self, image: &str) -> bool { + self.cli + .dir + .path() + .join("have") + .join(image.replace(['/', ':'], "_")) + .exists() + } + + fn log(&self) -> Vec { + self.log + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone() + } + } + + #[tokio::test] + async fn hosts_build_once_and_again_when_the_files_change() -> Result<()> { + let github = FakeGithub::start(&[ + ("README.md", "# app\n"), + ("docker/Dockerfile", "FROM debian\nCOPY . /app\n"), + ("app/main.rs", "fn main() {}\n"), + ]) + .await?; + let host = Host::new(&github)?; + let spec = spec("docker/Dockerfile", Some("app"))?; + let tag = host.image(&spec).await?; + assert!(tag.starts_with("fctl-build-prj_test1:"), "{tag}"); + assert!(host.has(&tag)); + let builds = host.builds(); + assert_eq!(builds.len(), 1, "{builds:?}"); + assert!( + builds[0].starts_with(&format!("build --tag {tag} --file ")), + "{builds:?}" + ); + // The Dockerfile GitHub sent, beside the repository's files. + assert!(builds[0].contains("/Dockerfile "), "{builds:?}"); + assert!(!builds[0].contains("/src/docker/Dockerfile"), "{builds:?}"); + assert!(builds[0].ends_with("/src/app"), "{builds:?}"); + assert_eq!( + std::fs::read_to_string(host.cli.dir.path().join("dockerfile"))?, + "FROM debian\nCOPY . /app\n" + ); + // It built in the context, from what the archive had there. + let context = std::fs::read_to_string(host.cli.dir.path().join("context"))?; + assert_eq!(context, ".\n./main.rs\n"); + let log = host.log(); + assert!( + log.iter() + .any(|l| l + == "Building the sandbox image from docker/Dockerfile in acme/app at 0000000"), + "{log:?}" + ); + assert!(log.iter().any(|l| l == "#5 [1/2] FROM debian"), "{log:?}"); + assert!(!log.iter().any(|l| l.contains("noise")), "{log:?}"); + // The token went to GitHub's API, not to where the archive came from. + for (path, authorized) in github.requests() { + assert_eq!(authorized, path.starts_with("/repos/"), "{path}"); + } + + // The next session starts from the build. + assert_eq!(host.image(&spec).await?, tag); + assert_eq!(host.builds().len(), 1); + // So does one after a change it doesn't read. + github.change(|r| { + r.files.insert("README.md".into(), "# App\n".into()); + }); + assert_eq!(host.image(&spec).await?, tag); + assert_eq!(host.builds().len(), 1); + // A change in the context builds again, and the old build goes. + github.change(|r| { + r.files.insert("app/lib.rs".into(), "\n".into()); + }); + let rebuilt = host.image(&spec).await?; + assert_ne!(rebuilt, tag); + assert_eq!(host.builds().len(), 2); + assert!(host.has(&rebuilt) && !host.has(&tag)); + // So does a change to the Dockerfile. + github.change(|r| { + r.files + .insert("docker/Dockerfile".into(), "FROM debian:13\n".into()); + }); + let third = host.image(&spec).await?; + assert_ne!(third, rebuilt); + assert_eq!(host.builds().len(), 3); + + // While GitHub is down, sessions start from the last build. + github.change(|r| r.down = true); + assert_eq!(host.image(&spec).await?, third); + assert!( + host.log() + .iter() + .any(|l| l.starts_with("Could not check acme/app for changes")) + ); + Ok(()) + } + + #[tokio::test] + async fn only_an_unreachable_github_starts_sessions_from_the_last_build() -> Result<()> { + let github = FakeGithub::start(&[("Dockerfile", "FROM debian\n")]).await?; + let host = Host::new(&github)?; + let tag = host.image(&spec("Dockerfile", None)?).await?; + // A setting that names a file the repository doesn't have fails, + // whatever was built before. + let e = host + .image(&spec("Containerfile", None)?) + .await + .err() + .map(|e| format!("{e:#}")); + assert_eq!( + e.as_deref(), + Some("Containerfile isn't in acme/app on main (Settings → Sandbox → Image)") + ); + // While GitHub is down, the last build of the same setting, and of + // no other. + github.change(|r| r.down = true); + assert_eq!(host.image(&spec("Dockerfile", None)?).await?, tag); + let other = host.image(&spec("Dockerfile", Some("app"))?).await; + assert!( + other + .as_ref() + .is_err_and(|e| format!("{e:#}").contains("GitHub answered 503")), + "{other:?}" + ); + Ok(()) + } + + #[tokio::test] + async fn builds_use_the_dockerfile_github_sends_not_a_link() -> Result<()> { + let github = + FakeGithub::start(&[("Dockerfile", "FROM debian\n"), ("app.txt", "x")]).await?; + github.change(|r| { + r.links.insert("Dockerfile".into(), "/etc/hostname".into()); + }); + let host = Host::new(&github)?; + host.image(&spec("Dockerfile", None)?).await?; + assert_eq!( + std::fs::read_to_string(host.cli.dir.path().join("dockerfile"))?, + "FROM debian\n" + ); + Ok(()) + } + + #[tokio::test] + async fn builds_that_fail_or_grow_too_big_start_no_session() -> Result<()> { + let github = FakeGithub::start(&[("Dockerfile", "FROM debian\n")]).await?; + let host = Host::new(&github)?; + // A file the repository doesn't have. + let e = host + .image(&spec("Containerfile", None)?) + .await + .err() + .map(|e| format!("{e:#}")); + assert_eq!( + e.as_deref(), + Some("Containerfile isn't in acme/app on main (Settings → Sandbox → Image)") + ); + // A build that fails says why. + std::fs::write(host.cli.dir.path().join("fail_build"), "")?; + let e = host + .image(&spec("Dockerfile", None)?) + .await + .err() + .map(|e| format!("{e:#}")); + assert!( + e.as_deref().is_some_and( + |e| e.starts_with("Building the image failed (exit status: 1): #3 ERROR:") + ), + "{e:?}" + ); + std::fs::remove_file(host.cli.dir.path().join("fail_build"))?; + // An image larger than the host keeps is removed. + std::fs::write( + host.cli.dir.path().join("size"), + format!("{}", 30_u64 << 30), + )?; + let e = host + .image(&spec("Dockerfile", None)?) + .await + .err() + .map(|e| format!("{e:#}")); + assert_eq!( + e.as_deref(), + Some( + "The image is 30 GiB, more than the 20 GiB this host keeps (max_build_mb in host.toml)" + ) + ); + assert!( + host.cli + .calls() + .iter() + .any(|c| c.starts_with("image rm fctl-build-prj_test1:")) + ); + // A dev container that names an image is that image, unbuilt. + github.change(|r| { + r.files.insert( + ".devcontainer/devcontainer.json".into(), + "{ // An image\n \"image\": \"mcr.microsoft.com/devcontainers/rust:1\", }".into(), + ); + }); + let image = host + .image(&spec(".devcontainer/devcontainer.json", None)?) + .await?; + assert_eq!(image, "mcr.microsoft.com/devcontainers/rust:1"); + Ok(()) + } +} diff --git a/crates/fctl-host/src/config.rs b/crates/fctl-host/src/config.rs index 55c63ad4..06d4f140 100644 --- a/crates/fctl-host/src/config.rs +++ b/crates/fctl-host/src/config.rs @@ -10,6 +10,12 @@ //! memory_mb = 8192 # per sandbox, when its project sets none //! max_cpus = 8 # per sandbox (default: what the machine has) //! max_memory_mb = 16384 # per sandbox (default: what the machine has) +//! # Images projects may choose (unset: any): references, or prefixes in /*. +//! allowed_images = ["ghcr.io/hyperstructure/sandbox", "ghcr.io/acme/*"] +//! # Repositories whose Dockerfile or devcontainer.json it builds (unset: none). +//! allowed_builds = ["acme/*"] +//! build_timeout_minutes = 60 +//! max_build_mb = 20480 # the largest image a build may make //! ``` //! //! The runner token lives next to it in `host-token` (mode 0600). @@ -23,10 +29,6 @@ use std::path::{Path, PathBuf}; /// https://github.com/hyperstructure/sandbox: a tag for every release. pub const IMAGE_REPOSITORY: &str = "ghcr.io/hyperstructure/sandbox"; -/// The default image before it was published, which hosts installed until -/// then have in host.toml. -const OLD_DEFAULT_IMAGE: &str = "fctl-agent:latest"; - /// The published sandbox image of this version of fctl. pub fn default_image() -> String { format!("{IMAGE_REPOSITORY}:{}", crate::agentbin::VERSION) @@ -35,8 +37,7 @@ pub fn default_image() -> String { /// Whether an image setting, the host's or a project's, means the host's /// default: empty, or the name the default had before it was published. pub fn means_default_image(image: &str) -> bool { - let image = image.trim(); - image.is_empty() || image == OLD_DEFAULT_IMAGE + fctl_domain::image::means_default(image) } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -70,10 +71,26 @@ pub struct HostConfig { pub agent_bin: Option, /// Where releases are downloaded from. pub download_base: String, + /// The GitHub API, which builds read repositories from (for tests). + #[serde(skip_serializing_if = "is_github")] + pub github_api: String, /// Push new commits after each turn. pub push_on_commit: bool, /// Keep a stopped session's sandbox (to resume it) for this many hours. pub keep_stopped_hours: u32, + /// The images this host runs for projects that choose their own, + /// besides its own `image`: references (`ghcr.io/acme/agent:1`, pinned + /// with `@sha256:…` or not, or without a tag for any), or prefixes that + /// end in `/*` (`ghcr.io/acme/*`). Unset: any image. + pub allowed_images: Option>, + /// The repositories (`owner/name`, or `owner/*` for an account's) whose + /// Dockerfile or devcontainer.json this host builds sandbox images from, + /// for projects that build theirs. Unset: none. + pub allowed_builds: Option>, + /// Minutes one image build may take. + pub build_timeout_minutes: u32, + /// The largest image a build may make, in MiB. + pub max_build_mb: u32, } impl Default for HostConfig { @@ -94,12 +111,23 @@ impl Default for HostConfig { claude_args: vec![], agent_bin: None, download_base: "https://fctl.sh/releases".into(), + github_api: GITHUB_API.into(), push_on_commit: true, keep_stopped_hours: 72, + allowed_images: None, + allowed_builds: None, + build_timeout_minutes: 60, + max_build_mb: 20 * 1024, } } } +const GITHUB_API: &str = "https://api.github.com"; + +fn is_github(url: &String) -> bool { + url == GITHUB_API +} + pub fn hostname() -> String { std::env::var("FCTL_HOSTNAME") .ok() @@ -224,6 +252,36 @@ impl HostConfig { .then(|| format!("{IMAGE_REPOSITORY}:latest")) } + /// The images and builds this host's sessions may start from. Builds + /// need a `driver` that makes sandboxes from images. + pub fn images(&self, driver_uses_images: bool) -> Images { + Images { + own: self.image(), + allowed: self.allowed_images.clone(), + builds: if driver_uses_images { + self.allowed_builds.clone() + } else { + None + }, + } + } + + /// What is wrong with `allowed_images` and `allowed_builds`: entries + /// that match nothing. + pub fn image_problems(&self) -> Vec { + let images = self.allowed_images.iter().flatten(); + let builds = self.allowed_builds.iter().flatten(); + images + .filter_map(|p| fctl_domain::image::check_image_pattern(p)) + .map(|e| format!("allowed_images: {e}")) + .chain( + builds + .filter_map(|p| fctl_domain::image::check_build_pattern(p)) + .map(|e| format!("allowed_builds: {e}")), + ) + .collect() + } + /// What this host gives each sandbox: the configured defaults and /// maximums, within `runtime`, what the sandboxes run on has (0: unknown). pub fn sandbox_limits(&self, runtime: SandboxSize) -> SandboxLimits { @@ -247,6 +305,27 @@ impl HostConfig { } } +/// The images a host's sessions may start from, as its hello says. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Images { + /// The image for sessions whose project sets none. + pub own: String, + /// `allowed_images`: `None` for any image. + pub allowed: Option>, + /// `allowed_builds`, when the driver can build: `None` for none. + pub builds: Option>, +} + +impl Images { + pub fn policy(&self) -> fctl_domain::image::HostImages<'_> { + fctl_domain::image::HostImages { + own: Some(&self.own), + allowed: self.allowed.as_deref(), + builds: self.builds.as_deref(), + } + } +} + pub fn write_private(path: &Path, contents: &str) -> Result<()> { let tmp = path.with_extension("tmp"); std::fs::write(&tmp, contents)?; @@ -292,6 +371,35 @@ mod tests { Ok(()) } + #[test] + fn hosts_say_which_images_they_run_and_build() -> Result<()> { + let cfg = HostConfig::from_toml( + "allowed_images = [\"ghcr.io/acme/*\", \"ghcr.io/acme*\"]\nallowed_builds = [\"acme/*\", \"acme\"]", + )?; + let images = cfg.images(true); + assert_eq!(images.own, default_image()); + assert!(images.policy().runs("ghcr.io/acme/agent:1")); + assert!(images.policy().runs(&default_image())); + assert!(!images.policy().runs("docker.io/library/ubuntu")); + assert_eq!(images.builds.as_deref().map(<[String]>::len), Some(2)); + // Lima and the process driver build nothing. + assert_eq!(cfg.images(false).builds, None); + assert_eq!( + cfg.image_problems(), + vec![ + "allowed_images: `ghcr.io/acme*`: a prefix ends in /*, like ghcr.io/acme/*", + "allowed_builds: `acme` is not owner/name or owner/*", + ] + ); + // Unset: any image, and no builds; host.toml leaves them out. + let cfg = HostConfig::default(); + assert!(cfg.images(true).policy().runs("docker.io/library/ubuntu")); + assert_eq!(cfg.images(true).builds, None); + let saved = toml::to_string_pretty(&cfg)?; + assert!(!saved.contains("allowed_"), "{saved}"); + Ok(()) + } + #[test] fn sandboxes_get_at_most_what_the_runtime_has() { let docker_vm = size(10, 15_990); diff --git a/crates/fctl-host/src/daemon.rs b/crates/fctl-host/src/daemon.rs index 67e2326c..cb5b7d88 100644 --- a/crates/fctl-host/src/daemon.rs +++ b/crates/fctl-host/src/daemon.rs @@ -17,6 +17,7 @@ use crate::session::{self, HostCtx, SessionCmd, SessionEvent, SessionHandle, Sta use crate::sysinfo; use anyhow::Result; use fctl_client::ws; +use fctl_domain::image::ImageWant; use fctl_domain::size::{self, Want}; use fctl_proto::runner::{ FleetToHost, HostHeartbeat, HostHello, HostSessionReport, HostToFleet, SessionSpec, StopReason, @@ -80,6 +81,8 @@ impl Daemon { pub async fn new(cfg: HostConfig, token: String) -> Result { let data = cfg.data_dir(); std::fs::create_dir_all(&data)?; + // What image builds left behind when the host last stopped. + let _ = std::fs::remove_dir_all(data.join("builds")); let driver = Driver::select(&cfg.driver, &data, cfg.agent_bin.clone()).await?; let ctx = Arc::new(HostCtx::new(cfg, driver, token)); ctx.refresh_facts().await; @@ -303,6 +306,7 @@ impl Daemon { } fn hello(&self) -> HostHello { + let images = self.ctx.images(); HostHello { protocol: PROTOCOL_VERSION, version: VERSION.into(), @@ -316,8 +320,10 @@ impl Daemon { capacity: self.ctx.cfg.capacity, labels: self.ctx.cfg.labels.clone(), sessions: self.reports(), - image: Some(self.ctx.cfg.image()), + image: Some(images.own), sandbox: Some(self.ctx.sandbox_limits()), + allowed_images: images.allowed, + allowed_builds: images.builds, } } @@ -572,6 +578,21 @@ impl Daemon { size::memory(limits.max.memory_mb.into()) )); } + let want = ImageWant::of(&spec.project.settings, spec.project.repo.as_ref()); + if !self.ctx.images().policy().takes(&want) { + return Err(match want { + ImageWant::Build(repo) => format!( + "{} builds no images from {} (allowed_builds in its host.toml)", + self.ctx.cfg.name, + repo.full_name() + ), + ImageWant::Image(image) => format!( + "{} doesn't run {image} (allowed_images in its host.toml)", + self.ctx.cfg.name + ), + ImageWant::Default => format!("{} can't take the session", self.ctx.cfg.name), + }); + } if let Some(existing) = self.sessions.get(&id) { let winding_down = existing .report diff --git a/crates/fctl-host/src/driver/container.rs b/crates/fctl-host/src/driver/container.rs index efece936..c43fe9a4 100644 --- a/crates/fctl-host/src/driver/container.rs +++ b/crates/fctl-host/src/driver/container.rs @@ -8,14 +8,16 @@ //! fctl-agent attach`. use super::{ - AGENT_PATH, AgentIo, COMMAND_TIMEOUT, SLOW_COMMAND_TIMEOUT, Sandbox, SandboxInfo, SandboxSpec, - TimedOut, attach_script, run_cmd_within, serve_script, timed_out, + AGENT_PATH, AgentIo, BuildJob, COMMAND_TIMEOUT, SLOW_COMMAND_TIMEOUT, Sandbox, SandboxInfo, + SandboxSpec, TimedOut, attach_script, run_cmd_within, serve_script, timed_out, }; use anyhow::{Context, Result}; use fctl_proto::runner::{DriverInfo, SandboxSize}; use serde_json::Value; +use std::collections::VecDeque; use std::path::Path; use std::process::Stdio; +use tokio::io::{AsyncBufReadExt, AsyncRead, BufReader}; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Flavor { @@ -267,6 +269,119 @@ impl ContainerDriver { self.run(&args).await.map(drop).map_err(failed) } + /// Whether the sandbox `name` exists. An error only when the runtime + /// doesn't answer. + pub async fn exists(&self, name: &str) -> Result { + Ok(self.state(name).await?.is_some()) + } + + /// Builds `job` with `docker build` or `container build`, saying in + /// `log` which step it is at. One still running after `job.limit` is + /// stopped. + pub async fn build( + &self, + job: &BuildJob<'_>, + log: &(dyn Fn(String) + Send + Sync), + ) -> Result<()> { + let mut args: Vec = vec![ + "build".into(), + "--tag".into(), + job.tag.into(), + "--file".into(), + job.dockerfile.display().to_string(), + ]; + for (k, v) in job.args { + args.extend(["--build-arg".into(), format!("{k}={v}")]); + } + if let Some(target) = job.target { + args.extend(["--target".into(), target.into()]); + } + args.push(job.context.display().to_string()); + let mut cmd = tokio::process::Command::new(&self.bin); + cmd.args(&args) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true); + if self.flavor == Flavor::Docker { + // Lines rather than a redrawn screen; other builders ignore it. + cmd.env("BUILDKIT_PROGRESS", "plain"); + } + let mut child = cmd.spawn().map_err(|e| super::spawn_error(&self.bin, e))?; + let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::(); + if let Some(out) = child.stdout.take() { + forward_lines(out, tx.clone()); + } + if let Some(err) = child.stderr.take() { + forward_lines(err, tx.clone()); + } + drop(tx); + let mut tail: VecDeque = VecDeque::new(); + let run = async { + while let Some(line) = rx.recv().await { + if build_step(&line) { + log(short(&line)); + } + if tail.len() == 20 { + tail.pop_front(); + } + tail.push_back(line); + } + child.wait().await + }; + let status = match tokio::time::timeout(job.limit, run).await { + Ok(status) => status.with_context(|| format!("running {} build", self.bin))?, + Err(_) => { + let _ = child.kill().await; + anyhow::bail!( + "The image build took longer than {} minutes, so the host stopped it (build_timeout_minutes in host.toml)", + job.limit.as_secs() / 60 + ); + } + }; + if !status.success() { + for line in &tail { + log(line.clone()); + } + let why = tail + .iter() + .rev() + .find(|l| l.contains("ERROR") || l.contains("error")) + .or(tail.back()) + .map(|l| format!(": {}", l.trim())) + .unwrap_or_default(); + anyhow::bail!("Building the image failed ({status}){why}"); + } + Ok(()) + } + + /// An image's size in bytes: Docker's, or with Apple's `container` the + /// size of its variant for this machine. + pub async fn image_size(&self, image: &str) -> Option { + match self.flavor { + Flavor::Docker => self + .run(&["image", "inspect", "--format", "{{.Size}}", image]) + .await + .ok()? + .trim() + .parse() + .ok(), + Flavor::Apple => { + let out = self.run(&["image", "inspect", image]).await.ok()?; + apple_image_size(&out, linux_arch(std::env::consts::ARCH)) + } + } + } + + /// Removes an image; fails while a sandbox uses it. + pub async fn remove_image(&self, image: &str) -> Result<()> { + let args: &[&str] = match self.flavor { + Flavor::Apple => &["image", "delete", image], + Flavor::Docker => &["image", "rm", image], + }; + self.run(args).await.map(drop) + } + pub async fn has_image(&self, image: &str) -> bool { let inspect: &[&str] = match self.flavor { Flavor::Apple => &["image", "inspect", image], @@ -451,6 +566,56 @@ impl ContainerDriver { } } +/// Sends each line `r` gives to `tx`. +fn forward_lines( + r: R, + tx: tokio::sync::mpsc::UnboundedSender, +) { + tokio::spawn(async move { + let mut lines = BufReader::new(r).lines(); + while let Ok(Some(line)) = lines.next_line().await { + if tx.send(line).is_err() { + break; + } + } + }); +} + +/// Whether a line of a build's output starts one of its steps, or says it +/// failed: BuildKit's `#5 [2/4] RUN …`, the older builder's `Step 2/4 :`, +/// Podman's `STEP 2/4:`. +fn build_step(line: &str) -> bool { + let l = line.trim_start(); + let buildkit = l.strip_prefix('#').is_some_and(|rest| { + let digits = rest.chars().take_while(char::is_ascii_digit).count(); + digits > 0 && rest[digits..].starts_with(" [") + }); + buildkit || l.starts_with("Step ") || l.starts_with("STEP ") || l.starts_with("ERROR") +} + +/// A line of a build's output, cut short when it is long. +fn short(line: &str) -> String { + let line = line.trim_end(); + match line.char_indices().nth(240) { + Some((cut, _)) => format!("{}…", &line[..cut]), + None => line.to_owned(), + } +} + +/// The size of an image's variant for `arch` in `container image inspect`'s +/// JSON, or of its largest one. +fn apple_image_size(json: &str, arch: &str) -> Option { + let v: Value = serde_json::from_str(json).ok()?; + let image = v.get(0).unwrap_or(&v); + let variants = image.get("variants")?.as_array()?; + let size = |v: &Value| v.get("size").and_then(Value::as_u64); + variants + .iter() + .find(|v| v.pointer("/platform/architecture").and_then(Value::as_str) == Some(arch)) + .and_then(size) + .or_else(|| variants.iter().filter_map(size).max()) +} + /// An architecture as images name it: `arm64` for Rust's `aarch64`. fn linux_arch(arch: &str) -> &str { match arch { @@ -479,39 +644,23 @@ fn installed(bin: &str) -> bool { } #[cfg(test)] -mod tests { +pub(crate) mod fake { use super::*; - - #[test] - fn reads_the_docker_vms_size() { - // OrbStack's VM in #14's session: 10 CPUs and 15.6 GiB. - assert_eq!( - docker_resources("10 16766767104\n"), - Some(SandboxSize { - cpus: 10, - memory_mb: 15_990 - }) - ); - for bad in [ - "", - "10", - "ten 1024", - "0 16766767104", - " ", - ] { - assert_eq!(docker_resources(bad), None, "{bad}"); - } - } + use anyhow::Result; + use std::sync::{Mutex, PoisonError}; /// A stand-in for the `docker` or `container` CLI with no sandboxes yet: /// `have` are the images on the host, `registry` those it can pull. It - /// writes each call to `calls`, and with `fail_run` every `run` fails. - struct FakeCli { - dir: tempfile::TempDir, + /// writes each call to `calls`; with `fail_run` every `run` fails, and with + /// `fail_build` every `build`. A build lists its context in `context` and + /// copies its Dockerfile to `dockerfile`; `size` holds the size images + /// have (1000 bytes without it). + pub struct FakeCli { + pub dir: tempfile::TempDir, } impl FakeCli { - fn new(have: &[&str], registry: &[&str]) -> Result { + pub fn new(have: &[&str], registry: &[&str]) -> Result { use std::os::unix::fs::PermissionsExt; let dir = tempfile::tempdir()?; let root = dir.path(); @@ -522,7 +671,7 @@ mod tests { } } let script = format!( - r#"#!/bin/sh + r##"#!/bin/sh d='{}' echo "$*" >> "$d/calls" key() {{ printf '%s' "$1" | tr '/:' '__'; }} @@ -532,7 +681,18 @@ pull() {{ touch "$d/have/$(key "$last")" }} case "$1 $2" in - "image inspect") [ -e "$d/have/$(key "$last")" ] ;; + "image inspect") + [ -e "$d/have/$(key "$last")" ] || exit 1 + [ "$4" != '{{{{.Size}}}}' ] || cat "$d/size" 2>/dev/null || echo 1000 ;; + "image rm" | "image delete") rm "$d/have/$(key "$last")" ;; + "build "*) + [ ! -e "$d/fail_build" ] || {{ echo "#3 ERROR: process did not complete successfully: exit code: 1" >&2; exit 1; }} + echo "#1 [internal] load build definition from Dockerfile" + echo "#5 [1/2] FROM debian" >&2 + echo "#5 0.1 noise" + (cd "$last" && find . | sort) > "$d/context" + cat "$5" > "$d/dockerfile" + touch "$d/have/$(key "$3")" ;; "image pull" | "pull "*) pull ;; "inspect "*) exit 1 ;; "run "*) @@ -541,7 +701,7 @@ case "$1 $2" in [ -e "$d/have/$(key "$2")" ] || {{ echo "Unable to find image '$2' locally" >&2; exit 125; }} ;; *) echo "unexpected: $*" >&2; exit 2 ;; esac -"#, +"##, root.display() ); let bin = root.join("cli"); @@ -550,14 +710,14 @@ esac Ok(Self { dir }) } - fn driver(&self, flavor: Flavor) -> ContainerDriver { + pub fn driver(&self, flavor: Flavor) -> ContainerDriver { ContainerDriver { flavor, bin: self.dir.path().join("cli").display().to_string(), } } - fn calls(&self) -> Vec { + pub fn calls(&self) -> Vec { std::fs::read_to_string(self.dir.path().join("calls")) .unwrap_or_default() .lines() @@ -566,7 +726,7 @@ esac } /// Creates the sandbox: the image it started from, and the log. - async fn ensure( + pub async fn ensure( &self, flavor: Flavor, spec: &SandboxSpec, @@ -595,6 +755,65 @@ esac )) } } +} + +#[cfg(test)] +mod tests { + use super::fake::FakeCli; + use super::*; + + #[test] + fn build_logs_show_the_steps() { + for step in [ + "#5 [2/4] RUN apt-get update", + "#12 [build 3/7] COPY . .", + "Step 2/4 : RUN make", + "STEP 2/4: RUN make", + "ERROR: failed to solve: process \"/bin/sh -c make\" did not complete successfully", + ] { + assert!(build_step(step), "{step}"); + } + for noise in [ + "#5 0.312 Get:1 http://deb.debian.org", + "#5 DONE 3.1s", + "", + "#x [1/2]", + ] { + assert!(!build_step(noise), "{noise}"); + } + } + + #[test] + fn reads_the_size_of_apples_images() { + let json = r#"[{"name":"fctl-build-prj_1:abc","index":{"size":1024},"variants":[ + {"platform":{"os":"linux","architecture":"amd64"},"size":900}, + {"platform":{"os":"linux","architecture":"arm64"},"size":800}]}]"#; + assert_eq!(apple_image_size(json, "arm64"), Some(800)); + assert_eq!(apple_image_size(json, "riscv64"), Some(900)); + assert_eq!(apple_image_size("[{\"name\":\"x\"}]", "arm64"), None); + assert_eq!(apple_image_size("not json", "arm64"), None); + } + + #[test] + fn reads_the_docker_vms_size() { + // OrbStack's VM in #14's session: 10 CPUs and 15.6 GiB. + assert_eq!( + docker_resources("10 16766767104\n"), + Some(SandboxSize { + cpus: 10, + memory_mb: 15_990 + }) + ); + for bad in [ + "", + "10", + "ten 1024", + "0 16766767104", + " ", + ] { + assert_eq!(docker_resources(bad), None, "{bad}"); + } + } use std::sync::{Mutex, PoisonError}; diff --git a/crates/fctl-host/src/driver/mod.rs b/crates/fctl-host/src/driver/mod.rs index cc7e19fd..c636bf55 100644 --- a/crates/fctl-host/src/driver/mod.rs +++ b/crates/fctl-host/src/driver/mod.rs @@ -39,6 +39,18 @@ pub struct SandboxSpec { pub memory_mb: u32, } +/// An image to build from a Dockerfile. +#[derive(Clone, Debug)] +pub struct BuildJob<'a> { + pub tag: &'a str, + pub dockerfile: &'a Path, + pub context: &'a Path, + pub args: &'a std::collections::BTreeMap, + pub target: Option<&'a str>, + /// How long the build may take. + pub limit: Duration, +} + /// A sandbox, as the host knows it. #[derive(Clone, Debug)] pub struct Sandbox { @@ -190,6 +202,42 @@ impl Driver { } } + /// Builds an image, saying in `log` what the build does. + pub async fn build( + &self, + job: &BuildJob<'_>, + log: &(dyn Fn(String) + Send + Sync), + ) -> Result<()> { + match self { + Self::Container(d) => d.build(job, log).await, + _ => anyhow::bail!("the {} driver does not use images", self.name()), + } + } + + /// An image's size in bytes, when the runtime says. + pub async fn image_size(&self, image: &str) -> Option { + match self { + Self::Container(d) => d.image_size(image).await, + _ => None, + } + } + + /// Removes an image; fails while a sandbox uses it. + pub async fn remove_image(&self, image: &str) -> Result<()> { + match self { + Self::Container(d) => d.remove_image(image).await, + _ => Ok(()), + } + } + + /// Whether the sandbox `name` exists, running or not. + pub async fn sandbox_exists(&self, name: &str) -> Result { + match self { + Self::Container(d) => d.exists(name).await, + _ => Ok(false), + } + } + pub async fn probe(&self) -> DriverInfo { match self { Self::Process(d) => d.probe().await, diff --git a/crates/fctl-host/src/lib.rs b/crates/fctl-host/src/lib.rs index 9d07afa3..620fa504 100644 --- a/crates/fctl-host/src/lib.rs +++ b/crates/fctl-host/src/lib.rs @@ -4,6 +4,7 @@ //! config and a launchd/systemd service. See docs/architecture.md. pub mod agentbin; +pub mod build; pub mod config; pub mod daemon; pub mod driver; diff --git a/crates/fctl-host/src/session.rs b/crates/fctl-host/src/session.rs index ed089558..26ba70e1 100644 --- a/crates/fctl-host/src/session.rs +++ b/crates/fctl-host/src/session.rs @@ -1,7 +1,7 @@ //! One task per session: sandbox, fctl-agent, and the bridge between the //! agent and the session's uplink. -use crate::config::{HostConfig, means_default_image}; +use crate::config::{HostConfig, Images}; use crate::driver::{AgentIo, Driver, Sandbox, SandboxSpec, sandbox_name}; use crate::files; use crate::liveness::{Check, Health, Reach, Reached, Retry}; @@ -9,6 +9,7 @@ use crate::uplink::Uplink; use anyhow::{Context, Result}; use base64::Engine; use base64::engine::general_purpose::STANDARD; +use fctl_domain::image::ImageWant; use fctl_domain::size::Want; use fctl_proto::agent::{AgentConfig, AgentToHost, HostToAgent, MAX_FRAME_BYTES}; use fctl_proto::event::{LogLevel, LogLine, LogSource, PublishedFile}; @@ -22,6 +23,7 @@ use fctl_proto::uplink::{ UplinkUp, }; use fctl_proto::{ApiError, ErrorCode, Millis, RunnerId, SessionId}; +use std::collections::HashMap; use std::sync::{Arc, Mutex, PoisonError}; use std::time::{Duration, Instant}; use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; @@ -37,6 +39,13 @@ pub struct HostCtx { pub sandbox: Mutex, /// The drivers this host could use, as last probed. pub drivers: Mutex>, + /// Held while an image builds: one build at a time. + pub builds: tokio::sync::Mutex<()>, + /// One per image being built: sessions that need the same one wait for + /// the session that builds it. + pub build_locks: Mutex>>>, + /// Held while `builds.json` changes. + pub build_record: Mutex<()>, } impl HostCtx { @@ -51,9 +60,27 @@ impl HostCtx { runner_id: Mutex::new(None), sandbox: Mutex::new(sandbox), drivers: Mutex::new(Vec::new()), + builds: tokio::sync::Mutex::new(()), + build_locks: Mutex::new(HashMap::new()), + build_record: Mutex::new(()), } } + /// The lock of the build of `tag`, shared by the sessions that need it. + pub fn build_lock(&self, tag: &str) -> Arc> { + let mut locks = self + .build_locks + .lock() + .unwrap_or_else(PoisonError::into_inner); + locks.retain(|_, lock| Arc::strong_count(lock) > 1); + locks.entry(tag.to_owned()).or_default().clone() + } + + /// The images and builds this host's sessions may start from. + pub fn images(&self) -> Images { + self.cfg.images(self.driver.uses_images()) + } + pub fn runner_id(&self) -> RunnerId { self.runner_id .lock() @@ -436,19 +463,31 @@ fn wall_ms(t: Instant) -> Millis { } async fn prepare(ctx: &HostCtx, spec: &SessionSpec, uplink: &Uplink) -> Result { - let (image, fallback_image) = match spec.project.settings.image.as_deref() { - Some(image) if !means_default_image(image) => (image.to_owned(), None), + let name = sandbox_name(spec.session_id.as_str()); + let log_uplink = |line: String| host_log(uplink, LogLevel::Info, line); + let want = ImageWant::of(&spec.project.settings, spec.project.repo.as_ref()); + let (image, fallback_image) = match want { + // A sandbox that is here already keeps the image it was made from. + ImageWant::Build(_) + if ctx.driver.uses_images() && !ctx.driver.sandbox_exists(&name).await? => + { + uplink.state(state( + SessionStatus::Provisioning, + &format!("Getting the sandbox image ready on {}", ctx.cfg.name), + )); + (crate::build::image(ctx, spec, &log_uplink).await?, None) + } + ImageWant::Image(image) => (image, None), _ => (ctx.cfg.image(), ctx.cfg.fallback_image()), }; let size = ctx.sandbox_size(spec); let sspec = SandboxSpec { - name: sandbox_name(spec.session_id.as_str()), + name, image, fallback_image, cpus: size.cpus, memory_mb: size.memory_mb, }; - let log_uplink = |line: String| host_log(uplink, LogLevel::Info, line); let sb = ctx.driver.ensure(&sspec, &log_uplink).await?; if ctx.driver.needs_linux_agent() { let bin = crate::agentbin::linux_agent(&ctx.cfg) diff --git a/crates/fctl-proto/src/api.rs b/crates/fctl-proto/src/api.rs index c270f517..1397a9e7 100644 --- a/crates/fctl-proto/src/api.rs +++ b/crates/fctl-proto/src/api.rs @@ -541,6 +541,15 @@ pub struct Runner { pub disk_available_mb: Option, /// What it gives each sandbox (absent from older hosts). pub sandbox: Option, + /// The image it gives sessions whose project sets none. + pub image: Option, + /// The other images it runs, as its hello says ([`HostHello`]). + /// Absent: any image. + /// + /// [`HostHello`]: crate::runner::HostHello + pub allowed_images: Option>, + /// The repositories it builds images from. Absent: none. + pub allowed_builds: Option>, pub labels: Vec, pub last_seen_at: Option, pub created_at: Millis, diff --git a/crates/fctl-proto/src/runner.rs b/crates/fctl-proto/src/runner.rs index 1bdd248c..72b7bb83 100644 --- a/crates/fctl-proto/src/runner.rs +++ b/crates/fctl-proto/src/runner.rs @@ -86,6 +86,14 @@ pub struct HostHello { /// What the host gives each sandbox. Older hosts send none and get /// sessions of any size. pub sandbox: Option, + /// The images it runs for projects that choose their own, besides + /// `image`: references, or prefixes ending in `/*`. Absent, as from + /// older hosts: any image. + pub allowed_images: Option>, + /// The repositories (`owner/name`, `owner/*`) whose files it builds + /// images from, for projects that build theirs. Absent, as from older + /// hosts and hosts whose driver uses no images: none. + pub allowed_builds: Option>, } #[skip_serializing_none] diff --git a/crates/fctl-proto/src/session.rs b/crates/fctl-proto/src/session.rs index 061b3af1..fe3ae6b6 100644 --- a/crates/fctl-proto/src/session.rs +++ b/crates/fctl-proto/src/session.rs @@ -345,6 +345,30 @@ impl RepoGrant { } } +/// A sandbox image built from the project's repository, as its default +/// branch has it. Hosts build it again when the files it comes from change. +#[skip_serializing_none] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS), ts(optional_fields))] +pub struct ImageBuild { + /// A Dockerfile, or a dev container configuration (a file ending in + /// `.json`, like `.devcontainer/devcontainer.json`), by its path in the + /// repository. + pub file: String, + /// For a Dockerfile, the directory in the repository it builds in. + /// Absent: the repository's root. A dev container configuration names + /// its own. + pub context: Option, +} + +impl ImageBuild { + /// Whether `file` is a dev container configuration rather than a + /// Dockerfile. + pub fn is_devcontainer(&self) -> bool { + self.file.to_ascii_lowercase().ends_with(".json") + } +} + /// Settings agents use for a project. Every field has a sensible default, so /// clients send only what they change. #[skip_serializing_none] @@ -354,6 +378,14 @@ impl RepoGrant { pub struct ProjectSettings { /// OCI image for sandboxes. Defaults to the host's configured agent image. pub image: Option, + /// Build the sandbox image from a file in the project's repository + /// instead, on hosts whose owners allow it (`allowed_builds`). It takes + /// the place of `image`. Older hosts and control planes ignore it. + /// Always sent, as `null` for none: an update without it keeps the + /// stored build (see `fctl_domain::settings::keep_unsent`). + #[serialize_always] + #[cfg_attr(feature = "ts", ts(optional = nullable))] + pub image_build: Option, /// Shell script run in the repository after cloning (for example `npm ci`). pub setup_script: Option, /// Claude model alias or name (`opus`, `sonnet`, …). Defaults to Claude's own default. @@ -400,6 +432,7 @@ impl Default for ProjectSettings { fn default() -> Self { Self { image: None, + image_build: None, setup_script: None, model: None, effort: None, diff --git a/crates/fctl/src/cmd/admin.rs b/crates/fctl/src/cmd/admin.rs index 788dc0b2..13f7d44b 100644 --- a/crates/fctl/src/cmd/admin.rs +++ b/crates/fctl/src/cmd/admin.rs @@ -191,6 +191,16 @@ pub async fn run(server: Option<&str>, cmd: AdminCmd, json: bool) -> Result<()> style::dim(&format!("sandboxes: {}", size::limits(l))) ); } + if let Some(own) = &r.image { + println!( + " {}", + style::dim(&format!( + "image: {own}; {}; {}", + fctl_domain::image::describe_allowed(r.allowed_images.as_deref()), + fctl_domain::image::describe_builds(r.allowed_builds.as_deref()) + )) + ); + } let free: Vec = [ r.memory_available_mb .map(|m| format!("{} of memory", size::memory(m))), diff --git a/crates/fctl/src/cmd/host.rs b/crates/fctl/src/cmd/host.rs index 430e0b59..f71fc798 100644 --- a/crates/fctl/src/cmd/host.rs +++ b/crates/fctl/src/cmd/host.rs @@ -5,7 +5,7 @@ use crate::config; use crate::style; use anyhow::{Context, Result, bail}; use clap::Subcommand; -use fctl_domain::size; +use fctl_domain::{image, size}; use fctl_host::config::means_default_image; use fctl_host::driver::Driver; use fctl_host::{HostConfig, service}; @@ -41,6 +41,17 @@ pub enum HostCmd { /// published image of this version; "" goes back to it). #[arg(long)] image: Option, + /// An image this host runs for projects that choose their own: a + /// reference, or a prefix like ghcr.io/acme/* (repeat it for more; + /// it replaces the list, and "*" allows any). Without the list, any + /// image runs. + #[arg(long = "allow-image", value_name = "REF")] + allow_images: Vec, + /// A repository, owner/name or owner/*, whose Dockerfile or + /// devcontainer.json this host builds images from (repeat it for + /// more; it replaces the list, and "" builds none). + #[arg(long = "allow-build", value_name = "REPO")] + allow_builds: Vec, /// Use an existing runner token instead of registering a new runner. #[arg(long, env = "FCTL_RUNNER_TOKEN", hide_env_values = true)] token: Option, @@ -126,6 +137,8 @@ pub async fn run(server: Option<&str>, cmd: HostCmd, json: bool) -> Result<()> { max_cpus, max_memory, image, + allow_images, + allow_builds, token, no_service, build_image, @@ -145,6 +158,25 @@ pub async fn run(server: Option<&str>, cmd: HostCmd, json: bool) -> Result<()> { if let Some(i) = image { cfg.image = (!means_default_image(&i)).then_some(i); } + if !allow_images.is_empty() { + let list: Vec = allow_images + .iter() + .map(|i| i.trim().to_owned()) + .filter(|i| !i.is_empty()) + .collect(); + cfg.allowed_images = (!list.iter().any(|i| i == "*")).then_some(list); + } + if !allow_builds.is_empty() { + let list: Vec = allow_builds + .iter() + .map(|r| r.trim().to_owned()) + .filter(|r| !r.is_empty()) + .collect(); + cfg.allowed_builds = (!list.is_empty()).then_some(list); + } + if let Some(problem) = cfg.image_problems().first() { + bail!("{problem}"); + } // Registering a runner leaves something behind, so it waits until // everything that can fail on this machine has passed. let api = if token.is_some() { @@ -228,6 +260,15 @@ pub async fn run(server: Option<&str>, cmd: HostCmd, json: bool) -> Result<()> { cfg.image() )) ); + let images = cfg.images(true); + println!( + "{}", + style::dim(&format!( + "It {} and {}.", + image::describe_allowed(images.allowed.as_deref()), + image::describe_builds(images.builds.as_deref()) + )) + ); } else if !build_image && d.needs_linux_agent() { println!( "{}", @@ -275,6 +316,14 @@ pub async fn run(server: Option<&str>, cmd: HostCmd, json: bool) -> Result<()> { println!("sandbox: {}", size::limits(l)); } println!("image: {}", cfg.image()); + if let Ok(d) = Driver::select(&cfg.driver, &cfg.data_dir(), None).await { + let images = cfg.images(d.uses_images()); + println!( + "images: {}; {}", + image::describe_allowed(images.allowed.as_deref()), + image::describe_builds(images.builds.as_deref()) + ); + } println!( "service: {}", match (installed, running) { @@ -749,6 +798,29 @@ async fn doctor() -> Result<()> { ); } } + let problems_with_images = cfg.image_problems(); + check( + problems_with_images.is_empty(), + &format!( + "images: {}; {}", + image::describe_allowed(cfg.allowed_images.as_deref()), + image::describe_builds(cfg.allowed_builds.as_deref()) + ), + &problems_with_images.join("; "), + ); + if cfg.allowed_builds.is_some() + && let Ok(d) = Driver::select(&cfg.driver, &cfg.data_dir(), None).await + && !d.uses_images() + { + println!( + "{} {}", + style::dim("·"), + style::dim(&format!( + "allowed_builds: the {} driver uses no images, so this host builds none", + d.name() + )) + ); + } let (installed, running) = service::status(); check( installed && running, diff --git a/crates/fctl/src/cmd/projects.rs b/crates/fctl/src/cmd/projects.rs index 7b1dd72e..e9e9ffbf 100644 --- a/crates/fctl/src/cmd/projects.rs +++ b/crates/fctl/src/cmd/projects.rs @@ -2,14 +2,14 @@ use crate::config; use crate::style; -use anyhow::{Result, bail}; +use anyhow::{Context, Result, bail}; use clap::builder::BoolishValueParser; use clap::{ArgGroup, Subcommand}; use fctl_proto::api::{ AddProjectRepoRequest, CreateProjectRequest, CreateRepoRequest, ProjectRepoList, UpdateProjectRequest, }; -use fctl_proto::session::{ProjectSettings, RepoAccess}; +use fctl_proto::session::{ImageBuild, ProjectSettings, RepoAccess}; use std::io::{BufRead, IsTerminal, Write}; #[derive(Subcommand)] @@ -46,8 +46,8 @@ pub enum ProjectsCmd { /// Change a project's settings (maintainers). Sessions take a change /// when they next start or resume. #[command( - group(ArgGroup::new("change").required(true).multiple(true).args(["credit_people", "credit_claude", "idle_timeout", "publish_screenshots", "screenshot_days"])), - after_help = "Examples:\n fctl projects set --credit-claude off # no Claude trailer\n fctl projects set -p my-app --credit-people off --credit-claude off # no trailers at all\n fctl projects set --idle-timeout 30 # idle sessions keep their sandbox for 30 minutes\n fctl projects set --publish-screenshots on --screenshot-days 7" + group(ArgGroup::new("change").required(true).multiple(true).args(["credit_people", "credit_claude", "idle_timeout", "publish_screenshots", "screenshot_days", "image", "build", "build_context", "setup_script", "env", "unset_env", "cpus", "memory"])), + after_help = "Examples:\n fctl projects set --credit-claude off # no Claude trailer\n fctl projects set -p my-app --credit-people off --credit-claude off # no trailers at all\n fctl projects set --idle-timeout 30 # idle sessions keep their sandbox for 30 minutes\n fctl projects set --publish-screenshots on --screenshot-days 7\n fctl projects set --image ghcr.io/acme/agent:1 # sessions start from this image\n fctl projects set --build .devcontainer/devcontainer.json # or one built from the repository\n fctl projects set --setup-script @setup.sh --env NODE_ENV=test\n fctl projects set --cpus 8 --memory 16g" )] Set { /// The project (slug or id); default from `fctl projects use`. @@ -77,6 +77,37 @@ pub enum ProjectsCmd { /// Days a published screenshot's link works, 1 to 60 (default 30). #[arg(long, value_name = "DAYS", value_parser = clap::value_parser!(u32).range(1..=60))] screenshot_days: Option, + /// The image sessions start from, like ghcr.io/acme/agent:1, on + /// runners whose owners allow it ("" for each runner's own). Only + /// the Apple container and Docker drivers use images. + #[arg(long, value_name = "REF", conflicts_with = "build")] + image: Option, + /// Build the image from this Dockerfile or devcontainer.json in the + /// project's repository, as its default branch has it ("" to stop), + /// on runners whose owners allow builds from it. + #[arg(long, value_name = "FILE")] + build: Option, + /// The directory in the repository the Dockerfile builds in + /// (default: its root; "" goes back to it). + #[arg(long, value_name = "DIR")] + build_context: Option, + /// The script that runs in the repository after cloning: the script + /// itself, @FILE to read it from a file, - from stdin, or "" for none. + #[arg(long, value_name = "SCRIPT")] + setup_script: Option, + /// Set an environment variable in the sandboxes (repeatable). Not + /// secret: put credentials in the project's secrets. + #[arg(long, value_name = "KEY=VALUE")] + env: Vec, + /// Remove an environment variable (repeatable). + #[arg(long, value_name = "KEY")] + unset_env: Vec, + /// CPUs per sandbox, 1 to 64 ("default" for each runner's default). + #[arg(long, value_name = "N|default", value_parser = cpus_arg)] + cpus: Option, + /// Memory per sandbox, like 16g ("default" for each runner's default). + #[arg(long, value_name = "SIZE|default", value_parser = memory_arg)] + memory: Option, }, /// Use this project by default for `fctl run` and `fctl sessions`. Use { @@ -183,6 +214,89 @@ fn credit(s: &ProjectSettings) -> &'static str { } } +/// A number, or `None` for `default`: each runner's own. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct OrDefault(Option); + +/// `--cpus`: a number, or `default`. +fn cpus_arg(s: &str) -> std::result::Result { + if s.trim().eq_ignore_ascii_case("default") { + return Ok(OrDefault(None)); + } + match s.trim().parse::() { + Ok(n) if (1..=64).contains(&n) => Ok(OrDefault(Some(n))), + _ => Err("use 1 to 64 CPUs, or default".into()), + } +} + +/// `--memory`: an amount like 16g, or `default`. +fn memory_arg(s: &str) -> std::result::Result { + if s.trim().eq_ignore_ascii_case("default") { + return Ok(OrDefault(None)); + } + fctl_domain::size::parse_memory_mb(s).map(|m| OrDefault(Some(m))) +} + +/// `--setup-script`: the script, `@FILE`, `-` for stdin; `""` for none. +fn setup_script(arg: &str) -> Result> { + let text = if arg == "-" { + std::io::read_to_string(std::io::stdin())? + } else if let Some(path) = arg.strip_prefix('@') { + std::fs::read_to_string(path).with_context(|| format!("reading {path}"))? + } else { + arg.to_owned() + }; + Ok((!text.trim().is_empty()).then_some(text)) +} + +/// Where a project's sessions start from, in words. +fn image(s: &ProjectSettings, repo: Option<&str>) -> String { + match (&s.image_build, s.image.as_deref()) { + (Some(b), _) => { + let from = repo.unwrap_or("the project's repository"); + let context = match &b.context { + Some(c) if !b.is_devcontainer() => format!(" (in {c}/)"), + _ => String::new(), + }; + format!( + "an image built from {from}'s {}{context}, on runners that build images from {from}", + b.file + ) + } + (None, Some(i)) if !fctl_domain::image::means_default(i) => { + format!("{i}, on runners that allow it") + } + _ => "each runner's own image".into(), + } +} + +/// A sandbox's size as a project sets it, in words. +fn sandbox(s: &ProjectSettings) -> String { + let cpus = s.cpus.map(fctl_domain::size::cpus); + let memory = s + .memory_mb + .map(|m| format!("{} of memory", fctl_domain::size::memory(m.into()))); + match (cpus, memory) { + (None, None) => "each runner's default size".into(), + (Some(c), None) => format!("{c} and the runner's default memory"), + (None, Some(m)) => format!("the runner's default CPUs and {m}"), + (Some(c), Some(m)) => format!("{c} and {m}"), + } +} + +/// The first line of a setup script, and how many more it has. +fn script_summary(script: &str) -> String { + let lines: Vec<&str> = script.lines().filter(|l| !l.trim().is_empty()).collect(); + match lines.as_slice() { + [] => "none".into(), + [one] => one.trim().to_owned(), + [first, rest @ ..] => { + let more = if rest.len() == 1 { "line" } else { "lines" }; + format!("{} (and {} more {more})", first.trim(), rest.len()) + } + } +} + async fn repos(api: &fctl_client::Api, cmd: ReposCmd, json: bool) -> Result<()> { let list = match cmd { ReposCmd::List { project } => { @@ -302,6 +416,17 @@ pub async fn run(server: Option<&str>, cmd: ProjectsCmd, json: bool) -> Result<( println!("also: {}", others.join(", ")); } println!("branch: {}", p.default_branch); + let repo = p.repo.as_ref().map(|r| r.full_name()); + println!("image: {}", image(&p.settings, repo.as_deref())); + println!("size: {}", sandbox(&p.settings)); + println!( + "setup: {}", + script_summary(p.settings.setup_script.as_deref().unwrap_or_default()) + ); + if !p.settings.env.is_empty() { + let keys: Vec<&str> = p.settings.env.keys().map(String::as_str).collect(); + println!("env: {}", keys.join(", ")); + } println!("idle: sessions {}", idle(&p.settings)); println!("role: {}", p.role.map(|r| r.as_str()).unwrap_or("—")); println!("{}", style::dim(&format!("{}/p/{}", api.base(), p.slug))); @@ -350,11 +475,65 @@ pub async fn run(server: Option<&str>, cmd: ProjectsCmd, json: bool) -> Result<( idle_timeout, publish_screenshots, screenshot_days, + image: new_image, + build, + build_context, + setup_script: script, + env, + unset_env, + cpus, + memory, } => { let project = super::sessions::project_or_default(project)?; let p = api.project(&project).await?; // An update replaces all settings: send the others as they are. let mut settings = p.settings.clone(); + if let Some(i) = &new_image { + settings.image = (!i.trim().is_empty()).then(|| i.trim().to_owned()); + settings.image_build = None; + } + if let Some(file) = &build { + settings.image_build = (!file.trim().is_empty()).then(|| { + let file = file.trim().to_owned(); + // A devcontainer.json says where it builds. + let context = settings + .image_build + .as_ref() + .and_then(|b| b.context.clone()) + .filter(|_| !file.to_ascii_lowercase().ends_with(".json")); + ImageBuild { file, context } + }); + if settings.image_build.is_some() { + settings.image = None; + } + } + if let Some(dir) = &build_context { + let Some(b) = settings.image_build.as_mut() else { + bail!( + "{} builds no image: name its Dockerfile with --build", + p.slug + ); + }; + b.context = (!dir.trim().is_empty()).then(|| dir.trim().to_owned()); + } + if let Some(arg) = &script { + settings.setup_script = setup_script(arg)?; + } + for pair in &env { + let Some((k, v)) = pair.split_once('=') else { + bail!("--env takes KEY=VALUE, not {pair}"); + }; + settings.env.insert(k.trim().to_owned(), v.to_owned()); + } + for k in &unset_env { + settings.env.remove(k.trim()); + } + if let Some(OrDefault(c)) = cpus { + settings.cpus = c; + } + if let Some(OrDefault(m)) = memory { + settings.memory_mb = m; + } if let Some(on) = credit_people { settings.credit_people = on; } @@ -398,6 +577,44 @@ pub async fn run(server: Option<&str>, cmd: ProjectsCmd, json: bool) -> Result<( idle(&p.settings) ); } + if new_image.is_some() || build.is_some() || build_context.is_some() { + let repo = p.repo.as_ref().map(|r| r.full_name()); + println!( + "{} Sessions in {} start from {}", + style::accent("✓"), + style::bold(&p.slug), + image(&p.settings, repo.as_deref()) + ); + } + if cpus.is_some() || memory.is_some() { + println!( + "{} Sandboxes in {} get {}", + style::accent("✓"), + style::bold(&p.slug), + sandbox(&p.settings) + ); + } + if script.is_some() { + println!( + "{} Setup script of {}: {}", + style::accent("✓"), + style::bold(&p.slug), + script_summary(p.settings.setup_script.as_deref().unwrap_or_default()) + ); + } + if !env.is_empty() || !unset_env.is_empty() { + let keys: Vec<&str> = p.settings.env.keys().map(String::as_str).collect(); + println!( + "{} Environment of {}: {}", + style::accent("✓"), + style::bold(&p.slug), + if keys.is_empty() { + "no variables".to_owned() + } else { + keys.join(", ") + } + ); + } if publish_screenshots.is_some() || screenshot_days.is_some() { println!( "{} Agents in {} {}", diff --git a/docs/README.md b/docs/README.md index 48c7a79d..e6a4f99a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -23,7 +23,7 @@ Three pieces make up a factory: | Start using fctl on an instance someone set up | [Getting started](getting-started.md) | | Drive everything from a terminal or a script | [The CLI](cli.md) | | Run your own instance on your Cloudflare account | [Self-hosting](self-hosting.md) | -| Know what sessions run in, or build an image for them | [Sandbox images](sandbox-images.md) | +| Know what sessions run in, give a project its own image, or choose the images a host runs | [Sandbox images](sandbox-images.md) | | Understand how the pieces talk to each other | [Architecture](architecture.md) | | Know what fctl does with your credentials | [Security](security.md) | | Work on fctl itself | [Development](development.md) | diff --git a/docs/adr/0017-hosts-choose-the-images-they-run-and-build.md b/docs/adr/0017-hosts-choose-the-images-they-run-and-build.md new file mode 100644 index 00000000..ebc0587b --- /dev/null +++ b/docs/adr/0017-hosts-choose-the-images-they-run-and-build.md @@ -0,0 +1,85 @@ +# 0017: Hosts choose the images they run and build + +2026-10-18 · accepted + +## Context + +Projects could choose a sandbox image since the first release (Settings → +Sandbox → Image), and every host pulled and ran whatever a project named. +That image runs on machines that belong to other people, with the Claude +access token of the person who started the session and the session's +GitHub tokens inside it. A token from `claude setup-token` is valid for a +year. Host owners had no say. + +Getting tools into sessions meant a setup script that installs them in +every session, or an image built and published by hand. Many repositories +already describe their environment in a Dockerfile or a dev container +configuration. And an image's own Claude settings (plugins, marketplaces, +hooks) were lost, because fctl-agent overwrote `~/.claude/settings.json`. + +[#16](https://github.com/hyperstructure/fctl/issues/16) asked for all of +this. This record covers the choices made in building it. + +## Decision + +- **Host owners list what runs.** `allowed_images` in `host.toml` holds + references (no tag: any tag; a tag, unpinned; or a digest, which the + project's reference must name too) and prefixes ending in `/*`. `allowed_builds` + holds GitHub repositories, `owner/name` or `owner/*`. The host's own + image always runs. Unset, `allowed_images` allows any image, as hosts did + before it, and `allowed_builds` allows no builds. +- **The fleet schedules by them.** Hosts send both lists as optional + fields of their hello; hosts of older versions send neither, and run any + image and build none. FleetDO gives a session only to a runner whose + lists take its project's image or build, and a session that no runner + online takes stays queued with the reason. The host checks again before + it accepts a session. Lima and the process driver use no images and send + no `allowed_builds`. +- **Projects build from their default branch.** `image_build` names a + Dockerfile (with a context) or a dev container configuration. Hosts + build from the tip of the project's default branch, never from a + session's base branch, so that what a build runs is what reached that + branch. fctl reads a dev container configuration's `image` and `build` + and leaves out features and lifecycle commands, saying so in the + session's log. +- **Builds are keyed by what they read.** At each session start in a new + sandbox, the host asks GitHub's API for the file, the Dockerfile and the + git tree of the context, and builds only when no image has their hash. + It builds one image at a time, within `build_timeout_minutes`, removes + an image over `max_build_mb`, keeps each project's newest build, and + starts from the last build of the same setting while GitHub can't be + reached. Builds run in the container runtime's builder with no secrets, + from the Dockerfile's text as GitHub sends it. The session's GitHub + token reads the repository on the host and never enters the build. +- **Registries use the host's own login.** fctl stores and sends no + registry credentials. Passing them from project secrets is left for a + later decision, if hosts' own logins prove too much work. +- **The image's Claude settings stay.** fctl-agent writes its keys + (permission mode, `attribution`, `disableClaudeAiConnectors`, the + auto-updater) over the image's settings. It drops `apiKeyHelper`, + `ANTHROPIC_*` and provider variables, and + `permissions.disableBypassPermissionsMode`, and keeps everything else. +- **Claude Code 2.1.101 or later.** It is the first release that asks its + host for fresh OAuth tokens (`oauth_token_refresh`). fctl-agent runs + `claude --version` before it starts Claude and fails the session, saying + why, when Claude Code is older or missing. + +## Consequences + +- A host owner who wants to limit what runs must set `allowed_images`: the + default stays as permissive as before, so that upgrading no host stops + projects' sessions. Making new hosts restrictive by default is a + decision for later. +- A session that starts from another base branch still gets the default + branch's image. +- A change to a build's files makes the next session that starts in a new + sandbox wait for the build. Each host builds its own copy, and caches + only on its own disk. +- fctl doesn't limit a build's CPUs and memory: it gets what the runtime's + builder has. Docker's builder has the daemon's resources, and Apple + container's builder VM is sized with `container builder start`. +- Hosts that build reach `api.github.com` and `codeload.github.com`, as + their sessions already reach GitHub. +- Images whose Claude Code is older than 2.1.101 stop working, with a + message that says what to update, instead of failing once a token + expires. diff --git a/docs/adr/README.md b/docs/adr/README.md index cc78a57f..2f3f462b 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -22,3 +22,4 @@ supersede an earlier one. Copy the newest record to start a new one. | [0014](0014-agents-publish-screenshots-at-unguessable-links.md) | Agents publish screenshots at unguessable links | | [0015](0015-runners-drain-and-their-sessions-carry-on.md) | Runners drain, and their sessions carry on | | [0016](0016-uninstalling-a-host-stops-its-sessions-first.md) | Uninstalling a host stops its sessions first | +| [0017](0017-hosts-choose-the-images-they-run-and-build.md) | Hosts choose the images they run and build | diff --git a/docs/architecture.md b/docs/architecture.md index a028094b..df40a3c5 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -150,14 +150,19 @@ queued → provisioning → preparing → running ⇄ idle → stopped ``` 1. `POST /api/v1/projects/{id}/sessions` creates the row and asks FleetDO to - schedule it. Without a free runner it stays `queued`. + schedule it, on a runner that runs or builds its project's image and + gives its sandbox the size it asks for. Without a free one it stays + `queued`, and says why. 2. FleetDO builds the `SessionSpec` just in time (repositories, branch, agent settings, secrets, fresh GitHub installation tokens, the starter's Claude credential) and sends `start_session` to a runner. -3. The host creates a sandbox with its driver, injects `fctl-agent`, starts it - detached, attaches to it, and opens the session uplink to SessionDO. +3. The host creates a sandbox with its driver (building the project's image + first when it builds one from its repository and the files changed), + injects `fctl-agent`, starts it detached, attaches to it, and opens the + session uplink to SessionDO. 4. fctl-agent clones the primary repository, creates the branch, runs the - project's setup script, writes Claude's configuration, and starts + project's setup script, checks that Claude Code is 2.1.101 or later, + writes Claude's configuration over the image's, and starts `claude -p --input-format stream-json --output-format stream-json`. 5. Claude's messages flow back as events; messages from people flow down as deliveries. A finished turn makes the session `idle`; the sandbox stays up @@ -560,6 +565,27 @@ pulled ([Sandbox images](sandbox-images.md)). `fctl host image build` builds it on the host instead, from that repository's `main`, which it downloads; the Lima driver's base VM gets the image's `provision.sh`. +A project names an image (`ProjectSettings.image`) or a file in its +repository to build one from (`image_build`: a Dockerfile and its context, +or a dev container configuration). Hosts send the images they run besides +their own (`HostHello.allowed_images`: references, `/*` prefixes, digest +pins; absent: any) and the repositories they build images from +(`allowed_builds`; absent: none, and always absent with Lima or the process +driver). FleetDO reads each queued session's image with its size +(`fctl_domain::image::ImageWant`) and only assigns it to a runner whose +lists take it (`fctl_domain::schedule::pick`); a session that none online +takes stays queued with the reason. The host checks its lists again before +it accepts a session. To build (`crates/fctl-host/src/build.rs`), it reads +the file at the tip of the project's default branch through GitHub's API +with the session's token, keys the build by the file, the Dockerfile and +the git tree of the context, and builds only when no image has that key: +it downloads the repository's archive, unpacks the context and the +Dockerfile, and runs `docker build` or `container build`, one at a time, +within `build_timeout_minutes`, removing an image over `max_build_mb`. It +records each project's last build in `builds.json` in its data directory, +removes the one before, and starts from the last build while GitHub can't +be reached. + | Driver | Isolation | Host | |---|---|---| | `apple` | one lightweight VM per session (Apple `container`) | Apple silicon, macOS 26+ | diff --git a/docs/cli.md b/docs/cli.md index 1a4cc7d8..bcd80958 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -110,6 +110,12 @@ fctl projects set -p my-app --credit-claude off # no Claude trailer in its com fctl projects set -p my-app --credit-people off # no trailers for the people who start sessions fctl projects set -p my-app --idle-timeout 30 # idle sessions keep their sandbox 30 minutes (default 10) fctl projects set -p my-app --publish-screenshots on --screenshot-days 7 # links work for a week +fctl projects set -p my-app --image ghcr.io/acme/agent:1 # sessions start from this image ("" for the runner's) +fctl projects set -p my-app --build .devcontainer/devcontainer.json # or one built from the repository +fctl projects set -p my-app --build Dockerfile --build-context app # a Dockerfile, built in app/ +fctl projects set -p my-app --setup-script 'npm ci' # or @setup.sh, or - to read it from stdin +fctl projects set -p my-app --env NODE_ENV=test --unset-env DEBUG +fctl projects set -p my-app --cpus 8 --memory 16g # "default" for the runner's fctl projects use my-app fctl projects delete my-app # with its sessions; asks you to type the slug ``` @@ -129,6 +135,12 @@ fctl projects repos remove my-org/design -p my-app Changes apply to sessions that start afterwards. +A project's image and its build run only on runners whose owners allow +them ([Sandbox images](sandbox-images.md#which-images-a-host-runs)); a +session that no runner allows stays queued and says why. `fctl projects +show` says where the project's sessions start from, their size, setup +script and variables. + ## Notifications A session you follow notifies you when it waits for your reply, pauses on a @@ -163,6 +175,7 @@ fctl claude status # your connections and how much of each plan is fctl host install # register this machine and start the service fctl host install --driver lima --capacity 2 --build-image fctl host install --cpus 4 --memory 8g --max-memory 12g # sandbox sizes +fctl host install --allow-image 'ghcr.io/acme/*' --allow-build 'acme/*' # what projects may run and build fctl host install --token fctl_rt_… # with a runner token from an admin fctl host install --no-service # write the config only (run it yourself) fctl host run # run in the foreground (what the service runs) @@ -214,12 +227,21 @@ memory_mb = 8192 # per sandbox, when the project sets none max_cpus = 8 # the most one sandbox may have (unset: all there is) max_memory_mb = 12288 keep_stopped_hours = 72 # then remove a stopped session's sandbox (0 keeps them) +# allowed_images = ["ghcr.io/acme/*"] # images projects may choose (unset: any) +# allowed_builds = ["acme/*"] # repositories it builds images from (unset: none) +build_timeout_minutes = 60 +max_build_mb = 20480 # the largest image a build may make ``` Sessions start from their project's image or, without one, the host's: unless `image` sets another, the published image of the host's own version, `ghcr.io/hyperstructure/sandbox:`, which the host pulls when it -starts. [Sandbox images](sandbox-images.md) has the details. +starts. A project's own image runs only if `allowed_images` allows it, or +is unset, and an image built from its repository only if `allowed_builds` +lists the repository. `--allow-image` and `--allow-build` replace the lists +(`--allow-image '*'` for any image, `--allow-build ""` for no builds), and +`fctl host status` shows them. [Sandbox images](sandbox-images.md) has the +details. A project's Settings → Sandbox can ask for more CPUs or memory than the host's default. Sessions only start on a host whose maximum covers that; @@ -246,7 +268,7 @@ fctl admin allow ada@example.com # also creates an invite link fctl admin allow '*@example.com' # a whole domain fctl admin allow grace@example.com --admin fctl admin disallow '*@example.com' # its users lose access -fctl admin runners # draining ones say who drained them and what still runs +fctl admin runners # with their images; draining ones say who drained them fctl admin drain studio-mac # no new sessions start on it (name or id) fctl admin undrain studio-mac fctl admin settings # what the instance allows diff --git a/docs/getting-started.md b/docs/getting-started.md index c564a0d2..3d470905 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -97,7 +97,10 @@ Any Mac with Apple silicon (or a Linux box) can be a host. On the Mac: hour. The host then pulls the [sandbox image](sandbox-images.md) (Debian with git, gh, Python, uv, Rust, Node, TypeScript, Bun and Claude Code) in the background. With Lima, add `--build-image`: it prepares the VM that - sessions are cloned from, which takes a few minutes. + sessions are cloned from, which takes a few minutes. A host runs any + image a project chooses; `--allow-image 'ghcr.io/acme/*'` limits that, + and `--allow-build 'acme/*'` lets it build images from those + repositories ([Which images a host runs](sandbox-images.md#which-images-a-host-runs)). 3. Check it: @@ -115,9 +118,13 @@ starts again after a reboot or a power cut. In the app, choose **New project**. Link an existing GitHub repository, create a new one, or start without a repository. Optionally add a setup script (it runs in the clone before the agent starts, e.g. `npm ci`) and instructions for -agents. The **claude.ai connectors** switch starts off: turn it on only if -agents should use the connectors (mail, files, docs) of whoever starts a -session, without asking. From the terminal: +agents. When sessions need tools the default image lacks, give the project an +image of its own, or one built from a Dockerfile or devcontainer.json in its +repository, under Settings → Sandbox → Image +([Your own image](sandbox-images.md#your-own-image)). The **claude.ai +connectors** switch starts off: turn it on only if agents should use the +connectors (mail, files, docs) of whoever starts a session, without asking. +From the terminal: ```sh fctl projects create "My app" --repo my-org/my-app diff --git a/docs/sandbox-images.md b/docs/sandbox-images.md index e19f6d6b..0187c8b3 100644 --- a/docs/sandbox-images.md +++ b/docs/sandbox-images.md @@ -1,8 +1,9 @@ # Sandbox images With the Apple container and Docker drivers, each session runs in a -container made from an image. This page describes the default image, how -hosts get it, and what images built from it can rely on. +container made from an image. This page describes the default image and +how hosts get it, how a project uses an image of its own or builds one from +its repository, which images a host runs, and what an image needs. ## The default image @@ -15,8 +16,8 @@ amd64. | Tag | Image | |---|---| -| `0.4.0` | fctl 0.4.0's. Hosts of that version use it. | -| `0.4` | The newest 0.4 release's. | +| `0.5.0` | fctl 0.5.0's. Hosts of that version use it. | +| `0.5` | The newest 0.5 release's. | | `latest` | The newest release's. | The sandbox repository's Image workflow builds every tag from its `main` @@ -27,7 +28,7 @@ old as the image: every day, the workflow rebuilds the latest release's image when Claude Code has a newer release, and once the image is a week old, for Debian's updates. A change to the image rebuilds it too. A rebuild moves the tags; to pin one build, add its digest: -`ghcr.io/hyperstructure/sandbox:0.4.0@sha256:…`. +`ghcr.io/hyperstructure/sandbox:0.5.0@sha256:…`. Its labels say what is inside: @@ -39,7 +40,7 @@ Its labels say what is inside: | `org.opencontainers.image.created` | when it was built | ```sh -docker image inspect ghcr.io/hyperstructure/sandbox:0.4.0 \ +docker image inspect ghcr.io/hyperstructure/sandbox:0.5.0 \ --format '{{ index .Config.Labels "sh.fctl.claude-code.version" }}' ``` @@ -59,10 +60,10 @@ Code, run `fctl host image pull`. While the version's image can't be pulled, it pulls `latest` instead, and says so. - **Another image.** `image` in `host.toml` (or `fctl host install --image - …`) sets the image for sessions whose project sets none, and a project's - Settings → Sandbox → Image sets one for its own sessions. - `fctl-agent:latest`, the name the default image had when hosts built it - themselves, means the default. + …`) sets the image for sessions whose project sets none, and a project + can choose one for its own sessions ([Your own + image](#your-own-image)). `fctl-agent:latest`, the name the default + image had when hosts built it themselves, means the default. - **Building it.** `fctl host image build` builds the default image on the host with its driver, from the `main` branch of hyperstructure/sandbox, under the same name, so sessions use the build: for a host that can't @@ -105,7 +106,7 @@ An image that installs something as root switches back to `agent` at the end: ```dockerfile -FROM ghcr.io/hyperstructure/sandbox:0.4 +FROM ghcr.io/hyperstructure/sandbox:0.5 USER root RUN apt-get update \ && apt-get install -y --no-install-recommends postgresql-client \ @@ -113,11 +114,246 @@ RUN apt-get update \ USER agent ``` -## Other images +## Your own image -An image that doesn't start from the default one needs `sh`, `tail`, git -and Claude Code on the PATH, and a user with a writable home. fctl copies -its supervisor into `/opt/fctl/bin` at session start and runs it as the -image's user, in `~/workspace`. With fctl on the PATH as well, agents name -their sessions; without it, sessions keep the title they start with. Apple -silicon hosts need arm64 images. +When sessions need tools the default image lacks, such as .NET or a +company's CLIs, give the project an image that has them. A setup script +would install them in every session instead. This example adds the .NET 8 +SDK. .NET needs ICU, which Debian has, and the SDK goes into `agent`'s +home, like Rust. The PATH is set twice: `ENV` for Claude and the commands +it runs, and `/etc/profile.d` for login shells, which setup scripts run in: + +```dockerfile +FROM ghcr.io/hyperstructure/sandbox:0.5 + +USER root +RUN apt-get update \ + && apt-get install -y --no-install-recommends libicu72 \ + && rm -rf /var/lib/apt/lists/* \ + && printf '%s\n' 'export DOTNET_ROOT="$HOME/.dotnet"' \ + 'PATH="$HOME/.dotnet:$HOME/.dotnet/tools:$PATH"' > /etc/profile.d/dotnet.sh +USER agent + +RUN curl -fsSL https://dot.net/v1/dotnet-install.sh -o /tmp/dotnet-install.sh \ + && bash /tmp/dotnet-install.sh --channel 8.0 --install-dir /home/agent/.dotnet \ + && rm /tmp/dotnet-install.sh +ENV DOTNET_ROOT=/home/agent/.dotnet \ + DOTNET_CLI_TELEMETRY_OPTOUT=1 \ + PATH=/home/agent/.dotnet:/home/agent/.dotnet/tools:$PATH +``` + +1. **Build it for both architectures** and push it to a registry: Apple + silicon hosts run arm64 images, most others amd64. + + ```sh + docker buildx build --platform linux/arm64,linux/amd64 \ + -t ghcr.io/acme/dotnet-agent:1 --push . + ``` + +2. **Choose it for the project.** In Settings → Sandbox → Image, pick + "An image from a registry" and enter `ghcr.io/acme/dotnet-agent:1`. From + the CLI: + + ```sh + fctl projects set -p my-app --image ghcr.io/acme/dotnet-agent:1 + fctl projects set -p my-app --image "" # back to each runner's own + ``` + + Maintainers and owners can change it. Sessions that start or resume + afterwards in a new sandbox use it; a sandbox that exists keeps the + image it was made from. + +3. **Hosts run it if their owners allow it.** A host that lists + `allowed_images` runs only the images on its list ([Which images a host + runs](#which-images-a-host-runs)); one that lists none runs any. + +4. **Private registries.** Hosts pull images with their own login to the + registry: run `docker login ghcr.io` (with Apple container, + `container registry login ghcr.io`) on each host, as the user its + service runs as. fctl sends no registry credentials. For GHCR, a token + with `read:packages` is enough. + +Lima runners don't use images: their sessions run in a clone of the VM +`fctl host image build` prepared. The process driver uses none either. + +## Building it from the repository + +A project can build its image from a Dockerfile or a dev container +configuration in its repository instead, so that the environment lives +next to the code and changes with it. In Settings → Sandbox → Image, pick +"Built from the repository" and name the file, or from the CLI: + +```sh +fctl projects set -p my-app --build Dockerfile +fctl projects set -p my-app --build docker/agent.Dockerfile --build-context app +fctl projects set -p my-app --build .devcontainer/devcontainer.json +fctl projects set -p my-app --build "" # stop building +``` + +The project needs a repository, and its build replaces any image it chose. +Only hosts whose owners allow builds from the repository take its sessions +(`allowed_builds`, [below](#which-images-a-host-runs)), and only those with +the Apple container or Docker driver. + +When a session starts in a new sandbox, the host: + +1. Reads the file at the tip of the project's default branch, with the + session's GitHub token, and works out what the build reads: the + Dockerfile, and the directory it builds in (the repository's root + unless the setting or the dev container configuration names another). +2. Uses its last build when none of that changed since. Changes outside + the directory don't count, but with the repository's root as the + directory, every change does. In a busy repository, give the image a + directory of its own (`--build-context docker`, say, or the + `.devcontainer` directory a dev container configuration builds in by + default); the builder's cache keeps the builds that follow short + anyway. +3. Otherwise downloads the repository at that commit (up to 2 GiB), keeps + only the directory, and builds the image with `docker build` or + `container build`, from the Dockerfile's text as GitHub sent it. The + session's log shows each step. Sessions that need the same build wait + for the one that builds it. Builds pass no secrets: a Dockerfile that + needs a token should leave that step to the setup script. +4. Tags the image `fctl-build-:`, and removes the + project's earlier build, unless the runtime keeps it for a sandbox made + from it (Docker does). + +When GitHub can't be reached, sessions start from the last build of the +same setting and say so; any other problem, like a file that isn't there, +fails the session with the reason. A dev container configuration's +`image`, with no build, is used as it is, from its registry. + +**Limits.** A host builds one image at a time, and reads no Dockerfile or +configuration larger than 1 MiB. A build that takes longer +than `build_timeout_minutes` (60) is stopped, and an image larger than +`max_build_mb` (20 GiB) is removed, where the runtime reports image sizes +(Docker does); the session fails and says why. A build runs in the +runtime's builder, not in a sandbox, so the project's CPU and memory +settings don't apply: with Docker it gets what the Docker daemon has (on +macOS, its VM), and with Apple container what its builder VM has +(`container builder start --cpus 8 --memory 16g` sizes it). + +**What fctl reads from a dev container configuration.** `image`, or +`build.dockerfile`, `build.context`, `build.args` and `build.target` +(`dockerFile` and `context` at the top level in older ones), with paths +relative to the configuration's directory. It leaves out features, +lifecycle commands like `postCreateCommand`, `containerEnv`, `mounts` and +`runArgs`, and says so in the session's log: put what they install into +the Dockerfile, and what they run into the project's setup script. +Sessions run as the image's user rather than `remoteUser`, so end the +Dockerfile with `USER vscode`, say, to run as that user. Docker Compose +configurations aren't built. + +The image still needs what every image needs ([below](#what-an-image-needs)): +most dev container images lack Claude Code. Start the Dockerfile `FROM +ghcr.io/hyperstructure/sandbox:0.5`, or install Claude Code in it as the +user sessions run as (`curl -fsSL https://claude.ai/install.sh | bash`). + +## Which images a host runs + +A project's image and its build run on machines that belong to other +people, with the session starter's Claude token and the session's GitHub +tokens inside them ([Security](security.md#sandboxes)). So each host's +owner decides, in its `host.toml`: + +```toml +# Images projects may choose; the host's own `image` always runs. Unset: any. +allowed_images = [ + "ghcr.io/hyperstructure/sandbox", # any tag of it + "ghcr.io/acme/*", # anything under ghcr.io/acme/ + "ghcr.io/partner/agent@sha256:4f1c…", # this build only +] +# Repositories whose Dockerfile or devcontainer.json it builds. Unset: none. +allowed_builds = ["acme/*", "partner/app"] +build_timeout_minutes = 60 +max_build_mb = 20480 +``` + +The same lists from the CLI, each flag repeated for more entries +(`--allow-image '*'` allows any image, `--allow-build ""` none): + +```sh +fctl host install --allow-image 'ghcr.io/acme/*' --allow-image ghcr.io/hyperstructure/sandbox \ + --allow-build 'acme/*' +``` + +- **An entry of `allowed_images`** is a reference without a tag, which + allows any tag or digest of that repository; with a tag, which allows + that tag but not a reference that pins a digest to it, since a pull by + digest ignores the tag; or pinned to a digest (`@sha256:…`), which allows + only that build, and only when the project's reference names the digest + too, since a tag can move. + An entry that ends in `/*` allows everything under that registry path, + and `*` alone allows any image. Docker Hub's names count as Docker does: + `ubuntu` is `docker.io/library/ubuntu`. +- **An entry of `allowed_builds`** is `owner/name` or `owner/*` on GitHub. + Whoever can push to a project's default branch decides what its builds + contain, including the images they start `FROM`. +- **Without the lists**, a host runs any image a project chooses, as hosts + did before them, and builds none. Hosts of older versions do the same. + +The host sends the lists to the control plane when it connects, so run +`fctl host restart` after editing `host.toml`. The control plane gives a +session only to a runner that runs its project's image or builds it, and +the runner checks again. A session that no runner online allows stays +queued and says why: "No runner allows this project's image, +ghcr.io/acme/agent:1", or "No runner builds this project's image from +acme/app". Admin → Runners, `fctl admin runners` and `fctl host status` +show each runner's image and lists. + +## What an image needs + +An image that doesn't start from the default one needs: + +- **`sh`, `tail` and git** on the PATH. The sandbox's main process is `tail + -f /dev/null`. +- **Claude Code 2.1.101 or later**, as `claude` on the PATH. fctl relies on + requests that Claude Code sends and answers in its stream-json mode: it + asks fctl for a fresh OAuth access token (`oauth_token_refresh`, from + 2.1.101), and fctl turns Remote Control on (`remote_control`, from + 2.1.63). fctl-agent runs `claude --version` before it starts Claude, and + the session fails with a message that says so when Claude Code is + missing or older. +- **`gh`**, which fctl-agent asks for the pull request of the session's + branch. Without it, sessions show no pull request. +- **fctl** on the PATH, for agents to name their sessions and publish + screenshots. Without it, sessions keep the title they start with. +- **A non-root user with a writable home.** fctl copies its supervisor into + `/opt/fctl/bin` as root at session start, and runs it as the image's + user (its `USER`), in `~/workspace`. +- **arm64** builds for Apple silicon hosts, and amd64 for most others. + +## What fctl writes into a sandbox + +At session start, fctl copies fctl-agent into `/opt/fctl/bin` and starts +it as the image's user, which writes: + +- **`~/.fctl/`**: fctl-agent's sockets, log and state, the session's GitHub + tokens (mode 0600) and the git credential helper that hands them out. +- **git's global configuration**: the GitHub App's bot as the author, the + credential helper for github.com, `push.autoSetupRemote`, + `init.defaultBranch` and a few quiet defaults. `~/.config/gh/hosts.yml` + gives `gh` the token that can write. +- **`~/.claude.json`**: Claude Code's onboarding and the bypass-permissions + prompt marked done, the auto-updater off, the working directory trusted, + and the Claude account of the person who started the session. The rest + of the file stays. +- **`~/.claude/settings.json`**: the image's own settings, with fctl's over + them. fctl decides the permission mode (bypass), `attribution` (the + trailers the project's settings ask for), `disableClaudeAiConnectors` and + the auto-updater, and removes `apiKeyHelper`, `ANTHROPIC_*` and provider + variables from `env`, and `permissions.disableBypassPermissionsMode`. + Plugins, marketplaces, hooks, permission rules and other variables stay, + and so do skills in `~/.claude/skills`. +- **`~/.claude/CLAUDE.md`**: a line that imports `~/AGENTS.md`, fctl's + instructions for agents, after whatever the image has there. +- **`~/workspace`**, the project's repository, on the session's branch; + `~/repos//` for its other repositories; and + `~/attachments/` for files people attach. + +Claude's environment has the project's variables and secrets, +`CLAUDE_CODE_OAUTH_TOKEN` and its scopes, `IS_SANDBOX=1`, +`DISABLE_AUTOUPDATER=1`, `GIT_TERMINAL_PROMPT=0`, `FCTL_SESSION_ID`, +`FCTL_SESSION_URL`, and the sandbox's size in `FCTL_SANDBOX_CPUS` and +`FCTL_SANDBOX_MEMORY_MB`. `ANTHROPIC_*` variables and the Bedrock, Vertex +and Foundry switches never reach it. diff --git a/docs/security.md b/docs/security.md index c02fab09..16d33d95 100644 --- a/docs/security.md +++ b/docs/security.md @@ -111,6 +111,15 @@ outside it except through these channels: a screenshot, as it can ask it to send what it sees elsewhere. - **Project secrets** are exported to the agent's environment. Treat them as readable by anything the agent runs. +- **The image and the setup script.** Whoever sets a project's image or + setup script decides what runs in its sandboxes, beside the credentials + of the people who start its sessions: their Claude access token and the + session's GitHub tokens. A full Claude login's access token lasts hours; + a token from `claude setup-token` is valid for a year. Maintainers and + owners set both, and for an image built from the repository, so does + whoever can push to its default branch. Hosts' owners decide which images + their machines run and which repositories they build images from + ([below](#hosts)). - **Attached files** are saved in the sandbox under `~/attachments/`, where anything the agent runs can read them, and Claude sees images in the message. @@ -160,8 +169,20 @@ by default `ghcr.io/hyperstructure/sandbox`, which the Image workflow of the public [hyperstructure/sandbox](https://github.com/hyperstructure/sandbox) repository builds, so anyone can read what goes into it. A host can use its own build of it instead (`fctl host image build`) or a copy in another -registry (`image` in `host.toml`); a project's own image still applies to -its sessions ([Sandbox images](sandbox-images.md)). +registry (`image` in `host.toml`). + +A project's own image runs only on hosts that allow it, and its build only +on hosts that build images from its repository: `allowed_images` and +`allowed_builds` in `host.toml`. Without `allowed_images`, a host runs any +image a project chooses, as hosts did before it, so set it on machines +where that matters. Without `allowed_builds`, it builds none. The control +plane schedules sessions by these lists, and the host checks them again +before it takes a session. To build, the host downloads the repository at +the tip of its default branch with the session's GitHub token, which the +build never sees, and runs the build in its container runtime's builder, +outside any sandbox ([Sandbox images](sandbox-images.md#which-images-a-host-runs)). +Hosts pull images with their own registry logins; fctl stores no registry +credentials. ## Retention diff --git a/docs/self-hosting.md b/docs/self-hosting.md index 3f529219..379d5f30 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -180,7 +180,11 @@ Hosts download their release from `https://fctl.sh/releases` by default; set `download_base` in `host.toml` to mirror them elsewhere (`fctl host image build` installs fctl in the default image from there too). They pull the sandbox image from `ghcr.io/hyperstructure/sandbox`; set `image` to use a -copy in your own registry ([Sandbox images](sandbox-images.md)). +copy in your own registry ([Sandbox images](sandbox-images.md)). A host runs +any image a project chooses until its owner lists the ones it allows +(`allowed_images`), and builds images from no repository until they list +some (`allowed_builds`): set both on machines that run other people's +projects ([Which images a host runs](sandbox-images.md#which-images-a-host-runs)). ## Operating it diff --git a/web/src/gen/ImageBuild.ts b/web/src/gen/ImageBuild.ts new file mode 100644 index 00000000..699a57b8 --- /dev/null +++ b/web/src/gen/ImageBuild.ts @@ -0,0 +1,19 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +/** + * A sandbox image built from the project's repository, as its default + * branch has it. Hosts build it again when the files it comes from change. + */ +export type ImageBuild = { +/** + * A Dockerfile, or a dev container configuration (a file ending in + * `.json`, like `.devcontainer/devcontainer.json`), by its path in the + * repository. + */ +file: string, +/** + * For a Dockerfile, the directory in the repository it builds in. + * Absent: the repository's root. A dev container configuration names + * its own. + */ +context?: string, }; diff --git a/web/src/gen/ProjectSettings.ts b/web/src/gen/ProjectSettings.ts index fb53a40b..be8749be 100644 --- a/web/src/gen/ProjectSettings.ts +++ b/web/src/gen/ProjectSettings.ts @@ -1,4 +1,5 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { ImageBuild } from "./ImageBuild"; /** * Settings agents use for a project. Every field has a sensible default, so @@ -9,6 +10,14 @@ export type ProjectSettings = { * OCI image for sandboxes. Defaults to the host's configured agent image. */ image?: string, +/** + * Build the sandbox image from a file in the project's repository + * instead, on hosts whose owners allow it (`allowed_builds`). It takes + * the place of `image`. Older hosts and control planes ignore it. + * Always sent, as `null` for none: an update without it keeps the + * stored build (see `fctl_domain::settings::keep_unsent`). + */ +image_build?: ImageBuild | null, /** * Shell script run in the repository after cloning (for example `npm ci`). */ diff --git a/web/src/gen/Runner.ts b/web/src/gen/Runner.ts index 185c4d7f..0ca9d6bd 100644 --- a/web/src/gen/Runner.ts +++ b/web/src/gen/Runner.ts @@ -18,7 +18,22 @@ memory_available_mb?: number, disk_available_mb?: number, /** * What it gives each sandbox (absent from older hosts). */ -sandbox?: SandboxLimits, labels: Array, last_seen_at?: number, created_at: number, created_by?: UserRef, +sandbox?: SandboxLimits, +/** + * The image it gives sessions whose project sets none. + */ +image?: string, +/** + * The other images it runs, as its hello says ([`HostHello`]). + * Absent: any image. + * + * [`HostHello`]: crate::runner::HostHello + */ +allowed_images?: Array, +/** + * The repositories it builds images from. Absent: none. + */ +allowed_builds?: Array, labels: Array, last_seen_at?: number, created_at: number, created_by?: UserRef, /** * Set while the runner drains: no new session starts on it, and the * sessions it has carry on there. Absent otherwise, and from control diff --git a/web/src/gen/index.ts b/web/src/gen/index.ts index 7a96e847..39bc4fd9 100644 --- a/web/src/gen/index.ts +++ b/web/src/gen/index.ts @@ -65,6 +65,7 @@ export type { GithubRepo } from "./GithubRepo"; export type { GithubRepoList } from "./GithubRepoList"; export type { Identity } from "./Identity"; export type { IdentityId } from "./IdentityId"; +export type { ImageBuild } from "./ImageBuild"; export type { InstanceInfo } from "./InstanceInfo"; export type { InstanceRole } from "./InstanceRole"; export type { InstanceSettings } from "./InstanceSettings"; diff --git a/web/src/lib/images.test.ts b/web/src/lib/images.test.ts new file mode 100644 index 00000000..dfee7a55 --- /dev/null +++ b/web/src/lib/images.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, it } from "vitest"; +import type * as G from "@/gen"; +import { buildsImages, imageSource, isDevcontainer, remember, runsImages, withSource } from "./images"; + +const settings = (s: Partial): G.ProjectSettings => ({ + auto_pull_request: true, + remote_control: true, + claude_ai_connectors: true, + credit_people: true, + credit_claude: true, + publish_screenshots: false, + env: {}, + ...s, +}); + +describe("sandbox images", () => { + it("knows where sessions start from", () => { + expect(imageSource(settings({}))).toBe("default"); + expect(imageSource(settings({ image: "fctl-agent:latest" }))).toBe("default"); + expect(imageSource(settings({ image: "ghcr.io/acme/agent:1" }))).toBe("image"); + expect(imageSource(settings({ image_build: { file: "Dockerfile" } }))).toBe("build"); + }); + + it("switches between them, sending null for no build", () => { + const image = withSource(settings({ image_build: { file: "Dockerfile" } }), "image"); + expect(image.image_build).toBeNull(); + expect(JSON.parse(JSON.stringify(image))).toHaveProperty("image_build", null); + const build = withSource(settings({ image: "ghcr.io/acme/agent:1" }), "build"); + expect(build.image).toBeUndefined(); + expect(build.image_build).toEqual({ file: "" }); + const kept = withSource(settings({ image_build: { file: "x/Dockerfile", context: "x" } }), "build"); + expect(kept.image_build).toEqual({ file: "x/Dockerfile", context: "x" }); + const none = withSource(settings({ image: "ghcr.io/acme/agent:1" }), "default"); + expect(none.image).toBeUndefined(); + expect(none.image_build).toBeNull(); + }); + + it("brings back what was typed when switching back", () => { + const start = settings({ image: "ghcr.io/acme/agent:1" }); + let typed = remember(start, {}); + const away = withSource(start, "build", typed); + typed = remember(away, typed); + const back = withSource(away, "image", typed); + expect(back.image).toBe("ghcr.io/acme/agent:1"); + expect(back.image_build).toBeNull(); + const built = withSource( + back, + "build", + remember(back, { build: { file: "Dockerfile", context: "app" } }), + ); + expect(built.image_build).toEqual({ file: "Dockerfile", context: "app" }); + }); + + it("tells dev container configurations from Dockerfiles", () => { + expect(isDevcontainer(".devcontainer/devcontainer.json")).toBe(true); + expect(isDevcontainer("Dockerfile")).toBe(false); + }); + + it("says what runners run and build", () => { + expect(runsImages(undefined)).toBe("runs any image a project chooses"); + expect(runsImages(["*"])).toBe("runs any image a project chooses"); + expect(runsImages([])).toBe("runs only its own image"); + expect(runsImages(["ghcr.io/acme/*", "ubuntu"])).toBe("runs its own image and ghcr.io/acme/*, ubuntu"); + expect(buildsImages(undefined)).toBe("builds no images"); + expect(buildsImages(["acme/*"])).toBe("builds images from acme/*"); + expect(buildsImages(["*"])).toBe("builds images from any repository"); + }); +}); diff --git a/web/src/lib/images.ts b/web/src/lib/images.ts new file mode 100644 index 00000000..f2d05cdb --- /dev/null +++ b/web/src/lib/images.ts @@ -0,0 +1,53 @@ +// Sandbox images: where a project's sessions start from, and what a runner's +// owner lets it run and build (fctl-domain `image`). + +import type * as G from "@/gen"; + +/** Where a project's sessions start from. */ +export type ImageSource = "default" | "image" | "build"; + +export function imageSource(s: G.ProjectSettings): ImageSource { + if (s.image_build) return "build"; + return s.image?.trim() && s.image.trim() !== "fctl-agent:latest" ? "image" : "default"; +} + +/** What was typed for each source, so that switching back brings it back. */ +export interface Typed { + image?: string; + build?: G.ImageBuild | null; +} + +/** What `s` has typed for each source, over what `typed` remembers. */ +export const remember = (s: G.ProjectSettings, typed: Typed): Typed => ({ + image: s.image ?? typed.image, + build: s.image_build ?? typed.build, +}); + +/** Settings that start sessions from `source`, with what was typed for it. */ +export function withSource(s: G.ProjectSettings, source: ImageSource, typed: Typed = {}): G.ProjectSettings { + switch (source) { + case "default": + return { ...s, image: undefined, image_build: null }; + case "image": + return { ...s, image: s.image ?? typed.image, image_build: null }; + case "build": + return { ...s, image: undefined, image_build: s.image_build ?? typed.build ?? { file: "" } }; + } +} + +/** Whether a build's file is a dev container configuration, which names its own directory. */ +export const isDevcontainer = (file: string) => file.trim().toLowerCase().endsWith(".json"); + +/** "runs any image a project chooses", "runs its own image and ghcr.io/acme/*". */ +export function runsImages(allowed?: string[]): string { + if (!allowed || allowed.some((p) => p.trim() === "*")) return "runs any image a project chooses"; + if (allowed.length === 0) return "runs only its own image"; + return `runs its own image and ${allowed.join(", ")}`; +} + +/** "builds no images", "builds images from acme/*". */ +export function buildsImages(builds?: string[]): string { + if (!builds || builds.length === 0) return "builds no images"; + if (builds.some((p) => p.trim() === "*")) return "builds images from any repository"; + return `builds images from ${builds.join(", ")}`; +} diff --git a/web/src/routes/admin.tsx b/web/src/routes/admin.tsx index 4b626e53..fc047e6b 100644 --- a/web/src/routes/admin.tsx +++ b/web/src/routes/admin.tsx @@ -24,6 +24,7 @@ import type * as G from "@/gen"; import { api } from "@/lib/api"; import { cn } from "@/lib/cn"; import { ago, cpus, memory, sandboxLimits } from "@/lib/format"; +import { buildsImages, runsImages } from "@/lib/images"; import { keys, queryClient, useMe, useRunners } from "@/lib/queries"; type Tab = "access" | "users" | "runners" | "settings" | "audit"; @@ -378,6 +379,7 @@ function Runners() { {r.drain && r.status !== "disabled" && } +

{r.status !== "disabled" && ( @@ -547,6 +549,17 @@ function RunnerResources({ runner: r }: { runner: G.Runner }) { return text ?
{text}
: null; } +/** The image a runner gives sessions, and the others it runs and builds. */ +function RunnerImages({ runner: r }: { runner: G.Runner }) { + if (!r.image) return null; + return ( +
+ Image {r.image} · {runsImages(r.allowed_images)} ·{" "} + {buildsImages(r.allowed_builds)} +
+ ); +} + function InstanceSettings() { const settings = useQuery({ queryKey: keys.instanceSettings, queryFn: api.instanceSettings }); const save = useMutation({ diff --git a/web/src/routes/project.tsx b/web/src/routes/project.tsx index d0573a17..4b4f4265 100644 --- a/web/src/routes/project.tsx +++ b/web/src/routes/project.tsx @@ -36,6 +36,14 @@ import { api } from "@/lib/api"; import { filesFromPaste } from "@/lib/attachments"; import { cn } from "@/lib/cn"; import { ago, agoShort, cpus, memory, stamp } from "@/lib/format"; +import { + type ImageSource, + imageSource, + isDevcontainer, + remember, + type Typed, + withSource, +} from "@/lib/images"; import { DAY_CHOICES, DEFAULT_DAYS, lasts } from "@/lib/published"; import { keys, @@ -481,6 +489,82 @@ function LargestSandbox() { ); } +/** Where a project's sessions can start from: a build only with a repository. */ +function imageSources(project: G.Project): SelectOption[] { + return [ + { + value: "default", + label: "Each runner's own", + description: "ghcr.io/hyperstructure/sandbox, unless the runner's owner chose another", + }, + { value: "image", label: "An image from a registry", description: "On runners that allow it" }, + { + value: "build", + label: "Built from the repository", + description: project.repo + ? "A Dockerfile or devcontainer.json on its default branch" + : "Link a repository first", + disabled: !project.repo, + }, + ]; +} + +/** The file a project's image builds from, and the directory a Dockerfile builds in. */ +function BuildFields({ + project, + build, + onChange, + error, +}: { + project: G.Project; + build: G.ImageBuild; + onChange: (b: G.ImageBuild) => void; + error?: string; +}) { + const repo = project.repo ? `${project.repo.owner}/${project.repo.name}` : "the repository"; + const devcontainer = isDevcontainer(build.file); + return ( +
+
+ + {(id) => ( + + onChange({ + file: e.target.value, + context: isDevcontainer(e.target.value) ? undefined : build.context, + }) + } + className="font-mono" + placeholder="Dockerfile" + /> + )} + + {!devcontainer && ( + + {(id) => ( + onChange({ ...build, context: e.target.value || undefined })} + className="font-mono" + placeholder="." + /> + )} + + )} +
+

+ Runners whose owners allow builds from {repo} (allowed_builds in host.toml) build it from{" "} + {project.default_branch}, keep it, and build it again when the files it comes from change. + {devcontainer ? " The devcontainer.json names its own Dockerfile and directory." : ""} +

+
+ ); +} + /** Claude's effort levels; the empty value leaves it to Claude. */ const EFFORTS: SelectOption[] = [ { value: "", label: "Claude's default" }, @@ -505,7 +589,12 @@ function Settings({ project }: { project: G.Project }) { .map(([k, v]) => `${k}=${v}`) .join("\n"), ); - useEffect(() => setS(project.settings), [project.settings]); + const [source, setSource] = useState(imageSource(project.settings)); + const typed = useRef({}); + useEffect(() => { + setS(project.settings); + setSource(imageSource(project.settings)); + }, [project.settings]); const save = useMutation({ mutationFn: () => { const env: Record = {}; @@ -740,19 +829,47 @@ function Settings({ project }: { project: G.Project }) { {(id) => ( - set("image", e.target.value || undefined)} - className="font-mono" - placeholder="ghcr.io/hyperstructure/sandbox" + value={source} + onChange={(v) => { + typed.current = remember(s, typed.current); + setSource(v); + setS((x) => withSource(x, v, typed.current)); + }} + options={imageSources(project)} + className="w-full" /> )} + {source === "image" && ( + + {(id) => ( + set("image", e.target.value || undefined)} + className="font-mono" + placeholder="ghcr.io/acme/agent:1" + /> + )} + + )} + {source === "build" && s.image_build && ( + set("image_build", b)} + error={err("settings.image_build")} + /> + )}
{(id) => (