mdbook: flatten input paths to support cross-package/repository dependencies

extensions/mdbook/private/BUILD.bazel

@@ -1,5 +1,5 @@
 load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
-load("@rules_rust//rust:defs.bzl", "rust_binary")
+load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_test")
 
 rust_binary(
     name = "process_wrapper",
@@ -20,6 +20,12 @@ rust_binary(
     ],
 )
 
+rust_test(
+    name = "server_unit_test",
+    crate = ":server",
+    edition = "2021",
+)
+
 bzl_library(
     name = "bzl_lib",
     srcs = glob(["*.bzl"]),

extensions/mdbook/private/mdbook.bzl

@@ -9,15 +9,32 @@ MdBookInfo = provider(
     },
 )
 
-def _map_inputs(file):
-    dest = file.short_path
-    if dest.startswith("../"):
-        # External repositories have short_paths starting with '../'.
-        # We need them to be staged at 'external/' within our shadow
-        # directory to match how 'file.path' (and thus 'file.dirname')
-        # refers to them.
-        dest = "external/" + dest.removeprefix("../")
+def _flatten_path(file):
+    """Flattens a file's path for staging.
+
+    We want to flatten the directory structure so that files from
+    different packages sit together. We do this by stripping the
+    file's repository and package prefixes.
+
+    Example:
+    file:    test/flattening/content/src/SUMMARY.md
+    package: test/flattening/content
+    result:  src/SUMMARY.md
+    """
+    path = file.short_path
+    if path.startswith("../"):
+        # External repositories: strip '../repo_name/'
+        parts = path.split("/")
+        path = "/".join(parts[2:])
+
+    package = file.owner.package
+    if package and path.startswith(package + "/"):
+        path = path[len(package) + 1:]
+
+    return path
 
+def _map_inputs(file):
+    dest = _flatten_path(file)
     return "{}={}".format(file.path, dest)
 
 def _mdbook_impl(ctx):
@@ -45,7 +62,13 @@ def _mdbook_impl(ctx):
     args.add(output.path)
     args.add(toolchain.mdbook)
     args.add("build")
-    args.add("${{pwd}}/{}".format(ctx.file.book.dirname))
+
+    book_dest = _flatten_path(ctx.file.book)
+    book_dirname = ""
+    if "/" in book_dest:
+        parts = book_dest.split("/")
+        book_dirname = "/" + "/".join(parts[:-1])
+    args.add("${{pwd}}{}".format(book_dirname))
 
     ctx.actions.run(
         mnemonic = "MdBookBuild",
@@ -71,7 +94,19 @@ def _mdbook_impl(ctx):
 
 mdbook = rule(
     implementation = _mdbook_impl,
-    doc = "Rules to create book from markdown files using `mdBook`.",
+    doc = """Rules to create book from markdown files using `mdbook`.
+
+This rule flattens all input files into a single directory structure
+before running `mdbook`. This is necessary because `mdbook` expects
+all source files, themes, and configuration to sit in a standard
+relative layout on disk.
+
+By flattening inputs, this rule allows you to pull the `book.toml`
+from one Bazel package (or external repository) and the sources from
+another, and have them sit as siblings during execution. This means
+your `book.toml` can use standard relative paths (like `src = "src"`)
+regardless of Bazel's internal directory structure.
+""",
     attrs = {
         "book": attr.label(
             doc = "The `book.toml` file.",
@@ -87,7 +122,11 @@ mdbook = rule(
             cfg = "exec",
         ),
         "srcs": attr.label_list(
-            doc = "All inputs to the book.",
+            doc = """All inputs to the book.
+
+These files will be flattened into a single directory structure alongside the
+`book.toml` file. Package and external repository prefixes are stripped.
+""",
             allow_files = True,
         ),
         "_process_wrapper": attr.label(
@@ -118,6 +157,21 @@ def _mdbook_server_impl(ctx):
 
     workspace_name = ctx.workspace_name
 
+    # Detect if we need to flatten files. We need flattening if any
+    # input comes from a different repository than the book.toml.
+    is_split = False
+    config_repo = book_info.config.owner.workspace_name
+    for f in book_info.srcs.to_list():
+        if f.owner.workspace_name != config_repo:
+            is_split = True
+            break
+
+    if is_split:
+        def _src_map(file):
+            return "--src={}={}".format(_rlocationpath(file, workspace_name), _flatten_path(file))
+
+        args.add_all(book_info.srcs, map_each = _src_map, allow_closure = True)
+
     def _runfile_map(file):
         return "--plugin={}".format(_rlocationpath(file, workspace_name))
 

extensions/mdbook/private/server.rs

@@ -1,6 +1,7 @@
 //! A process wrapper for `mdbook serve`.
 
-use std::path::PathBuf;
+use std::collections::BTreeMap;
+use std::path::{Path, PathBuf};
 use std::process::Command;
 use std::{env, fs};
 
@@ -23,6 +24,8 @@ struct Args {
 
     pub plugins: Vec<PathBuf>,
 
+    pub srcs: Vec<(PathBuf, PathBuf)>,
+
     pub mdbook_args: Vec<String>,
 }
 
@@ -39,6 +42,7 @@ impl Args {
         let mut hostname: Option<String> = None;
         let mut port: Option<String> = None;
         let mut plugins: Vec<PathBuf> = Vec::new();
+        let mut srcs: Vec<(PathBuf, PathBuf)> = Vec::new();
 
         for arg in raw_args {
             if arg.starts_with("--mdbook=") {
@@ -54,6 +58,10 @@ impl Args {
                 hostname = Some(arg.split_once("=").unwrap().1.to_string());
             } else if arg.starts_with("--port=") {
                 port = Some(arg.split_once("=").unwrap().1.to_string());
+            } else if arg.starts_with("--src=") {
+                let val = arg.split_once("=").unwrap().1;
+                let (rloc, dest) = val.split_once("=").unwrap();
+                srcs.push((rlocation!(runfiles, rloc).unwrap(), PathBuf::from(dest)));
             }
         }
 
@@ -63,6 +71,7 @@ impl Args {
             hostname: hostname.unwrap(),
             port: port.unwrap(),
             plugins,
+            srcs,
             mdbook_args: env::args().skip(1).collect(),
         }
     }
@@ -105,11 +114,62 @@ fn make_temp_dir() -> PathBuf {
     panic!("Could not determine how to create temp dir.")
 }
 
+#[cfg(target_family = "unix")]
+fn symlink<P: AsRef<Path>, Q: AsRef<Path>>(src: P, dst: Q) {
+    std::os::unix::fs::symlink(src.as_ref(), dst.as_ref()).unwrap_or_else(|e| {
+        panic!(
+            "Failed to create symlink: {} -> {}: {}",
+            src.as_ref().display(),
+            dst.as_ref().display(),
+            e
+        )
+    });
+}
+
+#[cfg(target_family = "windows")]
+fn symlink<P: AsRef<Path>, Q: AsRef<Path>>(src: P, dst: Q) {
+    fs::copy(src.as_ref(), dst.as_ref()).unwrap_or_else(|e| {
+        panic!(
+            "Failed to copy file: {} -> {}: {}",
+            src.as_ref().display(),
+            dst.as_ref().display(),
+            e
+        )
+    });
+}
+
+fn stage_files_internal(workdir: &Path, config: &Path, srcs: &BTreeMap<PathBuf, PathBuf>) {
+    symlink(config, workdir.join("book.toml"));
+    for (src, dest) in srcs {
+        let abs_dest = workdir.join(dest);
+        if let Some(parent) = abs_dest.parent() {
+            fs::create_dir_all(parent).unwrap();
+        }
+        symlink(src, abs_dest);
+    }
+}
+
 fn main() {
     let args = Args::parse();
 
     let mut command = Command::new(&args.mdbook);
 
+    let temp_dir = make_temp_dir();
+    let _ = fs::remove_dir_all(&temp_dir);
+    fs::create_dir_all(&temp_dir).unwrap();
+
+    // If we have source mappings, we need to stage the files in a flat directory
+    // so that mdbook can resolve them relative to book.toml.
+    if !args.srcs.is_empty() {
+        let workdir = temp_dir.join("stage");
+        fs::create_dir_all(&workdir).unwrap();
+        stage_files_internal(&workdir, &args.config, &args.srcs.iter().cloned().collect());
+        command.arg("serve").arg(&workdir);
+    } else {
+        // No flattening required, run in-place against the runfiles
+        command.arg("serve").arg(args.config.parent().unwrap());
+    };
+
     // Inject plugin paths into PATH
     let pwd = env::current_dir().expect("Unable to determine current working directory");
     if !args.plugins.is_empty() {
@@ -132,10 +192,7 @@ fn main() {
         command.env("PATH", format!("{}{}{}", plugin_path, PATH_SEP, path));
     }
 
-    command
-        .arg("serve")
-        .arg(args.config.parent().unwrap())
-        .args(&args.mdbook_args);
+    command.args(&args.mdbook_args);
 
     // Add default hostname value if commandline was not specified.
     if !args.mdbook_args.iter().any(|arg| {
@@ -155,29 +212,71 @@ fn main() {
         command.args(["--port", &args.port]);
     }
 
-    // Check if `-d` or `--dest-dir` was passed. If not, make a temp dir
-    let temp_dir: Option<PathBuf> = if !args.mdbook_args.iter().any(|a| {
+    // Check if `-d` or `--dest-dir` was passed. If not, make a temp dir for the output
+    if !args.mdbook_args.iter().any(|a| {
         ["-d", "--dest-dir"].contains(&a.as_str())
             || a.starts_with("-d=")
             || a.starts_with("--dest-dir=")
     }) {
-        let temp_dir = make_temp_dir();
-        command.arg("--dest-dir").arg(&temp_dir);
-        Some(temp_dir)
-    } else {
-        None
+        let output_dir = temp_dir.join("output");
+        command.arg("--dest-dir").arg(&output_dir);
     };
 
-    // Run mdbook and save output
+    // Run mdbook
     let status = command
         .status()
         .unwrap_or_else(|e| panic!("Failed to spawn mdbook command\n{:?}\n{:#?}", e, command));
 
-    if let Some(path) = temp_dir {
-        fs::remove_dir_all(&path).unwrap();
-    }
+    // Cleanup
+    fs::remove_dir_all(&temp_dir).unwrap();
 
     if !status.success() {
         std::process::exit(status.code().unwrap_or(1));
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::fs;
+
+    #[test]
+    fn test_stage_files() {
+        let base = env::temp_dir().join(format!("mdbook_test_{}", std::process::id()));
+        let src_dir = base.join("src_dir");
+        let workdir = base.join("workdir");
+        fs::create_dir_all(&src_dir).unwrap();
+        fs::create_dir_all(&workdir).unwrap();
+
+        let config = src_dir.join("book.toml");
+        fs::write(&config, "title = 'test'").unwrap();
+
+        let file1 = src_dir.join("file1.md");
+        fs::write(&file1, "content1").unwrap();
+
+        let sub_dir = src_dir.join("sub");
+        fs::create_dir_all(&sub_dir).unwrap();
+        let file2 = sub_dir.join("file2.md");
+        fs::write(&file2, "content2").unwrap();
+
+        let mut srcs = BTreeMap::new();
+        srcs.insert(file1.clone(), PathBuf::from("file1.md"));
+        srcs.insert(file2.clone(), PathBuf::from("sub/file2.md"));
+
+        stage_files_internal(&workdir, &config, &srcs);
+
+        assert!(workdir.join("book.toml").exists());
+        assert!(workdir.join("file1.md").exists());
+        assert!(workdir.join("sub/file2.md").exists());
+
+        #[cfg(target_family = "unix")]
+        {
+            assert!(fs::symlink_metadata(workdir.join("file1.md"))
+                .unwrap()
+                .file_type()
+                .is_symlink());
+        }
+
+        fs::remove_dir_all(&base).unwrap();
+    }
+}

extensions/mdbook/test/external_srcs/content/book.toml

@@ -1,5 +1,6 @@
 [book]
 authors = ["Test Author"]
 language = "en"
+# Normal src path, works because srcs and book are in the same package here.
 src = "src"
 title = "External Book"

extensions/mdbook/test/external_srcs/local_book_mixed/book.toml

@@ -1,5 +1,6 @@
 [book]
 authors = ["Test Author"]
 language = "en"
-src = "../content/src"
+# This path is relative to the workdir root because of path flattening.
+src = "src"
 title = "Local Book Mixed"

extensions/mdbook/test/flattening/BUILD.bazel

@@ -0,0 +1,35 @@
+load("@bazel_skylib//rules:build_test.bzl", "build_test")
+load("@rules_rust//rust:defs.bzl", "rust_test")
+load("//:defs.bzl", "mdbook", "mdbook_server")
+
+package(default_visibility = ["//visibility:public"])
+
+mdbook(
+    name = "flattening",
+    srcs = [
+        "//test/flattening/content:srcs",
+        "//test/flattening/theme:css_files",
+    ],
+    book = "//test/flattening/book:book.toml",
+)
+
+rust_test(
+    name = "flattening_test",
+    srcs = ["flattening_test.rs"],
+    data = [":flattening"],
+    edition = "2021",
+    rustc_env = {
+        "MDBOOK_FLATTENING_OUTPUT": "$(rlocationpath :flattening)",
+    },
+    deps = ["@rules_rust//rust/runfiles"],
+)
+
+mdbook_server(
+    name = "flattening_server",
+    book = ":flattening",
+)
+
+build_test(
+    name = "flattening_build_test",
+    targets = [":flattening"],
+)

extensions/mdbook/test/flattening/book/BUILD.bazel

@@ -0,0 +1 @@
+exports_files(["book.toml"])

extensions/mdbook/test/flattening/book/book.toml

@@ -0,0 +1,10 @@
+[book]
+authors = ["Test Author"]
+language = "en"
+# This directory is in a different Bazel package, but sits here after flattening.
+src = "src"
+title = "Flattening Test"
+
+[output.html]
+# This file is also from a different Bazel package.
+additional-css = ["theme/css/custom.css"]