diff --git a/.github/llvm-matrix.js b/.github/llvm-matrix.js deleted file mode 100644 index 6cc2cad46c..0000000000 --- a/.github/llvm-matrix.js +++ /dev/null @@ -1,62 +0,0 @@ -// Description: This script reads the matrix.json file and filters -// out entries that already have an existing llvm-archive-filename on the server. -// This creates a filtered matrix used to upload new archives to the server, but -// only the ones that don't already exist. - -const fs = require('fs'); -const core = require('@actions/core'); -const { exec } = require('child_process'); - -// Function to check if a file exists on the server -const fileExists = (url) => { - return new Promise((resolve) => { - exec(`curl -s -o /dev/null -w "%{http_code}" -I "${url}"`, (error, stdout) => { - if (error) { - resolve(false); - } else { - resolve(stdout.trim() === '200'); - } - }); - }); -}; - -// Read the JSON string from the file -const matrixJson = fs.readFileSync('matrix.json', 'utf8'); - -// Parse the JSON string into an array of objects -const matrixEntries = JSON.parse(matrixJson); - -// Create a new array to store unique entries based on llvm-archive-filename -const seenFilenames = new Set(); -const uniqueEntries = []; - -(async () => { - for (const entry of matrixEntries) { - const filename = entry['llvm-archive-filename']; - if (!seenFilenames.has(filename)) { - seenFilenames.add(filename); - const url = `https://www.mrdocs.com/llvm+clang/${filename}`; - const exists = await fileExists(url); - if (!exists) { - uniqueEntries.push(entry); - } - } - } - - // Convert the new array back to a JSON string - const uniqueMatrixJson = JSON.stringify(uniqueEntries); - - // Output the filtered JSON string using core.setOutput - core.setOutput('llvm-matrix', uniqueMatrixJson); - - // Print matrix to console - console.log(`LLVM Matrix (${uniqueEntries.length} entries):`) - uniqueEntries.forEach(obj => { - console.log(`- ${obj.name}`) - for (const key in obj) { - if (key !== 'name') { - console.log(` ${key}: ${JSON.stringify(obj[key])}`) - } - } - }) -})(); \ No newline at end of file diff --git a/.github/package-lock.json b/.github/package-lock.json deleted file mode 100644 index 37aadc58f0..0000000000 --- a/.github/package-lock.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "name": ".github", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "dependencies": { - "@actions/core": "^1.11.0" - } - }, - "node_modules/@actions/core": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.11.0.tgz", - "integrity": "sha512-I21jQUzEjbZolw3jFZ/0iHGCb+rePCww9MaA0SbVFae4FpBTQWP1GIvr/m5Y6GVaxrDz7p3RhBtpBzwkA3rPSA==", - "dependencies": { - "@actions/exec": "^1.1.1", - "@actions/http-client": "^2.0.1" - } - }, - "node_modules/@actions/exec": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@actions/exec/-/exec-1.1.1.tgz", - "integrity": "sha512-+sCcHHbVdk93a0XT19ECtO/gIXoxvdsgQLzb2fE2/5sIZmWQuluYyjPQtrtTHdU1YzTZ7bAPN4sITq2xi1679w==", - "dependencies": { - "@actions/io": "^1.0.1" - } - }, - "node_modules/@actions/http-client": { - "version": "2.2.3", - "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.2.3.tgz", - "integrity": "sha512-mx8hyJi/hjFvbPokCg4uRd4ZX78t+YyRPtnKWwIl+RzNaVuFpQHfmlGVfsKEJN8LwTCvL+DfVgAM04XaHkm6bA==", - "dependencies": { - "tunnel": "^0.0.6", - "undici": "^5.25.4" - } - }, - "node_modules/@actions/io": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/@actions/io/-/io-1.1.3.tgz", - "integrity": "sha512-wi9JjgKLYS7U/z8PPbco+PvTb/nRWjeoFlJ1Qer83k/3C5PHQi28hiVdeE2kHXmIL99mQFawx8qt/JPjZilJ8Q==" - }, - "node_modules/@fastify/busboy": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.1.tgz", - "integrity": "sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==", - "engines": { - "node": ">=14" - } - }, - "node_modules/tunnel": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", - "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==", - "engines": { - "node": ">=0.6.11 <=0.7.0 || >=0.7.3" - } - }, - "node_modules/undici": { - "version": "5.28.4", - "resolved": "https://registry.npmjs.org/undici/-/undici-5.28.4.tgz", - "integrity": "sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g==", - "dependencies": { - "@fastify/busboy": "^2.0.0" - }, - "engines": { - "node": ">=14.0" - } - } - } -} diff --git a/.github/package.json b/.github/package.json deleted file mode 100644 index 0c811a0a13..0000000000 --- a/.github/package.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "dependencies": { - "@actions/core": "^1.11.0" - } -} diff --git a/.github/releases-matrix.js b/.github/releases-matrix.js deleted file mode 100644 index 4391652e43..0000000000 --- a/.github/releases-matrix.js +++ /dev/null @@ -1,122 +0,0 @@ -// Description: This script reads the matrix.json file and filters -// the main entries for which we test the releases in a clean container. -// The releases are used to generate the demos in multiple environments, -// and one of these environments uploads the generated demos to the -// mrdocs.com server. - -const fs = require('fs'); -const core = require('@actions/core'); -const {exec} = require('child_process'); - -/** - * Compares the priority of two compiler entries based on their operating system. - * - * @param {Object} entryA - The first entry to compare. - * @param {string} entryA.os - The operating system of the first entry. - * @param {string} entryA.compiler - The compiler of the first entry. - * @param {Object} entryB - The second entry to compare. - * @param {string} entryB.os - The operating system of the second entry. - * @param {string} entryB.compiler - The compiler of the second entry. - * @returns {number} - A negative number if entryA has higher priority, - * a positive number if entryB has higher priority, - * or zero if they have the same priority. - */ -function compareCompilerPriority(entryA, entryB) { - // Define the compiler priority for each operating system - const compilerPriority = { - 'windows': ['msvc', 'clang', 'gcc'], - 'macos': ['clang', 'gcc'], - 'linux': ['gcc', 'clang'] - }; - - // Retrieve the priority list for the OS of entryA - const lcOs = entryA.os.toLowerCase(); - if (!compilerPriority.hasOwnProperty(lcOs)) { - throw new Error(`Unknown operating system: ${entryA.os}`) - } - const osPriority = compilerPriority[lcOs] - - // Get the index of the compiler for entryA and entryB in the priority list - const aPriority = osPriority.indexOf(entryA.compiler) - const bPriority = osPriority.indexOf(entryB.compiler) - - // If the compiler is not found in the list, assign it the lowest priority - const aFinalPriority = aPriority === -1 ? osPriority.length : aPriority - const bFinalPriority = bPriority === -1 ? osPriority.length : bPriority - - // Return the difference between the priorities of entryA and entryB - return aFinalPriority - bFinalPriority; -} - -/** - * Finds the highest priority entry among all entries that have the same specified value - * for the `mrdocs-release-package-artifact`. - * - * @param {Array} entries - The array of entries to search. - * @param {string} artifactName - The value of `mrdocs-release-package-artifact` to match. - * @returns {Object|null} - The highest priority entry or null if no matching entry is found. - */ -function findHighestPriorityEntry(entries, artifactName) { - /** @type {Object|null} */ - let highestPriorityEntry = null; - - for (const entry of entries) { - if (entry['is-main'] !== true) { - continue; - } - if (entry['mrdocs-release-package-artifact'] === artifactName) { - if (highestPriorityEntry === null) { - highestPriorityEntry = entry; - } else { - if (compareCompilerPriority(entry, highestPriorityEntry) < 0) { - highestPriorityEntry = entry; - } - } - } - } - - return highestPriorityEntry; -} - -(async () => { - // Read the JSON string from the file - const matrixJson = fs.readFileSync('matrix.json', 'utf8'); - - // Parse the JSON string into an array of objects - const matrixEntries = JSON.parse(matrixJson); - - // Create a new array to store unique entries based on llvm-archive-filename - const seenArtifactNames = new Set(); - const releaseMatrixEntries = []; - - for (const entry of matrixEntries) { - if (entry['is-main'] !== true) { - continue; - } - const artifactName = entry['mrdocs-release-package-artifact']; - if (!seenArtifactNames.has(artifactName)) { - seenArtifactNames.add(artifactName); - const highestPriorityEntry = findHighestPriorityEntry(matrixEntries, artifactName); - if (highestPriorityEntry !== null) { - releaseMatrixEntries.push(highestPriorityEntry); - } - } - } - - // Convert the new array back to a JSON string - const uniqueMatrixJson = JSON.stringify(releaseMatrixEntries); - - // Output the filtered JSON string using core.setOutput - core.setOutput('releases-matrix', uniqueMatrixJson); - - // Print matrix to console - console.log(`Releases Matrix (${releaseMatrixEntries.length} entries):`) - releaseMatrixEntries.forEach(obj => { - console.log(`- ${obj.name}`) - for (const key in obj) { - if (key !== 'name') { - console.log(` ${key}: ${JSON.stringify(obj[key])}`) - } - } - }) -})(); diff --git a/.github/scripts/compare-demos.sh b/.github/scripts/compare-demos.sh new file mode 100755 index 0000000000..2d7f03a2df --- /dev/null +++ b/.github/scripts/compare-demos.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# +# Licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# Copyright (c) 2026 Alan de Freitas (alandefreitas@gmail.com) +# +# Official repository: https://github.com/cppalliance/mrdocs +# + +# Compare current demos against previous develop demos. +# Expected env vars: GITHUB_OUTPUT +set -x + +LOCAL_DEMOS_DIR="./demos/" +PREV_DEMOS_DIR="./demos-previous/" +DIFF_DIR="./demos-diff/" + +if [[ ! -d $PREV_DEMOS_DIR || -z $(ls -A $PREV_DEMOS_DIR) ]]; then + echo "No previous demos found." + echo "diff=false" >> "$GITHUB_OUTPUT" + exit 0 +fi + +mkdir -p $PREV_DEMOS_DIR $DIFF_DIR + +find $PREV_DEMOS_DIR -type f | while read previous_file; do + local_file="${LOCAL_DEMOS_DIR}${previous_file#$PREV_DEMOS_DIR}" + diff_output="$DIFF_DIR${previous_file#$PREV_DEMOS_DIR}" + if [[ -f $local_file ]]; then + mkdir -p "$(dirname "$diff_output")" + diff "$previous_file" "$local_file" > "$diff_output" + if [[ ! -s $diff_output ]]; then + rm "$diff_output" + fi + else + echo "LOCAL FILE $local_file DOES NOT EXITS." > "$diff_output" + echo "PREVIOUS CONTENT OF THE FILE WAS:" >> "$diff_output" + cat "$previous_file" >> "$diff_output" + fi +done + +find $LOCAL_DEMOS_DIR -type f | while read local_file; do + previous_file="${PREV_DEMOS_DIR}${local_file#$LOCAL_DEMOS_DIR}" + diff_output="$DIFF_DIR${local_file#$LOCAL_DEMOS_DIR}" + if [[ ! -f $previous_file ]]; then + echo "PREVIOUS $previous_file DOES NOT EXIST." > "$diff_output" + echo "IT HAS BEEN INCLUDED IN THIS VERSION." >> "$diff_output" + echo "NEW CONTENT OF THE FILE IS:" >> "$diff_output" + fi +done + +if [[ -z $(ls -A $DIFF_DIR) ]]; then + echo "No differences found." + echo "diff=false" >> "$GITHUB_OUTPUT" +else + N_FILES=$(find $DIFF_DIR -type f | wc -l) + echo "Differences found in $N_FILES output files." + echo "diff=true" >> "$GITHUB_OUTPUT" +fi diff --git a/.github/scripts/generate-demos.sh b/.github/scripts/generate-demos.sh new file mode 100755 index 0000000000..23a81b6553 --- /dev/null +++ b/.github/scripts/generate-demos.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# +# Licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# Copyright (c) 2026 Alan de Freitas (alandefreitas@gmail.com) +# +# Official repository: https://github.com/cppalliance/mrdocs +# + +# Generate demos for each (project, variant, generator) combination. +# Expected env vars: GITHUB_EVENT_NAME, RUNNER_OS, GITHUB_ENV +set -x + +declare -a generators=("adoc") +if [[ "$GITHUB_EVENT_NAME" != 'pull_request' ]]; then + generators+=("xml" "html") +fi + +demo_failures="" + +for variant in single multi; do + for generator in "${generators[@]}"; do + [[ $generator = xml && $variant = multi ]] && continue + [[ $variant = multi ]] && multipage="true" || multipage="false" + for project_args in \ + "boost-url|$(pwd)/boost/libs/url/doc/mrdocs.yml|../CMakeLists.txt" \ + "beman-optional|$(pwd)/beman-optional/docs/mrdocs.yml|" \ + "nlohmann-json|$(pwd)/nlohmann-json/docs/mrdocs.yml|" \ + "mp-units|$(pwd)/mp-units/docs/mrdocs.yml|" \ + "fmt|$(pwd)/fmt/doc/mrdocs.yml|" \ + "mrdocs|$(pwd)/docs/mrdocs.yml|$(pwd)/CMakeLists.txt" \ + ; do + IFS='|' read -r project config extra <<< "$project_args" + outdir="$(pwd)/demos/$project/$variant/$generator" + cmd=(mrdocs --config="$config" $extra --output="$outdir" --multipage=$multipage --generator="$generator" --log-level=debug) + if ! "${cmd[@]}"; then + echo "FAILED: $project/$variant/$generator" + demo_failures="$demo_failures $project/$variant/$generator\n ${cmd[*]}\n" + rm -rf "$outdir" + fi + done + done + + if [[ "$RUNNER_OS" == 'Linux' ]]; then + for project in boost-url beman-optional mrdocs fmt nlohmann-json mp-units; do + root="$(pwd)/demos/$project/$variant" + src="$root/adoc" + dst="$root/adoc-asciidoc" + stylesheet="$(pwd)/share/mrdocs/addons/generator/common/layouts/style.css" + + [[ -d "$src" ]] || continue + + mkdir -p "$dst" + + find "$src" -type f -name '*.adoc' -print0 | + while IFS= read -r -d '' f; do + rel="${f#"$src/"}" + outdir="$dst/$(dirname "$rel")" + mkdir -p "$outdir" + asciidoctor -a stylesheet="${stylesheet}" -D "$outdir" "$f" + done + done + fi +done + +tar -cjf "$(pwd)/demos.tar.gz" -C "$(pwd)/demos" --strip-components 1 . +echo "demos_path=$(pwd)/demos.tar.gz" >> "$GITHUB_ENV" + +if [[ -n "$demo_failures" ]]; then + echo "The following demos failed:" + printf "$demo_failures" + exit 1 +fi diff --git a/.github/scripts/install-mrdocs-package.sh b/.github/scripts/install-mrdocs-package.sh new file mode 100755 index 0000000000..269df626b1 --- /dev/null +++ b/.github/scripts/install-mrdocs-package.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# +# Licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# Copyright (c) 2026 Alan de Freitas (alandefreitas@gmail.com) +# +# Official repository: https://github.com/cppalliance/mrdocs +# + +# Extract and install MrDocs from a build artifact package. +# Expected env vars: RUNNER_OS, GITHUB_ENV, GITHUB_PATH +set -eu + +MRDOCS_INSTALL_DIR="$(pwd)/.install/mrdocs" +mkdir -p "$MRDOCS_INSTALL_DIR" + +rm -rf packages/_CPack_Packages + +echo "::group::Extract MrDocs package to $MRDOCS_INSTALL_DIR" +if [[ "$RUNNER_OS" != 'Windows' ]]; then + find packages -maxdepth 1 -name 'MrDocs-*.tar.gz' -exec tar -xzf {} -C "$MRDOCS_INSTALL_DIR" --strip-components=1 \; +else + package=$(find packages -maxdepth 1 -name "MrDocs-*.7z" -print -quit) + filename=$(basename "$package") + name="${filename%.*}" + 7z x "${package}" -o"${MRDOCS_INSTALL_DIR}" + set +e + robocopy "${MRDOCS_INSTALL_DIR}/${name}" "${MRDOCS_INSTALL_DIR}" //move //e //np //nfl + exit_code=$? + set -e + if (( exit_code >= 8 )); then + exit 1 + fi +fi +echo "::endgroup::" + +echo "::group::Verify installation" +echo "Install root: $MRDOCS_INSTALL_DIR" +echo "" +echo "Top-level contents:" +ls -1 "$MRDOCS_INSTALL_DIR" +echo "" +echo "Binaries:" +ls -1 "$MRDOCS_INSTALL_DIR/bin/" +echo "::endgroup::" + +echo "::group::Export environment variables" +echo "MRDOCS_ROOT=$MRDOCS_INSTALL_DIR" +echo "MRDOCS_ROOT=$MRDOCS_INSTALL_DIR" >> "$GITHUB_ENV" +echo "PATH += $MRDOCS_INSTALL_DIR/bin" +echo "$MRDOCS_INSTALL_DIR/bin" >> "$GITHUB_PATH" +echo "::endgroup::" + +$MRDOCS_INSTALL_DIR/bin/mrdocs --version diff --git a/.github/scripts/verify-snippets.sh b/.github/scripts/verify-snippets.sh new file mode 100755 index 0000000000..8e8e43593a --- /dev/null +++ b/.github/scripts/verify-snippets.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# +# Licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# Copyright (c) 2026 Alan de Freitas (alandefreitas@gmail.com) +# +# Official repository: https://github.com/cppalliance/mrdocs +# + +# Verify that snippet .cpp files match golden tests. +set -euo pipefail +shopt -s nullglob + +SRC="docs/website/snippets" +DST="test-files/golden-tests/snippets" + +[[ -d "$SRC" ]] || { echo "Source directory not found: $SRC"; exit 2; } +[[ -d "$DST" ]] || { echo "Destination directory not found: $DST"; exit 2; } + +missing=() +mismatched=() + +while IFS= read -r -d '' src; do + rel="${src#$SRC/}" + dst="$DST/$rel" + if [[ ! -f "$dst" ]]; then + missing+=("$rel") + continue + fi + if ! git diff --no-index --ignore-cr-at-eol --quiet -- "$src" "$dst"; then + mismatched+=("$rel") + fi +done < <(find "$SRC" -type f -name '*.cpp' -print0) + +if (( ${#missing[@]} || ${#mismatched[@]} )); then + if (( ${#missing[@]} )); then + echo "Missing corresponding golden files:" + printf ' %s\n' "${missing[@]}" + fi + if (( ${#mismatched[@]} )); then + echo "Content mismatches:" + printf ' %s\n' "${mismatched[@]}" + fi + exit 1 +fi +echo "All snippet .cpp files are present and match." diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml new file mode 100644 index 0000000000..c23ca0ecce --- /dev/null +++ b/.github/workflows/ci-build.yml @@ -0,0 +1,168 @@ +# +# Licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# Copyright (c) 2026 Alan de Freitas (alandefreitas@gmail.com) +# +# Official repository: https://github.com/cppalliance/mrdocs +# + +name: Build & Test + +on: + workflow_call: + inputs: + matrix: + description: 'JSON-encoded build matrix from cpp-matrix' + type: string + required: true + +jobs: + build: + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(inputs.matrix) }} + + defaults: + run: + shell: bash + + name: ${{ matrix.name }} + runs-on: ${{ matrix.runs-on }} + container: ${{ matrix.container }} + env: ${{ matrix.env }} + steps: + - name: Container Bootstrap + uses: alandefreitas/cpp-actions/container-bootstrap@v1.9.4 + + - name: Setup C++ + uses: alandefreitas/cpp-actions/setup-cpp@v1.9.4 + id: setup-cpp + with: + compiler: ${{ matrix.compiler }} + version: ${{ matrix.version }} + + - name: Install System Packages + uses: alandefreitas/cpp-actions/package-install@v1.9.4 + id: package-install + with: + packages: ${{ matrix.install }} + apt-get-source-keys: ${{ matrix.coverage == 'true' && 'https://apt.llvm.org/llvm-snapshot.gpg.key' || '' }} + apt-get-sources: ${{ matrix.coverage == 'true' && 'deb https://apt.llvm.org/noble/ llvm-toolchain-noble-21 main' || '' }} + + - name: Clone MrDocs + uses: actions/checkout@v4 + + - name: Restore LLVM Cache + id: llvm-cache + uses: actions/cache/restore@v4 + with: + path: build/third-party/install/llvm + key: ${{ matrix.llvm-cache-key }} + + # Install third-party deps (LLVM, JerryScript, Lua, Boost, libxml2). + # --skip-build: only install deps, the MrDocs build is done by cmake-workflow. + # --cache-dir: flatten install paths to / for stable cache keys. + # --env-file: write _ROOT paths so we can inject them into GITHUB_ENV. + # Bootstrap writes BOOTSTRAP_REBUILT= when it rebuilds any recipe. + - name: Bootstrap Dependencies + run: | + python3 bootstrap.py --yes \ + --skip-build --no-run-configs \ + --build-type "${{ matrix.build-type }}" \ + --cc "${{ steps.setup-cpp.outputs.cc || matrix.cc }}" \ + --cxx "${{ steps.setup-cpp.outputs.cxx || matrix.cxx }}" \ + --cache-dir build/third-party/install \ + --env-file "$RUNNER_TEMP/bootstrap-env.txt" \ + ${{ matrix.bootstrap-sanitizer && format('--sanitizer {0}', matrix.bootstrap-sanitizer) || '' }} \ + ${{ matrix.common-ccflags && format('--cflags="{0}" --cxxflags="{0}"', matrix.common-ccflags) || '' }} + cat "$RUNNER_TEMP/bootstrap-env.txt" >> "$GITHUB_ENV" + + # Save the LLVM cache when bootstrap rebuilt it (cache miss or + # stale stamp). BOOTSTRAP_REBUILT lists rebuilt recipes. + # + # `actions:write` (needed for the DELETE call below) is only granted + # to workflows on the base repo, so fork PRs get a read-only token + # and can't delete or overwrite the base-branch cache anyway. Skip + # the whole save path on fork PRs. + - name: Delete Stale LLVM Cache + if: | + contains(env.BOOTSTRAP_REBUILT, 'llvm') + && steps.llvm-cache.outputs.cache-hit == 'true' + && (github.event_name != 'pull_request' + || github.event.pull_request.head.repo.full_name == github.repository) + env: + GH_TOKEN: ${{ github.token }} + run: | + code=$(curl -s -o /dev/null -w "%{http_code}" -X DELETE \ + -H "Authorization: Bearer $GH_TOKEN" \ + "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/actions/caches?key=${{ matrix.llvm-cache-key }}") + case "$code" in + 200) echo "Deleted old cache for key ${{ matrix.llvm-cache-key }}" ;; + 404) echo "No existing cache for key ${{ matrix.llvm-cache-key }}" ;; + 403) echo "::warning::Skipping cache delete: token lacks actions:write (HTTP 403)" ;; + *) echo "::error::Failed to delete cache (HTTP $code)"; exit 1 ;; + esac + + - name: Save Updated LLVM Cache + if: | + contains(env.BOOTSTRAP_REBUILT, 'llvm') + && (github.event_name != 'pull_request' + || github.event.pull_request.head.repo.full_name == github.repository) + uses: actions/cache/save@v4 + with: + path: build/third-party/install/llvm + key: ${{ matrix.llvm-cache-key }} + + - name: CMake Workflow + uses: alandefreitas/cpp-actions/cmake-workflow@v1.9.4 + with: + cmake-version: '>=3.26' + cxxstd: ${{ matrix.cxxstd }} + cc: ${{ steps.setup-cpp.outputs.cc || matrix.cc }} + ccflags: ${{ matrix.ccflags }} + cxx: ${{ steps.setup-cpp.outputs.cxx || matrix.cxx }} + cxxflags: ${{ matrix.cxxflags }} ${{ env.BOOTSTRAP_CXXFLAGS }} + ldflags: ${{ env.BOOTSTRAP_LDFLAGS }} + generator: Ninja + build-type: ${{ matrix.build-type }} + install-prefix: .local + extra-args: -D MRDOCS_EXPENSIVE_TESTS=${{ matrix.is-bottleneck && 'OFF' || 'ON' }} + export-compile-commands: true + run-tests: true + install: true + package: ${{ matrix.is-main }} + package-dir: packages + package-generators: ${{ matrix.mrdocs-package-generators }} + package-artifact: false + ctest-timeout: 9000 + + # Upload packages for ci-releases.yml to pick up + - name: Upload GitHub Release Artifacts + if: matrix.is-release-build == 'true' + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.mrdocs-release-package-artifact }} + path: | + build/packages + !build/packages/_CPack_Packages + retention-days: 1 + + - name: FlameGraph + uses: alandefreitas/cpp-actions/flamegraph@v1.9.4 + if: matrix.time-trace + with: + build-dir: build + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Process Coverage + if: matrix.coverage == 'true' + uses: alandefreitas/cpp-actions/process-coverage@v1.9.4 + with: + cxx: ${{ steps.setup-cpp.outputs.cxx || matrix.cxx }} + build-dir: build + html-report: true + codecov-token: ${{ secrets.CODECOV_TOKEN }} + codecov-flags: cpp diff --git a/.github/workflows/ci-matrix.yml b/.github/workflows/ci-matrix.yml new file mode 100644 index 0000000000..e72acb6dd6 --- /dev/null +++ b/.github/workflows/ci-matrix.yml @@ -0,0 +1,120 @@ +# +# Licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# Copyright (c) 2026 Alan de Freitas (alandefreitas@gmail.com) +# +# Official repository: https://github.com/cppalliance/mrdocs +# + +name: Generate Test Matrix + +on: + workflow_call: + outputs: + matrix: + description: 'JSON-encoded build matrix' + value: ${{ jobs.generate.outputs.matrix }} + submatrices: + description: 'JSON-encoded sub-matrices object' + value: ${{ jobs.generate.outputs.submatrices }} + +jobs: + generate: + runs-on: ubuntu-24.04 + container: + image: ubuntu:24.04 + name: Generate Test Matrix + outputs: + matrix: ${{ steps.cpp-matrix.outputs.matrix }} + submatrices: ${{ steps.cpp-matrix.outputs.submatrices }} + steps: + - name: Generate Test Matrix + uses: alandefreitas/cpp-actions/cpp-matrix@v1.9.4 + id: cpp-matrix + with: + compilers: | + gcc >=14 + clang >=18 + msvc >=14.40 + apple-clang >=17 + standards: '23' + main-entry-factors: | + clang Coverage + latest-factors: | + clang UBSan + factors: | + gcc UBSan + clang ASan MSan + apple-clang UBSan ASan + build-types: | + gcc: Release + clang: Release + apple-clang: Release + msvc: RelWithDebInfo + append-install: | + gcc: cmake ninja-build unzip openjdk-11-jdk-headless libncurses-dev nodejs npm + clang: cmake ninja-build unzip openjdk-11-jdk-headless libncurses-dev libstdc++-14-dev nodejs npm + append-ccflags: | + gcc: -Werror -Wall + gcc is-main: -static + clang: -Werror -Wall + apple-clang: -Werror -Wall + msvc: /WX /W4 + append-cxxflags: | + gcc: -Werror -Wall + gcc is-main: -static + clang: -Werror -Wall + apple-clang: -Werror -Wall + msvc: /WX /W4 + # Flags shared between dep builds (bootstrap.py) and the main MrDocs + # build. -gz=zstd compresses debug info and must be applied to both. + append-common-ccflags: | + clang: -gz=zstd + append-common-cxxflags: | + clang: -gz=zstd + extra-values: | + # libc++ runtimes: clang + (ASan or MSan) needs instrumented libc++ + # built separately from the main LLVM build + use-libcxx: {{and (ieq compiler 'clang') (or msan asan)}} + libcxx-runtimes: {{select (ne compiler 'msvc') "libcxx;libcxxabi" "libcxx"}} + llvm-runtimes: {{{select (ine use-libcxx 'true') libcxx-runtimes ""}}} + + # LLVM build configuration + llvm-hash: dc4cef81d47c7bc4a3c4d58fbacf8a6359683fae + llvm-short-hash: {{{substr llvm-hash 0 7}}} + llvm-preset-build-type: {{{lowercase build-type}}} + llvm-preset-os: {{select (ieq os 'windows') "win" "unix"}} + llvm-preset: {{{ llvm-preset-build-type }}}-{{{ llvm-preset-os }}} + + # LLVM cache key: encodes hash, build type, OS, and sanitizer + # so each distinct LLVM configuration gets its own cache entry. + # Clang + (ASan or MSan) appends compiler version and sanitizer + # because those builds include instrumented libc++ runtimes. + llvm-os-key: {{{replace (lowercase (default container runs-on)) ":" "-" "." "-"}}} + # ASan, MSan, and TSan need different LLVM builds (instrumented deps). + # UBSan shares the plain LLVM cache (compile-time checks only). + llvm-cache-sanitizer: {{select asan "asan" msan "msan" tsan "tsan"}} + llvm-cache-sanitizer-suffix: {{#if llvm-cache-sanitizer}}-{{{compiler}}}-{{{version}}}-{{{llvm-cache-sanitizer}}}{{/if}} + llvm-cache-key: llvm-{{{llvm-short-hash}}}-{{{llvm-preset-build-type}}}-{{{llvm-os-key}}}{{{llvm-cache-sanitizer-suffix}}} + + # Maps sanitizer factor to LLVM_USE_SANITIZER cmake value + llvm-sanitizer-config: {{#if (includes (list "clang" "apple-clang") compiler)}}{{select asan "Address" msan "MemoryWithOrigins"}}{{/if}} + + # Sanitizer name for bootstrap's --sanitizer flag + bootstrap-sanitizer: {{select asan "address" msan "memory" ubsan "undefined" tsan "thread"}} + + # Packaging and release configuration + is-release-build: {{and is-main (includes (list "gcc" "msvc" "apple-clang") compiler)}} + mrdocs-package-generators: {{select (ieq os 'windows') "7Z ZIP WIX" "TGZ TXZ"}} + mrdocs-release-package-artifact: release-packages-{{{ lowercase os }}} + + # Bottleneck builds: skip expensive tests that ended up being redundant to save CI time + is-bottleneck: {{#if (or msan (and (ieq compiler 'apple-clang') (or asan ubsan))) }}true{{/if}} + + submatrices: | + releases: {{ieq is-release-build 'true'}} + trace-commands: true + github-token: ${{ secrets.GITHUB_TOKEN }} + diff --git a/.github/workflows/ci-releases.yml b/.github/workflows/ci-releases.yml new file mode 100644 index 0000000000..2167bc9634 --- /dev/null +++ b/.github/workflows/ci-releases.yml @@ -0,0 +1,350 @@ +# +# Licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# Copyright (c) 2026 Alan de Freitas (alandefreitas@gmail.com) +# +# Official repository: https://github.com/cppalliance/mrdocs +# + +name: Releases + +on: + workflow_call: + inputs: + submatrices: + description: 'JSON-encoded sub-matrices object from cpp-matrix' + type: string + required: true + +jobs: + releases: + # `fromJSON(...).releases` deserialises to an array, so comparing it + # to the literal string '[]' never matches. Re-serialise with toJSON + # so the emptiness check actually triggers and the job is skipped + # cleanly on non-release runs (PRs, sanitizer-only matrices, etc.). + if: ${{ toJSON(fromJSON(inputs.submatrices).releases) != '[]' }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(inputs.submatrices).releases }} + + defaults: + run: + shell: bash + + name: ${{ matrix.os }} + runs-on: ${{ matrix.runs-on }} + container: ${{ matrix.container }} + permissions: + contents: write + + steps: + - name: Container Bootstrap + uses: alandefreitas/cpp-actions/container-bootstrap@v1.9.4 + + - name: Install packages + uses: alandefreitas/cpp-actions/package-install@v1.9.4 + id: package-install + with: + apt-get: build-essential asciidoctor cmake bzip2 git rsync + + - name: Clone MrDocs + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + + - name: Install Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Setup Ninja + uses: seanmiddleditch/gha-setup-ninja@v5 + if: ${{ runner.os == 'Windows' }} + + - name: Setup C++ + uses: alandefreitas/cpp-actions/setup-cpp@v1.9.4 + id: setup-cpp + with: + compiler: ${{ matrix.compiler }} + version: ${{ matrix.version }} + + - name: Download MrDocs package + uses: actions/download-artifact@v4 + with: + name: ${{ matrix.mrdocs-release-package-artifact }} + path: packages + + - name: Install MrDocs from Package + run: .github/scripts/install-mrdocs-package.sh + + - name: Clone Boost.URL + uses: alandefreitas/cpp-actions/boost-clone@v1.9.4 + id: boost-url-clone + with: + branch: develop + modules: url + boost-dir: boost + modules-scan-paths: '"test example"' + modules-exclude-paths: '' + trace-commands: true + + - name: Generate Landing Page + working-directory: docs/website + run: | + npm ci + node render.js + mkdir -p ../../build/website + cp index.html ../../build/website/index.html + cp robots.txt ../../build/website/robots.txt + cp styles.css ../../build/website/styles.css + cp -r assets ../../build/website/assets + + - name: Generate Antora UI + working-directory: docs/ui + run: | + # This playbook renders the documentation + # content for the website. It includes + # master, develop, and tags. + GH_TOKEN="${{ secrets.GITHUB_TOKEN }}" + export GH_TOKEN + npm ci + npx gulp lint + npx gulp + + # Website publishing gate: + # - Only publish on pushes to master/develop and on tags + # - Only on Linux runners (the publish steps assume GNU tooling) + # + # Use `if: env.PUBLISH_WEBSITE == 'true'` for all steps that + # write into `build/website/` and get deployed to the website. + - name: Set website publish gate + run: | + is_publish_ref='false' + if [[ "${{ github.event_name }}" == 'push' ]]; then + if [[ "${{ github.ref_name }}" == 'master' || "${{ github.ref_name }}" == 'develop' ]]; then + is_publish_ref='true' + fi + if [[ "${{ github.ref }}" == refs/tags/* ]]; then + is_publish_ref='true' + fi + fi + + publish_website="$is_publish_ref" + if [[ "${{ runner.os }}" != 'Linux' ]]; then + publish_website='false' + fi + + { + echo "IS_PUBLISH_REF=$is_publish_ref" + echo "PUBLISH_WEBSITE=$publish_website" + } >> "$GITHUB_ENV" + + - name: Ensure all refs for Antora + if: env.PUBLISH_WEBSITE == 'true' + run: | + set -euo pipefail + # Make sure Antora sees every branch and tag from the upstream repo, + # regardless of who triggered the workflow. + git remote set-url origin https://github.com/cppalliance/mrdocs.git + git fetch --prune --prune-tags origin \ + '+refs/heads/*:refs/remotes/origin/*' \ + '+refs/tags/*:refs/tags/*' + + - name: Generate Remote Documentation + # This step fetches and builds develop, master and all tags. That's + # unrelated to a PR, and is only needed for website publishing. So, skip + # it for a PR. + if: github.event_name != 'pull_request' + working-directory: docs + run: | + # This playbook renders the documentation + # content for the website. It includes + # master, develop, and tags. + GH_TOKEN="${{ secrets.GITHUB_TOKEN }}" + export GH_TOKEN + set -x + npm ci + npx antora --clean --fetch antora-playbook.yml --log-level=debug + mkdir -p ../build/website/docs + cp -vr build/site/* ../build/website/docs + + - name: Upload Website as Artifact + uses: actions/upload-artifact@v4 + with: + name: Website ${{ runner.os }} + path: build/website + retention-days: 30 + + - name: Generate Local Documentation + working-directory: docs + run: | + # This playbook allows us to render the + # documentation content and visualize it + # before a workflow that pushes to the + # website is triggered. + GH_TOKEN="${{ secrets.GITHUB_TOKEN }}" + export GH_TOKEN + set -x + npm ci + npx antora antora-playbook.yml --attribute branchesarray=HEAD --stacktrace --log-level=debug + mkdir -p ../build/docs-local + cp -vr build/site/* ../build/docs-local + + - name: Clone Beman.Optional + uses: actions/checkout@v4 + with: + repository: steve-downey/optional + ref: main + path: beman-optional + + - name: Clone Fmt + uses: actions/checkout@v4 + with: + repository: fmtlib/fmt + ref: main + path: fmt + + - name: Clone Nlohmann.Json + uses: actions/checkout@v4 + with: + repository: nlohmann/json + ref: develop + path: nlohmann-json + + - name: Clone MpUnits + uses: actions/checkout@v4 + with: + repository: mpusz/mp-units + ref: master + path: mp-units + + - name: Patch Demo Projects + shell: bash + run: | + set -euo pipefail + set -x + + for project in beman-optional fmt nlohmann-json mp-units; do + src="./examples/third-party/$project" + dst="./$project" + + [ -d "$src" ] || { echo "Source not found: $src" >&2; exit 1; } + mkdir -p "$dst" + + # Mirror contents of $src into $dst, overwriting existing files + tar -C "$src" -cf - . | tar -C "$dst" -xpf - + done + + + - name: Generate Demos + run: .github/scripts/generate-demos.sh + + - name: Upload Demos as Artifacts + uses: actions/upload-artifact@v4 + with: + name: demos${{ (contains(fromJSON('["master", "develop"]'), github.ref_name ) && format('-{0}', github.ref_name)) || '' }}-${{ runner.os }} + path: demos.tar.gz + # develop and master are retained for longer so that they can be compared + retention-days: ${{ contains(fromJSON('["master", "develop"]'), github.ref_name) && '30' || '1' }} + + - name: Download Previous Demos + if: startsWith(github.ref, 'refs/tags/') && runner.os == 'Linux' + id: download-prev-demos + uses: actions/download-artifact@v4 + continue-on-error: true + with: + name: demos-develop-${{ runner.os }} + path: demos-previous + + - name: Compare demos + if: startsWith(github.ref, 'refs/tags/') && steps.download-prev-demos.outputs.cache-hit == 'true' && runner.os == 'Linux' + id: compare-demos + run: .github/scripts/compare-demos.sh + + - name: Upload Demo Diff as Artifacts + if: startsWith(github.ref, 'refs/tags/') && steps.download-prev-demos.outputs.cache-hit == 'true' && steps.compare-demos.outputs.diff == 'true' && runner.os == 'Linux' + uses: actions/upload-artifact@v4 + with: + name: demos-diff + path: demos-diff + retention-days: 30 + + - name: Publish Website to GitHub Pages + if: env.PUBLISH_WEBSITE == 'true' + uses: peaceiris/actions-gh-pages@v3 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: build/website + force_orphan: true + + - name: Publish website + if: env.PUBLISH_WEBSITE == 'true' + env: + SSH_AUTH_SOCK: /tmp/ssh_agent.sock + run: | + set -euvx + # Add SSH key + mkdir -p /home/runner/.ssh + ssh-keyscan dev-websites.cpp.al >> /home/runner/.ssh/known_hosts + chmod 600 /home/runner/.ssh/known_hosts + echo "${{ secrets.DEV_WEBSITES_SSH_KEY }}" > /home/runner/.ssh/github_actions + chmod 600 /home/runner/.ssh/github_actions + ssh-agent -a $SSH_AUTH_SOCK > /dev/null + ssh-add /home/runner/.ssh/github_actions + + rsyncopts=(--recursive --delete --links --times --chmod=D0755,F0755 --compress --compress-choice=zstd --rsh="ssh -o StrictHostKeyChecking=no" --human-readable) + website_dir="ubuntu@dev-websites.cpp.al:/var/www/mrdox.com" + demo_dir="$website_dir/demos/${{ github.ref_name }}" + + # Copy files: This step will copy the landing page and the documentation to www.mrdocs.com + time rsync "${rsyncopts[@]}" --exclude=demos/ --exclude=roadmap/ $(pwd)/build/website/ "$website_dir"/ + + # Copy demos: This step will copy the demos to www.mrdocs.com/demos + time rsync "${rsyncopts[@]}" $(pwd)/demos/ "$demo_dir"/ + + - name: Create changelog + uses: alandefreitas/cpp-actions/create-changelog@v1.9.4 + with: + output-path: CHANGELOG.md + thank-non-regular: ${{ startsWith(github.ref, 'refs/tags/') }} + github-token: ${{ secrets.GITHUB_TOKEN }} + limit: 150 + update-summary: ${{ runner.os == 'Linux' && 'true' || 'false' }} + + # For non-tag publishes (the develop-release and master-release + # rolling releases), strip the project version out of the package + # filenames and insert the branch name instead. Subsequent pushes + # to the same branch then overwrite the existing GitHub-release + # assets cleanly; without this, a version bump would leave the + # previous version's files behind as stale assets. Tag releases + # keep their versioned filenames. + - name: Rebrand branch packages with the ref name + if: env.IS_PUBLISH_REF == 'true' && !startsWith(github.ref, 'refs/tags/') + run: | + set -euxo pipefail + cd packages + for f in MrDocs-*.*.*-*.*; do + [ -e "$f" ] || continue + new=$(echo "$f" | sed -E 's|^MrDocs-[0-9]+\.[0-9]+\.[0-9]+-|MrDocs-${{ github.ref_name }}-|') + mv -- "$f" "$new" + done + + - name: Create GitHub Package Release + if: env.IS_PUBLISH_REF == 'true' + uses: softprops/action-gh-release@v2 + with: + # After the rebrand step, branch packages have the form + # MrDocs--. (no semver), so the glob + # is broader than the historical MrDocs-*.*.*-*.*. + files: packages/MrDocs-*-*.* + fail_on_unmatched_files: true + name: ${{ github.ref_name || github.ref }} + tag_name: ${{ github.ref_name || github.ref }}${{ ((!startsWith(github.ref, 'refs/tags/')) && '-release') || '' }} + body_path: CHANGELOG.md + prerelease: false + draft: false + token: ${{ github.token }} diff --git a/.github/workflows/ci-utility-tests.yml b/.github/workflows/ci-utility-tests.yml new file mode 100644 index 0000000000..f1a2069fff --- /dev/null +++ b/.github/workflows/ci-utility-tests.yml @@ -0,0 +1,60 @@ +# +# Licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# Copyright (c) 2026 Alan de Freitas (alandefreitas@gmail.com) +# +# Official repository: https://github.com/cppalliance/mrdocs +# + +name: Utility Tests + +on: + workflow_call: + +jobs: + utility-tests: + runs-on: ubuntu-24.04 + name: Utility Tests + + steps: + - name: Clone MrDocs + uses: actions/checkout@v4 + + - name: Install Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Bootstrap Python Tests + run: | + set -eux + pip install --quiet coverage + python3 -m coverage run -m unittest discover -s util/bootstrap/tests/ + python3 -m coverage report + python3 -m coverage report --fail-under=84 + python3 -m coverage xml -o bootstrap-coverage.xml + + - name: Upload Bootstrap Coverage to Codecov + uses: codecov/codecov-action@v5 + with: + files: bootstrap-coverage.xml + flags: bootstrap + fail_ci_if_error: true + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true + + - name: Danger.js Tests + working-directory: util/danger + run: | + npm ci --ignore-scripts + npx vitest run + + - name: Check YAML schema + run: | + python3 ./util/generate-yaml-schema.py --check + npx -y -p ajv-cli -- ajv compile -s docs/mrdocs.schema.json + + - name: Verify snippet .cpp files match golden tests + run: .github/scripts/verify-snippets.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dea4850705..f31900299b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,3 +1,13 @@ +# +# Licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# Copyright (c) 2026 Alan de Freitas (alandefreitas@gmail.com) +# +# Official repository: https://github.com/cppalliance/mrdocs +# + name: Continuous Integration on: @@ -22,1151 +32,31 @@ concurrency: jobs: cpp-matrix: - runs-on: ubuntu-24.04 - container: - image: ubuntu:24.04 name: Generate Test Matrix - outputs: - matrix: ${{ steps.cpp-matrix.outputs.matrix }} - llvm-matrix: ${{ steps.llvm-matrix.outputs.llvm-matrix }} - releases-matrix: ${{ steps.releases-matrix.outputs.releases-matrix }} - steps: - - name: Install prerequisites - run: | - set -e - apt-get update - apt-get install -y git ca-certificates curl nodejs npm - if ! command -v node >/dev/null 2>&1 && command -v nodejs >/dev/null 2>&1; then - ln -s /usr/bin/nodejs /usr/bin/node - fi - - - name: Checkout - uses: actions/checkout@v4 - - - name: Generate Test Matrix - uses: alandefreitas/cpp-actions/cpp-matrix@v1.9.2 - id: cpp-matrix - with: - compilers: | - gcc >=14 - clang >=18 - msvc >=14.40 - apple-clang * - standards: '23' - latest-factors: | - gcc UBSan - clang UBSan ASan MSan Coverage - apple-clang UBSan ASan - factors: '' - runs-on: | - apple-clang: macos-15 - containers: | - clang: ubuntu:24.04 - build-types: | - gcc: Release - clang: Release - apple-clang: Release - msvc: RelWithDebInfo - install: | - gcc: git cmake ninja-build build-essential pkg-config python3 curl unzip openjdk-11-jdk-headless libncurses-dev libxml2-utils libxml2-dev - clang: git cmake ninja-build build-essential pkg-config python3 curl unzip openjdk-11-jdk-headless libncurses-dev libxml2-utils libxml2-dev libstdc++-14-dev - clang Coverage: git cmake ninja-build build-essential pkg-config python3 curl unzip openjdk-11-jdk-headless libncurses-dev libxml2-utils libxml2-dev libstdc++-14-dev elfutils llvm-21-tools - msvc: '' - extra-values: | - # libc++ runtimes: clang + (ASan or MSan) needs instrumented libc++ - # built separately from the main LLVM build - use-libcxx: {{#if (and (ieq compiler 'clang') (or msan asan)) }}true{{else}}false{{/if}} - libcxx-runtimes: libcxx{{#if (ne compiler 'msvc')}};libcxxabi{{/if}} - llvm-runtimes: {{#if (ine use-libcxx 'true') }}{{{ libcxx-runtimes }}}{{/if}} - - # LLVM build configuration - llvm-hash: dc4cef81d47c7bc4a3c4d58fbacf8a6359683fae - llvm-build-preset-prefix: {{{lowercase build-type}}} - llvm-build-preset-os: {{#if (ieq os 'windows') }}win{{else}}unix{{/if}} - llvm-build-preset: {{{ llvm-build-preset-prefix }}}-{{{ llvm-build-preset-os }}} - - # LLVM cache key: encodes hash, build type, OS, and sanitizer - # so each distinct LLVM configuration gets its own cache entry. - # Clang + (ASan or MSan) appends compiler version and sanitizer - # because those builds include instrumented libc++ runtimes. - llvm-os-key: {{#if container}}{{{ replace (replace (lowercase container) ":" "-") "." "-" }}}{{else}}{{{ replace (lowercase runs-on) "." "-" }}}{{/if}} - llvm-archive-sanitizer-str: {{#if (ieq compiler 'clang')}}{{#if ubsan}}ubsan{{else if asan}}asan{{else if msan}}msan{{/if}}{{/if}} - llvm-archive-basename: llvm-{{{ substr llvm-hash 0 7 }}}-{{{ llvm-build-preset-prefix }}}-{{{ llvm-os-key }}}{{#if (and (ieq compiler 'clang') (or msan asan)) }}-{{{ compiler }}}-{{{ version }}}-{{{ llvm-archive-sanitizer-str }}}{{/if}} - - # LLVM archive (for the llvm-releases job that uploads pre-built LLVM) - llvm-archive-extension: {{#if (ieq os 'windows') }}7z{{else}}tar.bz2{{/if}} - llvm-archive-filename: {{{ llvm-archive-basename }}}.{{{ llvm-archive-extension }}} - - # Maps sanitizer factor to LLVM_USE_SANITIZER cmake value - llvm-sanitizer-config: {{#if (and (ne compiler 'clang') (ne compiler 'apple-clang'))}}{{else if asan}}Address{{else if msan}}MemoryWithOrigins{{/if}} - - # Compiler and linker flags passed to the MrDocs build (not deps). - # common-flags-base: clang-specific flags (e.g. -gz=zstd for compression) - # common-flags: adds MSan origin tracking on top of base - # mrdocs-flags: warnings-as-errors, static linking (gcc), coverage (clang) - warning-flags: {{#if (eq compiler 'msvc') }}/WX /W4 {{else}}-Werror -Wall {{/if}} - common-flags-base: {{#if (ieq compiler 'clang')}}-gz=zstd {{/if}} - common-flags: {{{ common-flags-base }}}{{#if msan }}-fsanitize-memory-track-origins {{/if}} - common-ccflags: {{{ ccflags }}} {{{ common-flags }}} - mrdocs-flags: {{{ warning-flags }}}{{#if (and (eq compiler 'gcc') (not asan)) }}-static {{/if}}{{#if (and (ieq compiler 'clang') coverage)}}-fprofile-instr-generate -fcoverage-mapping {{/if}} - mrdocs-ccflags: {{{ common-ccflags }}} {{{ mrdocs-flags }}} - - # Packaging and release configuration - mrdocs-package-generators: {{#if (ieq os 'windows') }}7Z ZIP WIX{{else}}TGZ TXZ{{/if}} - mrdocs-release-package-artifact: release-packages-{{{ lowercase os }}} - - # Bottleneck builds: skip expensive tests to save CI time - is-bottleneck: {{#if (or msan (and (ieq compiler 'apple-clang') (or asan ubsan))) }}true{{/if}} - - # Sanitizer name for bootstrap's --sanitizer flag - bootstrap-sanitizer: {{#if asan}}address{{else if msan}}memory{{else if ubsan}}undefined{{else if tsan}}thread{{/if}} - output-file: matrix.json - trace-commands: true - github-token: ${{ secrets.GITHUB_TOKEN }} - - # Set up the version as expected by the LLVM matrix script and @actions/core - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: 20 - - - name: Generate LLVM Test Matrix - id: llvm-matrix - run: | - set -x - cd .github - npm ci - cd .. - node .github/llvm-matrix.js - - - name: Generate Releases Test Matrix - id: releases-matrix - run: | - set -x - cd .github - npm ci - cd .. - node .github/releases-matrix.js + uses: ./.github/workflows/ci-matrix.yml + secrets: inherit build: - needs: cpp-matrix - - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.cpp-matrix.outputs.matrix) }} - - defaults: - run: - shell: bash - - name: ${{ matrix.name }} - runs-on: ${{ matrix.runs-on }} - container: ${{ matrix.container }} - env: ${{ matrix.env }} + name: Build & Test + needs: [cpp-matrix, utility-tests] permissions: - contents: write - - steps: - - name: Add LLVM apt repository - if: matrix.coverage - run: | - apt-get update && apt-get install -y --no-install-recommends ca-certificates wget gnupg lsb-release - wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc - codename=$(lsb_release -cs) - echo "deb http://apt.llvm.org/$codename/ llvm-toolchain-$codename-21 main" >> /etc/apt/sources.list - - - name: Install System Packages - uses: alandefreitas/cpp-actions/package-install@v1.9.2 - if: matrix.install != '' - id: package-install - env: - DEBIAN_FRONTEND: 'noninteractive' - TZ: 'Etc/UTC' - with: - apt-get: ${{ matrix.install }} - - - name: Clone MrDocs - uses: actions/checkout@v4 - - - name: Configure Git Safe Directory - if: matrix.container != '' - run: git config --global --add safe.directory "$(pwd)" - - - name: Setup C++ - uses: alandefreitas/cpp-actions/setup-cpp@v1.9.2 - id: setup-cpp - with: - compiler: ${{ matrix.compiler }} - version: ${{ matrix.version }} - - - name: Configure symbolizer paths - if: matrix.compiler != 'msvc' - shell: bash - run: | - set -e - candidates=() - # 1) Anything on PATH - if command -v llvm-symbolizer >/dev/null 2>&1; then - candidates+=("$(command -v llvm-symbolizer)") - fi - uname_out="$(uname -s || true)" - # 2) Platform-specific common locations - case "$uname_out" in - Darwin) - if xcrun --find llvm-symbolizer >/dev/null 2>&1; then - candidates+=("$(xcrun --find llvm-symbolizer)") - fi - candidates+=("/opt/homebrew/opt/llvm/bin/llvm-symbolizer") - ;; - Linux) - for dir in /usr/lib/llvm-* /usr/lib/llvm; do - if [ -x "$dir/bin/llvm-symbolizer" ]; then - candidates+=("$dir/bin/llvm-symbolizer") - fi - done - ;; - MINGW*|MSYS*|CYGWIN*) - for dir in "/c/Program Files/LLVM/bin" "/c/ProgramData/chocolatey/lib/llvm/tools/llvm/bin"; do - if [ -x "$dir/llvm-symbolizer.exe" ]; then - candidates+=("$dir/llvm-symbolizer.exe") - fi - done - ;; - esac - sym="" - for c in "${candidates[@]}"; do - if [ -n "$c" ] && [ -x "$c" ]; then - sym="$c" - break - fi - done - if [ -n "$sym" ]; then - echo "Using llvm-symbolizer at: $sym" - echo "LLVM_SYMBOLIZER_PATH=$sym" >> "$GITHUB_ENV" - echo "ASAN_SYMBOLIZER_PATH=$sym" >> "$GITHUB_ENV" - else - echo "Warning: llvm-symbolizer not found; ASan stacks may be unsymbolized." >&2 - fi - - - name: Select Xcode 16.4 - if: matrix.compiler == 'apple-clang' - run: | - set -x - sudo ls -1 /Applications | grep Xcode - sudo xcode-select -s /Applications/Xcode_16.4.app/Contents/Developer - ${{ steps.setup-cpp.outputs.cxx }} -v - ${{ steps.setup-cpp.outputs.cxx }} --print-targets - ${{ steps.setup-cpp.outputs.cxx }} --print-target-triple - - # Compute absolute paths and the LLVM cache key. - # Paths depend on the runner's working directory, so they can't be - # set in the matrix template. The cache key is generated by - # bootstrap so the recipe file is the single source of truth for - # the LLVM revision. - - name: Resolve absolute paths - id: paths - run: | - set -euvx - - third_party_dir="$(realpath $(pwd)/..)/third-party" - if [[ "${{ runner.os }}" == 'Windows' ]]; then - third_party_dir="$(echo "$third_party_dir" | sed 's/\\/\//g; s|^/d/|D:/|')" - fi - echo "third-party-dir=$third_party_dir" >> $GITHUB_OUTPUT - echo "llvm-path=$third_party_dir/llvm" >> $GITHUB_OUTPUT - - - name: Cached LLVM Binaries - id: llvm-cache - uses: actions/cache@v4 - with: - path: ${{ steps.paths.outputs.llvm-path }} - key: ${{ matrix.llvm-archive-basename }} - - # Bootstrap handles all dependency installation: LLVM, libc++, - # JerryScript, Lua, Boost.Mp11, Boost.Describe, and libxml2 (MSVC only). - # When LLVM is restored from cache, bootstrap detects the existing - # install directory and skips rebuilding it. - # The --env-file flag writes computed _ROOT paths and flags (libc++, - # sanitizer ldflags) to a file that is sourced into GITHUB_ENV so - # downstream steps use bootstrap as the single source of truth. - - name: Install Dependencies via Bootstrap - env: - PYTHONIOENCODING: utf-8 - run: | - set -eux - - python=$(command -v python3 || command -v python) - - args=( - --yes - --skip-build --no-build-tests --no-run-configs - --build-type "${{ matrix.build-type }}" - --cc "${{ steps.setup-cpp.outputs.cc || matrix.cc }}" - --cxx "${{ steps.setup-cpp.outputs.cxx || matrix.cxx }}" - # Install deps to the CI cache path so actions/cache can save/restore them - --cache-dir "${{ steps.paths.outputs.third-party-dir }}" - # Write _ROOT paths and flags for GITHUB_ENV injection - --env-file bootstrap-env.txt - ) - - # Pass sanitizer flag when a sanitizer is active - if [[ -n "${{ matrix.bootstrap-sanitizer }}" ]]; then - args+=(--sanitizer "${{ matrix.bootstrap-sanitizer }}") - fi + actions: write + uses: ./.github/workflows/ci-build.yml + with: + matrix: ${{ needs.cpp-matrix.outputs.matrix }} + secrets: inherit - # Pass through common compiler flags for dependency builds - # (e.g. -gz=zstd for clang, -fsanitize-memory-track-origins for msan) - common_flags="${{ matrix.common-flags }}" - if [[ -n "$common_flags" ]]; then - args+=(--cflags "$common_flags" --cxxflags "$common_flags") - fi - - # On non-Windows, exclude libxml2 (system libxml2-dev is used instead) - if [[ "${{ runner.os }}" != 'Windows' ]]; then - args+=(--recipe-filter llvm,boost_mp11,boost_describe,jerryscript,lua) - fi - - "$python" bootstrap.py "${args[@]}" - - # Source bootstrap-computed paths and flags into GITHUB_ENV - if [[ -f bootstrap-env.txt ]]; then - cat bootstrap-env.txt >> "$GITHUB_ENV" - echo "--- bootstrap-env.txt ---" - cat bootstrap-env.txt - fi - - - name: Install Node.js - uses: actions/setup-node@v4 - with: - node-version: '20' - - # Bootstrap's --env-file writes _ROOT paths and computed flags - # (libc++, sanitizer) into GITHUB_ENV. The cmake-workflow step - # references them via ${{ env.* }} so bootstrap is the single - # source of truth for dependency locations and link flags. - - name: CMake Workflow - uses: alandefreitas/cpp-actions/cmake-workflow@v1.9.2 - env: - LLVM_PROFILE_FILE: mrdocs-%b-%m.profraw - with: - cmake-version: '>=3.26' - cxxstd: ${{ matrix.cxxstd }} - cc: ${{ steps.setup-cpp.outputs.cc || matrix.cc }} - ccflags: ${{ matrix.mrdocs-ccflags }} - cxx: ${{ steps.setup-cpp.outputs.cxx || matrix.cxx }} - cxxflags: ${{ matrix.mrdocs-ccflags }} ${{ env.BOOTSTRAP_CXXFLAGS }} - generator: Ninja - toolchain: ${{ steps.package-install.outputs.vcpkg_toolchain || steps.package-install.outputs.vcpkg-toolchain }} - build-type: ${{ matrix.build-type }} - install-prefix: .local - extra-args: | - -D MRDOCS_BUILD_DOCS=OFF - -D MRDOCS_EXPENSIVE_TESTS=${{ matrix.is-bottleneck && 'OFF' || 'ON' }} - -D CMAKE_EXE_LINKER_FLAGS="${{ env.BOOTSTRAP_LDFLAGS }}" - -D LLVM_ROOT="${{ env.LLVM_ROOT }}" - -D jerryscript_ROOT="${{ env.jerryscript_ROOT }}" - -D LUA_ROOT="${{ env.Lua_ROOT }}" - -D Lua_ROOT="${{ env.Lua_ROOT }}" - -D lua_ROOT="${{ env.Lua_ROOT }}" - -D boost_mp11_ROOT="${{ env.boost_mp11_ROOT }}" - -D boost_describe_ROOT="${{ env.boost_describe_ROOT }}" - ${{ env.LibXml2_ROOT && format('-D LibXml2_ROOT="{0}"', env.LibXml2_ROOT) || '' }} - export-compile-commands: true - run-tests: true - install: true - package: ${{ matrix.is-main }} - package-dir: packages - package-generators: ${{ matrix.mrdocs-package-generators }} - package-artifact: false - ctest-timeout: 9000 - - - name: Upload GitHub Release Artifacts - if: ${{ matrix.is-main && matrix.compiler != 'clang' }} - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.mrdocs-release-package-artifact }} - path: | - build/packages - !build/packages/_CPack_Packages - retention-days: 1 - - - name: FlameGraph - uses: alandefreitas/cpp-actions/flamegraph@v1.9.2 - if: matrix.time-trace - with: - build-dir: build - github_token: ${{ secrets.GITHUB_TOKEN }} - - - name: Codecov - id: codecov - if: matrix.coverage - run: | - set -euvx - - cd build - build_id=$(eu-readelf -n bin/mrdocs-test | awk '/Build ID:/{print $NF}') - llvm-profdata-${{ matrix.major }} merge -sparse mrdocs-${build_id}-*.profraw -o default.profdata - llvm-cov-${{ matrix.major }} export -format=lcov -instr-profile=default.profdata bin/mrdocs-test > "mrdocs-test.raw.info" - - # Strip lines annotated with LCOV_EXCL_LINE / LCOV_EXCL_START..STOP - # from the LCOV data, since llvm-cov doesn't process these markers. - python3 -c " - import re, sys - excl_lines = {} - for path in set(re.findall(r'^SF:(.+)$', open('mrdocs-test.raw.info', errors='replace').read(), re.M)): - try: - src = open(path, errors='replace').readlines() - except OSError: - continue - skip = False - for i, line in enumerate(src, 1): - if 'LCOV_EXCL_STOP' in line: - skip = False - if skip or 'LCOV_EXCL_LINE' in line: - excl_lines.setdefault(path, set()).add(i) - if 'LCOV_EXCL_START' in line: - skip = True - cur_sf = None - for line in open('mrdocs-test.raw.info', errors='replace'): - if line.startswith('SF:'): - cur_sf = line[3:].strip() - if line.startswith('DA:'): - lineno = int(line.split(',')[0][3:]) - if cur_sf in excl_lines and lineno in excl_lines[cur_sf]: - continue - if line.startswith('BRDA:'): - lineno = int(line.split(',')[0][5:]) - if cur_sf in excl_lines and lineno in excl_lines[cur_sf]: - continue - sys.stdout.write(line) - " > "mrdocs-test.info" - - echo "file=$(realpath "mrdocs-test.info")" >> $GITHUB_OUTPUT - - - name: Upload Coverage as Artifact - if: matrix.coverage - uses: actions/upload-artifact@v4 - with: - name: Coverage - path: ${{ steps.codecov.outputs.file }} - retention-days: 30 - - - name: Codecov Upload - uses: codecov/codecov-action@v5 - if: matrix.coverage - with: - fail_ci_if_error: true - files: ${{ steps.codecov.outputs.file }} - flags: cpp - disable_search: true - token: ${{ secrets.CODECOV_TOKEN }} - verbose: true - - - # Utility checks that only need to run once, not per compiler. - # Tests bootstrap Python code, Danger.js rules, and YAML schema. utility-tests: - runs-on: ubuntu-24.04 name: Utility Tests - - steps: - - name: Clone MrDocs - uses: actions/checkout@v4 - - - name: Install Node.js - uses: actions/setup-node@v4 - with: - node-version: '20' - - - name: Bootstrap Python Tests - run: | - set -eux - pip install --quiet coverage - python3 -m coverage run -m unittest discover -s util/bootstrap/tests/ - python3 -m coverage report - python3 -m coverage report --fail-under=84 - python3 -m coverage xml -o bootstrap-coverage.xml - - - name: Upload Bootstrap Coverage to Codecov - uses: codecov/codecov-action@v5 - with: - files: bootstrap-coverage.xml - flags: bootstrap - fail_ci_if_error: true - token: ${{ secrets.CODECOV_TOKEN }} - verbose: true - - - name: Danger.js Tests - working-directory: util/danger - run: | - npm ci --ignore-scripts - npx vitest run - - - name: Check YAML schema - run: | - python3 ./util/generate-yaml-schema.py --check - npx -y -p ajv-cli -- ajv compile -s docs/mrdocs.schema.json - - - name: Verify snippet .cpp files match golden tests - shell: bash - run: | - set -euo pipefail - shopt -s nullglob - - SRC="docs/website/snippets" - DST="test-files/golden-tests/snippets" - - [[ -d "$SRC" ]] || { echo "Source directory not found: $SRC"; exit 2; } - [[ -d "$DST" ]] || { echo "Destination directory not found: $DST"; exit 2; } - - missing=() - mismatched=() - - while IFS= read -r -d '' src; do - rel="${src#$SRC/}" - dst="$DST/$rel" - if [[ ! -f "$dst" ]]; then - missing+=("$rel") - continue - fi - if ! git diff --no-index --ignore-cr-at-eol --quiet -- "$src" "$dst"; then - mismatched+=("$rel") - fi - done < <(find "$SRC" -type f -name '*.cpp' -print0) - - if (( ${#missing[@]} || ${#mismatched[@]} )); then - if (( ${#missing[@]} )); then - echo "Missing corresponding golden files:" - printf ' %s\n' "${missing[@]}" - fi - if (( ${#mismatched[@]} )); then - echo "Content mismatches:" - printf ' %s\n' "${mismatched[@]}" - fi - exit 1 - fi - echo "All snippet .cpp files are present and match." - + uses: ./.github/workflows/ci-utility-tests.yml + secrets: inherit releases: - needs: [ cpp-matrix, build ] - if: ${{ needs.cpp-matrix.outputs.releases-matrix != '[]' && needs.cpp-matrix.outputs.releases-matrix != '' }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.cpp-matrix.outputs.releases-matrix) }} - - defaults: - run: - shell: bash - - name: ${{ matrix.os }} MrDocs Releases - runs-on: ${{ matrix.runs-on }} - container: ${{ matrix.container }} + name: Releases + needs: [utility-tests, cpp-matrix, build] permissions: contents: write - - steps: - - name: Resolve absolute paths and cache key - id: paths - run: | - set -euvx - - third_party_dir="$(realpath $(pwd)/..)/third-party" - if [[ "${{ runner.os }}" == 'Windows' ]]; then - third_party_dir="$(echo "$third_party_dir" | sed 's/\\/\//g; s|^/d/|D:/|')" - fi - echo "third-party-dir=$third_party_dir" >> $GITHUB_OUTPUT - echo "llvm-path=$third_party_dir/llvm" >> $GITHUB_OUTPUT - - - name: Ensure Node - if: matrix.container != '' && env.ACT == 'true' - run: | - set -e - apt-get update - apt-get install -y nodejs npm - if ! command -v node >/dev/null 2>&1 && command -v nodejs >/dev/null 2>&1; then - ln -s /usr/bin/nodejs /usr/bin/node - fi - - - name: Install packages - uses: alandefreitas/cpp-actions/package-install@v1.9.2 - id: package-install - with: - apt-get: build-essential asciidoctor cmake bzip2 git rsync - - - name: Clone MrDocs - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - - name: Set Repository Ownership - run: | - git config --global --add safe.directory "$(pwd)" - - - name: Install Node.js - uses: actions/setup-node@v4 - with: - node-version: '18' - - - name: Setup Ninja - uses: seanmiddleditch/gha-setup-ninja@v5 - if: ${{ runner.os == 'Windows' }} - - - name: Setup C++ - uses: alandefreitas/cpp-actions/setup-cpp@v1.9.2 - id: setup-cpp - with: - compiler: ${{ matrix.compiler }} - version: ${{ matrix.version }} - - - name: Cached LLVM Binaries - id: llvm-cache - uses: actions/cache@v4 - with: - path: ${{ steps.paths.outputs.llvm-path }} - key: ${{ matrix.llvm-archive-basename }} - fail-on-cache-miss: true - - - name: Download MrDocs package - uses: actions/download-artifact@v4 - with: - name: ${{ matrix.mrdocs-release-package-artifact }} - path: packages - - - name: Install MrDocs from Package - run: | - set -euvx - - # Delete packages/_CPack_Packages files from previous runs - rm -rf packages/_CPack_Packages - - # Print tree structure - find packages -print | sed 's;[^/]*/;|____;g;s;____|; |;g' - - dest_dir="${{ steps.paths.outputs.llvm-path }}" - - if [[ ${{ runner.os }} != 'Windows' ]]; then - find packages -maxdepth 1 -name 'MrDocs-*.tar.gz' -exec tar -vxzf {} -C $dest_dir --strip-components=1 \; - else - package=$(find packages -maxdepth 1 -name "MrDocs-*.7z" -print -quit) - filename=$(basename "$package") - name="${filename%.*}" - 7z x "${package}" -o${dest_dir} - set +e - robocopy "${dest_dir}/${name}" "${dest_dir}" //move //e //np //nfl - exit_code=$? - set -e - if (( exit_code >= 8 )); then - exit 1 - fi - fi - MRDOCS_ROOT="$dest_dir" - echo -e "MRDOCS_ROOT=$MRDOCS_ROOT" >> $GITHUB_ENV - echo -e "$MRDOCS_ROOT/bin" >> $GITHUB_PATH - $MRDOCS_ROOT/bin/mrdocs --version - - - name: Clone Boost.URL - uses: alandefreitas/cpp-actions/boost-clone@v1.9.2 - id: boost-url-clone - with: - branch: develop - modules: url - boost-dir: boost - modules-scan-paths: '"test example"' - modules-exclude-paths: '' - trace-commands: true - - - name: Set up llvm-symbolizer - if: ${{ runner.os != 'Windows' }} - run: | - set -x - - if [[ $RUNNER_OS == 'macOS' ]]; then - # Step 1: Check if llvm-symbolizer is installed - if ! command -v llvm-symbolizer &> /dev/null; then - echo "llvm-symbolizer is not installed. Installing via Homebrew..." - # Step 2: Install llvm if not installed - if command -v brew &> /dev/null; then - brew install llvm - else - echo "Homebrew is not installed. Please install Homebrew first: https://brew.sh/" - exit 1 - fi - fi - - # Step 3: Ensure llvm-symbolizer is in your PATH - llvm_bin_path=$(brew --prefix)/opt/llvm/bin - PATH="$PATH:$llvm_bin_path" - LLVM_SYMBOLIZER_PATH=$(which llvm-symbolizer) - if [ -z "$LLVM_SYMBOLIZER_PATH" ]; then - echo "llvm-symbolizer installation failed or it's not in the PATH." - exit 1 - else - echo "llvm-symbolizer found at: $LLVM_SYMBOLIZER_PATH" - fi - elif [[ $RUNNER_OS == 'Linux' ]]; then - # Step 1: Check if llvm-symbolizer is installed - if ! command -v llvm-symbolizer &> /dev/null; then - echo "llvm-symbolizer is not installed. Installing via apt-get..." - apt-get update - apt-get install -y llvm - fi - - # Step 2: Ensure llvm-symbolizer is in your PATH - LLVM_SYMBOLIZER_PATH=$(which llvm-symbolizer) - if [ -z "$LLVM_SYMBOLIZER_PATH" ]; then - echo "llvm-symbolizer installation failed or it's not in the PATH." - exit 1 - else - echo "llvm-symbolizer found at: $LLVM_SYMBOLIZER_PATH" - fi - else - echo "Unsupported OS: $RUNNER_OS" - exit 1 - fi - - # Step 4: Export LLVM_SYMBOLIZER_PATH environment variable - export LLVM_SYMBOLIZER_PATH="$LLVM_SYMBOLIZER_PATH" - echo -e "LLVM_SYMBOLIZER_PATH=$LLVM_SYMBOLIZER_PATH" >> $GITHUB_ENV - echo "Environment variable LLVM_SYMBOLIZER_PATH set to: $LLVM_SYMBOLIZER_PATH" - - - name: Generate Landing Page - working-directory: docs/website - run: | - npm ci - node render.js - mkdir -p ../../build/website - cp index.html ../../build/website/index.html - cp robots.txt ../../build/website/robots.txt - cp styles.css ../../build/website/styles.css - cp -r assets ../../build/website/assets - - - name: Generate Antora UI - working-directory: docs/ui - run: | - # This playbook renders the documentation - # content for the website. It includes - # master, develop, and tags. - GH_TOKEN="${{ secrets.GITHUB_TOKEN }}" - export GH_TOKEN - npm ci - npx gulp lint - npx gulp - - # Website publishing gate: - # - Only publish on pushes to master/develop and on tags - # - Only on Linux runners (the publish steps assume GNU tooling) - # - # Use `if: env.PUBLISH_WEBSITE == 'true'` for all steps that - # write into `build/website/` and get deployed to the website. - - name: Set website publish gate - run: | - is_publish_ref='false' - if [[ "${{ github.event_name }}" == 'push' ]]; then - if [[ "${{ github.ref_name }}" == 'master' || "${{ github.ref_name }}" == 'develop' ]]; then - is_publish_ref='true' - fi - if [[ "${{ github.ref }}" == refs/tags/* ]]; then - is_publish_ref='true' - fi - fi - - publish_website="$is_publish_ref" - if [[ "${{ runner.os }}" != 'Linux' ]]; then - publish_website='false' - fi - - { - echo "IS_PUBLISH_REF=$is_publish_ref" - echo "PUBLISH_WEBSITE=$publish_website" - } >> "$GITHUB_ENV" - - - name: Ensure all refs for Antora - if: env.PUBLISH_WEBSITE == 'true' - run: | - set -euo pipefail - # Make sure Antora sees every branch and tag from the upstream repo, - # regardless of who triggered the workflow. - git remote set-url origin https://github.com/cppalliance/mrdocs.git - git fetch --prune --prune-tags origin \ - '+refs/heads/*:refs/remotes/origin/*' \ - '+refs/tags/*:refs/tags/*' - - - name: Generate Remote Documentation - # This step fetches and builds develop, master and all tags. That's - # unrelated to a PR, and is only needed for website publishing. So, skip - # it for a PR. - if: github.event_name != 'pull_request' - working-directory: docs - run: | - # This playbook renders the documentation - # content for the website. It includes - # master, develop, and tags. - GH_TOKEN="${{ secrets.GITHUB_TOKEN }}" - export GH_TOKEN - set -x - npm ci - npx antora --clean --fetch antora-playbook.yml --log-level=debug - mkdir -p ../build/website/docs - cp -vr build/site/* ../build/website/docs - - - name: Upload Website as Artifact - uses: actions/upload-artifact@v4 - with: - name: Website ${{ runner.os }} - path: build/website - retention-days: 30 - - - name: Generate Local Documentation - working-directory: docs - run: | - # This playbook allows us to render the - # documentation content and visualize it - # before a workflow that pushes to the - # website is triggered. - GH_TOKEN="${{ secrets.GITHUB_TOKEN }}" - export GH_TOKEN - set -x - npm ci - npx antora antora-playbook.yml --attribute branchesarray=HEAD --stacktrace --log-level=debug - mkdir -p ../build/docs-local - cp -vr build/site/* ../build/docs-local - - - name: Clone Beman.Optional - uses: actions/checkout@v4 - with: - repository: steve-downey/optional - ref: main - path: beman-optional - - - name: Clone Fmt - uses: actions/checkout@v4 - with: - repository: fmtlib/fmt - ref: main - path: fmt - - - name: Clone Nlohmann.Json - uses: actions/checkout@v4 - with: - repository: nlohmann/json - ref: develop - path: nlohmann-json - - - name: Clone MpUnits - uses: actions/checkout@v4 - with: - repository: mpusz/mp-units - ref: master - path: mp-units - - - name: Patch Demo Projects - shell: bash - run: | - set -euo pipefail - set -x - - for project in beman-optional fmt nlohmann-json mp-units; do - src="./examples/third-party/$project" - dst="./$project" - - [ -d "$src" ] || { echo "Source not found: $src" >&2; exit 1; } - mkdir -p "$dst" - - # Mirror contents of $src into $dst, overwriting existing files - tar -C "$src" -cf - . | tar -C "$dst" -xpf - - done - - - - name: Generate Demos - run: | - set -x - - declare -a generators=( - "adoc" - ${{ github.event_name != 'pull_request' && '"xml" - "html"' || '' }} - ) - - demo_failures="" - - # Generate the demos for each variant and generator - for variant in single multi; do - for generator in "${generators[@]}"; do - [[ $generator = xml && $variant = multi ]] && continue - [[ $variant = multi ]] && multipage="true" || multipage="false" - for project_args in \ - "boost-url|$(pwd)/boost/libs/url/doc/mrdocs.yml|../CMakeLists.txt" \ - "beman-optional|$(pwd)/beman-optional/docs/mrdocs.yml|" \ - "nlohmann-json|$(pwd)/nlohmann-json/docs/mrdocs.yml|" \ - "mp-units|$(pwd)/mp-units/docs/mrdocs.yml|" \ - "fmt|$(pwd)/fmt/doc/mrdocs.yml|" \ - "mrdocs|$(pwd)/docs/mrdocs.yml|$(pwd)/CMakeLists.txt" \ - ; do - IFS='|' read -r project config extra <<< "$project_args" - outdir="$(pwd)/demos/$project/$variant/$generator" - cmd=(mrdocs --config="$config" $extra --output="$outdir" --multipage=$multipage --generator="$generator" --log-level=debug) - if ! "${cmd[@]}"; then - echo "FAILED: $project/$variant/$generator" - demo_failures="$demo_failures $project/$variant/$generator\n ${cmd[*]}\n" - rm -rf "$outdir" - fi - done - done - - # Render the asciidoc files to html using asciidoctor - if [[ ${{ runner.os }} == 'Linux' ]]; then - for project in boost-url beman-optional mrdocs fmt nlohmann-json mp-units; do - root="$(pwd)/demos/$project/$variant" - src="$root/adoc" - dst="$root/adoc-asciidoc" - stylesheet="$(pwd)/share/mrdocs/addons/generator/common/layouts/style.css" - - # Skip if adoc generation failed for this project - [[ -d "$src" ]] || continue - - # Create the top-level output dir - mkdir -p "$dst" - - # Find every .adoc (recursively), mirror the directory structure, and render - find "$src" -type f -name '*.adoc' -print0 | - while IFS= read -r -d '' f; do - rel="${f#"$src/"}" # path relative to $src - outdir="$dst/$(dirname "$rel")" # mirror subdir inside $dst - mkdir -p "$outdir" - asciidoctor -a stylesheet="${stylesheet}" -D "$outdir" "$f" - done - done - fi - done - - # Compress demos for the artifact - tar -cjf $(pwd)/demos.tar.gz -C $(pwd)/demos --strip-components 1 . - echo "demos_path=$(pwd)/demos.tar.gz" >> $GITHUB_ENV - - if [[ -n "$demo_failures" ]]; then - echo "The following demos failed:" - printf "$demo_failures" - exit 1 - fi - - - name: Upload Demos as Artifacts - uses: actions/upload-artifact@v4 - with: - name: demos${{ (contains(fromJSON('["master", "develop"]'), github.ref_name ) && format('-{0}', github.ref_name)) || '' }}-${{ runner.os }} - path: demos.tar.gz - # develop and master are retained for longer so that they can be compared - retention-days: ${{ contains(fromJSON('["master", "develop"]'), github.ref_name) && '30' || '1' }} - - - name: Download Previous Demos - if: startsWith(github.ref, 'refs/tags/') && runner.os == 'Linux' - id: download-prev-demos - uses: actions/download-artifact@v4 - continue-on-error: true - with: - name: demos-develop-${{ runner.os }} - path: demos-previous - - - name: Compare demos - if: startsWith(github.ref, 'refs/tags/') && steps.download-prev-demos.outputs.cache-hit == 'true' && runner.os == 'Linux' - id: compare-demos - run: | - set -x - - # Define URLs and directories - LOCAL_DEMOS_DIR="./demos/" - PREV_DEMOS_DIR="./demos-previous/" - DIFF_DIR="./demos-diff/" - - # Check if PREV_DEMOS_DIR exists and is not empty - if [[ ! -d $PREV_DEMOS_DIR || -z $(ls -A $PREV_DEMOS_DIR) ]]; then - echo "No previous demos found." - echo "diff=false" >> $GITHUB_OUTPUT - exit 0 - fi - - # Create directories if they don't exist - mkdir -p $PREV_DEMOS_DIR $DIFF_DIR - - # Iterate over the previous files and compare them with the corresponding local files - find $PREV_DEMOS_DIR -type f | while read previous_file; do - # Derive the corresponding local file path - local_file="${LOCAL_DEMOS_DIR}${previous_file#$PREV_DEMOS_DIR}" - diff_output="$DIFF_DIR${previous_file#$PREV_DEMOS_DIR}" - if [[ -f $local_file ]]; then - mkdir -p "$(dirname "$diff_output")" - diff "$previous_file" "$local_file" > "$diff_output" - if [[ ! -s $diff_output ]]; then - rm "$diff_output" - fi - else - echo "LOCAL FILE $local_file DOES NOT EXITS." > "$diff_output" - echo "PREVIOUS CONTENT OF THE FILE WAS:" >> "$diff_output" - cat "$previous_file" >> "$diff_output" - fi - done - - # Iterate over the local files to find new files - find $LOCAL_DEMOS_DIR -type f | while read local_file; do - previous_file="${PREV_DEMOS_DIR}${local_file#$LOCAL_DEMOS_DIR}" - diff_output="$DIFF_DIR${local_file#$LOCAL_DEMOS_DIR}" - if [[ ! -f $previous_file ]]; then - echo "PREVIOUS $previous_file DOES NOT EXIST." > "$diff_output" - echo "IT HAS BEEN INCLUDED IN THIS VERSION." >> "$diff_output" - echo "NEW CONTENT OF THE FILE IS:" >> "$diff_output" - fi - done - - # Check if the diff directory is empty - if [[ -z $(ls -A $DIFF_DIR) ]]; then - echo "No differences found." - # Store this as an output for the next step - echo "diff=false" >> $GITHUB_OUTPUT - else - # Calculate number of files in the diff directory - N_FILES=$(find $DIFF_DIR -type f | wc -l) - echo "Differences found in $N_FILES output files." - echo "diff=true" >> $GITHUB_OUTPUT - fi - - - name: Upload Demo Diff as Artifacts - if: startsWith(github.ref, 'refs/tags/') && steps.download-prev-demos.outputs.cache-hit == 'true' && steps.compare-demos.outputs.diff == 'true' && runner.os == 'Linux' - uses: actions/upload-artifact@v4 - with: - name: demos-diff - path: demos-diff - retention-days: 30 - - - name: Publish Website to GitHub Pages - if: env.PUBLISH_WEBSITE == 'true' - uses: peaceiris/actions-gh-pages@v3 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: build/website - force_orphan: true - - - name: Publish website - if: env.PUBLISH_WEBSITE == 'true' - env: - SSH_AUTH_SOCK: /tmp/ssh_agent.sock - run: | - set -euvx - # Add SSH key - mkdir -p /home/runner/.ssh - ssh-keyscan dev-websites.cpp.al >> /home/runner/.ssh/known_hosts - chmod 600 /home/runner/.ssh/known_hosts - echo "${{ secrets.DEV_WEBSITES_SSH_KEY }}" > /home/runner/.ssh/github_actions - chmod 600 /home/runner/.ssh/github_actions - ssh-agent -a $SSH_AUTH_SOCK > /dev/null - ssh-add /home/runner/.ssh/github_actions - - rsyncopts=(--recursive --delete --links --times --chmod=D0755,F0755 --compress --compress-choice=zstd --rsh="ssh -o StrictHostKeyChecking=no" --human-readable) - website_dir="ubuntu@dev-websites.cpp.al:/var/www/mrdox.com" - demo_dir="$website_dir/demos/${{ github.ref_name }}" - - # Copy files: This step will copy the landing page and the documentation to www.mrdocs.com - time rsync "${rsyncopts[@]}" --exclude=llvm+clang/ --exclude=demos/ --exclude=roadmap/ $(pwd)/build/website/ "$website_dir"/ - - # Copy demos: This step will copy the demos to www.mrdocs.com/demos - time rsync "${rsyncopts[@]}" $(pwd)/demos/ "$demo_dir"/ - - - name: Create changelog - uses: alandefreitas/cpp-actions/create-changelog@v1.9.2 - with: - output-path: CHANGELOG.md - thank-non-regular: ${{ startsWith(github.ref, 'refs/tags/') }} - github-token: ${{ secrets.GITHUB_TOKEN }} - limit: 150 - update-summary: ${{ runner.os == 'Linux' && 'true' || 'false' }} - - # For non-tag publishes (the develop-release and master-release - # rolling releases), strip the project version out of the package - # filenames and insert the branch name instead. Subsequent pushes - # to the same branch then overwrite the existing GitHub-release - # assets cleanly; without this, a version bump would leave the - # previous version's files behind as stale assets. Tag releases - # keep their versioned filenames. - - name: Rebrand branch packages with the ref name - if: env.IS_PUBLISH_REF == 'true' && !startsWith(github.ref, 'refs/tags/') - run: | - set -euxo pipefail - cd packages - for f in MrDocs-*.*.*-*.*; do - [ -e "$f" ] || continue - new=$(echo "$f" | sed -E 's|^MrDocs-[0-9]+\.[0-9]+\.[0-9]+-|MrDocs-${{ github.ref_name }}-|') - mv -- "$f" "$new" - done - - - name: Create GitHub Package Release - if: env.IS_PUBLISH_REF == 'true' - uses: softprops/action-gh-release@v2 - with: - # After the rebrand step, branch packages have the form - # MrDocs--. (no semver), so the glob - # is broader than the historical MrDocs-*.*.*-*.*. - files: packages/MrDocs-*-*.* - fail_on_unmatched_files: true - name: ${{ github.ref_name || github.ref }} - tag_name: ${{ github.ref_name || github.ref }}${{ ((!startsWith(github.ref, 'refs/tags/')) && '-release') || '' }} - body_path: CHANGELOG.md - prerelease: false - draft: false - token: ${{ github.token }} - - llvm-releases: - needs: [ cpp-matrix, build ] - if: ${{ needs.cpp-matrix.outputs.llvm-matrix != '[]' && needs.cpp-matrix.outputs.llvm-matrix != '' }} - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.cpp-matrix.outputs.llvm-matrix) }} - - defaults: - run: - shell: bash - - name: ${{ matrix.name }} LLVM Release - runs-on: ${{ matrix.runs-on }} - container: ${{ matrix.container }} - env: ${{ matrix.env }} - permissions: - contents: write - - steps: - - name: Ensure Node - if: matrix.container != '' && env.ACT == 'true' - run: | - set -e - apt-get update - apt-get install -y nodejs npm - if ! command -v node >/dev/null 2>&1 && command -v nodejs >/dev/null 2>&1; then - ln -s /usr/bin/nodejs /usr/bin/node - fi - - - name: Resolve absolute paths and cache key - id: paths - run: | - set -euvx - - third_party_dir=$(realpath $(pwd)/..)/third-party - if [[ "${{ runner.os }}" == 'Windows' ]]; then - third_party_dir=$(echo "$third_party_dir" | sed 's/\\/\//g; s|^/d/|D:/|') - fi - echo "third-party-dir=$third_party_dir" >> $GITHUB_OUTPUT - echo "llvm-path=$third_party_dir/llvm" >> $GITHUB_OUTPUT - - - name: Install packages - uses: alandefreitas/cpp-actions/package-install@v1.9.2 - id: package-install - with: - apt-get: ${{ matrix.install }} - - - name: LLVM Binaries - id: llvm-cache - uses: actions/cache@v4 - with: - path: ${{ steps.paths.outputs.llvm-path }} - key: ${{ matrix.llvm-archive-basename }} + uses: ./.github/workflows/ci-releases.yml + with: + submatrices: ${{ needs.cpp-matrix.outputs.submatrices }} + secrets: inherit diff --git a/CMakeLists.txt b/CMakeLists.txt index edebc5100d..2a0a745131 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -40,11 +40,7 @@ else() set(REQUIRED_IF_STRICT "") endif() option(MRDOCS_REQUIRE_GIT "Git is required: not being able to extract version build is an error" ON) -if (MRDOCS_BUILD_TESTS OR MRDOCS_INSTALL) - option(MRDOCS_BUILD_DOCS "Build documentation" ON) -else() - option(MRDOCS_BUILD_DOCS "Build documentation" OFF) -endif() +option(MRDOCS_BUILD_DOCS "Build documentation" OFF) option(MRDOCS_GENERATE_REFERENCE "Generate MrDocs reference" ${MRDOCS_BUILD_DOCS}) option(MRDOCS_GENERATE_ANTORA_REFERENCE "Generate MrDocs reference in Antora module pages" OFF) @@ -233,6 +229,13 @@ endif() set(CMAKE_FOLDER Dependencies) # LLVM + Clang +# Import LLVM_ROOT from environment if not set as a CMake variable. +# CMP0074 makes find_package() search _ROOT env vars +# automatically, but the validation and Clang_ROOT derivation below +# need the CMake variable to be set explicitly. +if (NOT LLVM_ROOT AND DEFINED ENV{LLVM_ROOT}) + set(LLVM_ROOT "$ENV{LLVM_ROOT}") +endif() if (LLVM_ROOT) # LLVM_ROOT is absolute get_filename_component(LLVM_ROOT "${LLVM_ROOT}" ABSOLUTE) diff --git a/docs/website/render.js b/docs/website/render.js index 210b3718b4..8ae1504385 100644 --- a/docs/website/render.js +++ b/docs/website/render.js @@ -1,3 +1,13 @@ +// +// Licensed under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// Copyright (c) 2026 Alan de Freitas (alandefreitas@gmail.com) +// +// Official repository: https://github.com/cppalliance/mrdocs +// + const Handlebars = require('handlebars'); const hljs = require('highlight.js/lib/core'); hljs.registerLanguage('cpp', require('highlight.js/lib/languages/cpp')); @@ -38,6 +48,16 @@ if (!mrdocsRoot) { const mrdocsExecutable = path.join(mrdocsRoot, 'bin', 'mrdocs') + (process.platform === 'win32' ? '.exe' : ''); if (!fs.existsSync(mrdocsExecutable)) { console.log(`mrdocs executable not found at ${mrdocsExecutable}`); + // Walk up the path to find the first directory that exists + let dir = path.dirname(mrdocsExecutable); + while (dir && dir !== path.dirname(dir)) { + if (fs.existsSync(dir)) { + console.log(`Nearest existing directory: ${dir}`); + console.log(`Contents: ${fs.readdirSync(dir).join(', ')}`); + break; + } + dir = path.dirname(dir); + } process.exit(1); } diff --git a/src/lib/Metadata/Finalizers/DocCommentFinalizer.cpp b/src/lib/Metadata/Finalizers/DocCommentFinalizer.cpp index 497808085d..142f76cb7a 100644 --- a/src/lib/Metadata/Finalizers/DocCommentFinalizer.cpp +++ b/src/lib/Metadata/Finalizers/DocCommentFinalizer.cpp @@ -1271,10 +1271,21 @@ setAutoRelates(Symbol& ctx) }(); } - // Remove duplicates from relatedRecordsOrEnums - std::ranges::sort(relatedRecordsOrEnums); + // Remove duplicates from relatedRecordsOrEnums. + // + // Use plain std::sort/std::unique here instead of the ranges + // versions: libstdc++-15's `ranges::less` probes `operator<=>` + // via ADL on the element type, which on `Symbol*` reaches our + // generic mrdocs::operator<=> template. Clang 19 hard-errors + // when substituting T = Symbol* into that template (operator<=> + // requires a class/enum parameter), instead of SFINAE'ing the + // candidate out of overload resolution -- a regression that's + // present in 19 but absent in 18 and >=20. + std::sort( + relatedRecordsOrEnums.begin(), relatedRecordsOrEnums.end()); relatedRecordsOrEnums.erase( - std::ranges::unique(relatedRecordsOrEnums).begin(), + std::unique( + relatedRecordsOrEnums.begin(), relatedRecordsOrEnums.end()), relatedRecordsOrEnums.end()); // Insert the records with valid ids into the doc relates section diff --git a/src/tool/CompilerInfo.cpp b/src/tool/CompilerInfo.cpp index 595cbe865d..0991c60c13 100644 --- a/src/tool/CompilerInfo.cpp +++ b/src/tool/CompilerInfo.cpp @@ -18,7 +18,7 @@ namespace mrdocs { Optional -getCompilerVerboseOutput(llvm::StringRef compilerPath) +getCompilerVerboseOutput(llvm::StringRef compilerPath) { if ( ! llvm::sys::fs::exists(compilerPath)) { @@ -26,7 +26,7 @@ getCompilerVerboseOutput(llvm::StringRef compilerPath) } llvm::SmallString<128> outputPath; - if (auto ec = llvm::sys::fs::createTemporaryFile("compiler-info", "txt", outputPath)) + if (auto ec = llvm::sys::fs::createTemporaryFile("compiler-info", "txt", outputPath)) { return std::nullopt; } @@ -35,7 +35,7 @@ getCompilerVerboseOutput(llvm::StringRef compilerPath) std::vector const args = {compilerPath, "-v", "-E", "-x", "c++", "-"}; llvm::ArrayRef emptyEnv; int const result = ExecuteAndWaitWithLogging(compilerPath, args, emptyEnv, redirects); - if (result != 0) + if (result != 0) { llvm::sys::fs::remove(outputPath); return std::nullopt; @@ -51,26 +51,26 @@ getCompilerVerboseOutput(llvm::StringRef compilerPath) return bufferOrError.get()->getBuffer().str(); } -std::vector -parseIncludePaths(std::string const& compilerOutput) +std::vector +parseIncludePaths(std::string const& compilerOutput) { std::vector includePaths; std::istringstream stream(compilerOutput); std::string line; bool capture = false; - while (std::getline(stream, line)) + while (std::getline(stream, line)) { - if (line.find("#include <...> search starts here:") != std::string::npos) + if (line.find("#include <...> search starts here:") != std::string::npos) { capture = true; continue; } - if (line.find("End of search list.") != std::string::npos) + if (line.find("End of search list.") != std::string::npos) { break; } - if (capture) + if (capture) { line.erase(0, line.find_first_not_of(" ")); includePaths.push_back(line); @@ -80,8 +80,30 @@ parseIncludePaths(std::string const& compilerOutput) return includePaths; } -std::unordered_map> -getCompilersDefaultIncludeDir(clang::tooling::CompilationDatabase const& compDb, bool useSystemStdlib) +namespace { + +// Try to get include paths from a compiler found by name in PATH. +// Returns the include paths if successful, empty vector otherwise. +std::vector +tryCompilerByName(llvm::StringRef name) +{ + auto found = llvm::sys::findProgramByName(name); + if (!found) + { + return {}; + } + auto output = getCompilerVerboseOutput(*found); + if (!output) + { + return {}; + } + return parseIncludePaths(*output); +} + +} // anonymous namespace + +std::unordered_map> +getCompilersDefaultIncludeDir(clang::tooling::CompilationDatabase const& compDb, bool useSystemStdlib) { if (!useSystemStdlib) { @@ -100,14 +122,30 @@ getCompilersDefaultIncludeDir(clang::tooling::CompilationDatabase const& compDb, continue; } - std::vector includePaths; - auto const compilerOutput = getCompilerVerboseOutput(compilerPath); - if (!compilerOutput) + // Try the compiler specified in the compilation database + auto compilerOutput = getCompilerVerboseOutput(compilerPath); + if (compilerOutput) { - res.emplace(compilerPath, includePaths); + auto includePaths = parseIncludePaths(*compilerOutput); + res.emplace(compilerPath, std::move(includePaths)); continue; } - includePaths = parseIncludePaths(*compilerOutput); + + // The compiler from the database wasn't found. + // Try common fallback compilers to discover system + // include paths. + static constexpr std::string_view fallbackCompilers[] = { + "g++", "clang++", "c++", "gcc", "clang" + }; + std::vector includePaths; + for (auto const& fallback : fallbackCompilers) + { + includePaths = tryCompilerByName(fallback); + if (!includePaths.empty()) + { + break; + } + } res.emplace(compilerPath, std::move(includePaths)); } } @@ -116,4 +154,3 @@ getCompilersDefaultIncludeDir(clang::tooling::CompilationDatabase const& compDb, } } // mrdocs - diff --git a/util/bootstrap/src/core/ui.py b/util/bootstrap/src/core/ui.py index 5806c4fc9c..73cc54a27a 100644 --- a/util/bootstrap/src/core/ui.py +++ b/util/bootstrap/src/core/ui.py @@ -57,6 +57,10 @@ def __init__(self, enable_color: bool = False, enable_emoji: bool = False): self.base_path: Optional[str] = None self.base_token: str = "." self.dry_run: bool = False + self._ci = bool(os.environ.get("GITHUB_ACTIONS")) + self._ci_group_open = False + self._ci_group_title = "" + self._ci_group_start = 0.0 @staticmethod def _supports_color() -> bool: @@ -68,6 +72,9 @@ def _supports_color() -> bool: def _supports_emoji() -> bool: if os.environ.get("BOOTSTRAP_PLAIN"): return False + encoding = getattr(sys.stdout, 'encoding', '') or '' + if encoding.lower().replace('-', '') in ('ascii', 'charmap', 'cp1252', 'latin1', 'iso88591'): + return False return True def _fmt(self, text: str, kind: str, icon: Optional[str] = None) -> str: @@ -117,6 +124,7 @@ def plain(self) -> bool: def section(self, title: str, icon: Optional[str] = None): prefix = (icon + " ") if (self.emoji_enabled and icon) else "" + self.start_group(f"{prefix}{title}") line = ("=" if self.plain else "\u2501") * 60 out = self._out print(file=out) @@ -129,6 +137,7 @@ def command(self, cmd: str, icon: Optional[str] = None): def subsection(self, title: str, icon: Optional[str] = None): prefix = (icon + " ") if (self.emoji_enabled and icon) else "" + self.start_group(f"{prefix}{title}") if self.plain: banner = f"--- {prefix}{title}" else: @@ -141,6 +150,31 @@ def subsection(self, title: str, icon: Optional[str] = None): underline_len = max(15, len(banner.strip()) + 4) print(self._fmt("-" * underline_len, "subsection", ""), file=out) + def start_group(self, title: str): + """Start a CI group. Closes any already-open group first.""" + if self._ci: + import time + self.end_group() + print(f"::group::{title}", file=self._out) + self._ci_group_open = True + self._ci_group_title = title + self._ci_group_start = time.monotonic() + + def end_group(self): + """Close the current CI group if one is open. No-op otherwise.""" + if self._ci_group_open: + import time + elapsed = time.monotonic() - self._ci_group_start + if elapsed >= 60: + mins = int(elapsed) // 60 + secs = int(elapsed) % 60 + duration = f"{mins}m {secs}s" + else: + duration = f"{elapsed:.1f}s" + print(f"{self._ci_group_title} completed in {duration}", file=self._out) + print("::endgroup::", file=self._out) + self._ci_group_open = False + def shorten_path(self, path: str) -> str: if not path: return path diff --git a/util/bootstrap/src/installer.py b/util/bootstrap/src/installer.py index 114e1ed66b..c3acaba30c 100644 --- a/util/bootstrap/src/installer.py +++ b/util/bootstrap/src/installer.py @@ -20,7 +20,7 @@ import os import re import shlex -from typing import Optional, Dict, Any, Set +from typing import Optional, Dict, Any, List, Set def _shquote(s: str) -> str: @@ -43,6 +43,7 @@ def _shquote(s: str) -> str: SANITIZERS, ) from .tools import find_tool, probe_compilers, install_ninja, is_tool_executable, probe_msvc_dev_env +from .tools.compilers import sanitizer_flag_name from .tools.prerequisites import check_prerequisites, report_missing_prerequisites, try_install_system_deps from .recipes import ( Recipe, @@ -55,6 +56,7 @@ def _shquote(s: str) -> str: needs_libcxx_runtimes, libcxx_runtime_flags, write_recipe_stamp, + recipe_stamp_path, is_recipe_up_to_date, generate_cache_key, detect_compiler_for_cache_key, @@ -113,6 +115,7 @@ def __init__(self, cmd_line_args: Optional[Dict[str, Any]] = None, source_dir: O self.compiler_info: Dict[str, str] = {} self.package_roots: Dict[str, str] = {} self.recipe_info: Dict[str, Recipe] = {} + self.rebuilt_recipes: List[str] = [] self._libcxx_cxxflags: str = "" self._libcxx_ldflags: str = "" self.env = os.environ.copy() @@ -447,18 +450,35 @@ def install_dependencies(self): src = recipe.source resolved_ref = src.commit or src.tag or src.branch or src.ref or "" - # Only pass sanitizer to dependency builds for clang. - # GCC doesn't support LLVM's sanitizer infrastructure flags. + # Only pass sanitizer to dependency builds when the sanitizer + # actually requires instrumented dependencies. ASan, MSan, and + # TSan need instrumented deps (ASan/MSan for libc++, TSan for + # correct race detection across library boundaries). UBSan + # only checks the project's own code at compile time, so passing + # it to deps would cause unnecessary rebuilds. compiler_id = self.compiler_info.get("CMAKE_CXX_COMPILER_ID", "") - recipe_sanitizer = self.options.sanitizer if compiler_id.lower() == "clang" else "" - - # Parameters that affect the build output, used for stamp hashing - stamp_args = dict( - sanitizer=recipe_sanitizer, - cc=self.options.cc, cxx=self.options.cxx, - cflags=self.options.cflags, cxxflags=self.options.cxxflags, - ldflags=self.options.ldflags, + san = sanitizer_flag_name(self.options.sanitizer.lower()) if self.options.sanitizer else "" + deps_need_sanitizer = ( + compiler_id.lower() == "clang" + and san in ("address", "memory", "thread") ) + recipe_sanitizer = self.options.sanitizer if deps_need_sanitizer else "" + + # Parameters that affect the build output, used for stamp + # hashing. Only include build parameters when the sanitizer + # actually requires instrumented deps (ASan/MSan/TSan + Clang). + # For all other builds, the only flag reaching deps is -gz=zstd + # (debug info compression), which doesn't change library + # behavior, so the stamp just checks version and ref. + if deps_need_sanitizer: + stamp_args = dict( + sanitizer=recipe_sanitizer, + cc=self.options.cc, cxx=self.options.cxx, + cflags=self.options.cflags, cxxflags=self.options.cxxflags, + ldflags=self.options.ldflags, + ) + else: + stamp_args = dict() # For LLVM with sanitizers: always compute libc++ flags # even on cache hit, since downstream builds need them. @@ -473,7 +493,8 @@ def install_dependencies(self): # Skip build if already up to date (unless force, clean, or dry-run) # In dry-run mode, always show all commands so the output is a # complete manual reference regardless of local cache state. - if not self.options.dry_run and not self.options.force and not self.options.clean and is_recipe_up_to_date(recipe, resolved_ref, **stamp_args): + stale_reason = is_recipe_up_to_date(recipe, resolved_ref, **stamp_args) if not self.options.dry_run else "dry-run" + if not stale_reason and not self.options.force and not self.options.clean: self.ui.ok(f"[{recipe.name}] already up to date ({resolved_ref or 'HEAD'}). Skipping build.") self.print_recipe_summary(recipe) self.recipe_info[recipe.name] = recipe @@ -481,6 +502,13 @@ def install_dependencies(self): self.package_roots[recipe.package_root_var] = recipe.install_dir continue + if stale_reason: + self.ui.info(f"[{recipe.name}] rebuilding: {stale_reason}") + stamp_path = recipe_stamp_path(recipe) + if os.path.exists(stamp_path): + with open(stamp_path, "r", encoding="utf-8") as f: + self.ui.info(f"[{recipe.name}] existing stamp ({stamp_path}):\n{f.read()}") + self._dry_comment(f"Fetch {recipe.name} source") fetch_recipe_source( recipe, @@ -506,7 +534,8 @@ def install_dependencies(self): ui=self.ui, ) - # Build libc++ runtimes if needed (LLVM + clang + asan/msan) + # Build libc++ runtimes if needed (LLVM + clang + asan/msan/tsan) + extra_cmake_options = None if recipe.name == "llvm": compiler_id = self.compiler_info.get("CMAKE_CXX_COMPILER_ID", "") if needs_libcxx_runtimes(self.options.sanitizer, compiler_id): @@ -524,9 +553,7 @@ def install_dependencies(self): ) # Disable runtimes in the main LLVM build so it doesn't # overwrite the instrumented ones we just built - for step in recipe.build: - if step.get("type", "").lower() == "cmake": - step.setdefault("options", []).append("-DLLVM_ENABLE_RUNTIMES=") + extra_cmake_options = ["-DLLVM_ENABLE_RUNTIMES="] self._dry_comment(f"Build and install {recipe.name}") build_recipe( @@ -542,6 +569,7 @@ def install_dependencies(self): self.options.cflags, self.options.cxxflags, self.options.ldflags, + extra_cmake_options, self.options.force, self.options.dry_run, self.options.verbose, @@ -551,6 +579,7 @@ def install_dependencies(self): ) write_recipe_stamp(recipe, resolved_ref, **stamp_args, dry_run=self.options.dry_run, ui=self.ui) + self.rebuilt_recipes.append(recipe.name) self.ui.ok(f"[{recipe.name}] installed successfully.") self.print_recipe_summary(recipe) @@ -728,6 +757,10 @@ def write_env_file(self): if bootstrap_ldflags: lines.append(f"BOOTSTRAP_LDFLAGS={bootstrap_ldflags}") + # Write which recipes were rebuilt (empty if all were cached) + if self.rebuilt_recipes: + lines.append(f"BOOTSTRAP_REBUILT={','.join(self.rebuilt_recipes)}") + content = "\n".join(lines) + "\n" if lines else "" if self.options.dry_run: @@ -1160,7 +1193,7 @@ def _dry_preamble(self): print("#!/usr/bin/env bash") print("set -euo pipefail") print() - print("# MrDocs bootstrap — equivalent manual steps") + print("# MrDocs bootstrap - equivalent manual steps") print(f"# Generated for: {get_os_name()}") print(f"# Source directory: {self.options.source_dir}") @@ -1304,3 +1337,4 @@ def run(self): self.run_mrdocs_tests() self.ui.ok("Bootstrap complete!") + self.ui.end_group() diff --git a/util/bootstrap/src/recipes/archive.py b/util/bootstrap/src/recipes/archive.py index d659d29bfa..22b9839fd1 100644 --- a/util/bootstrap/src/recipes/archive.py +++ b/util/bootstrap/src/recipes/archive.py @@ -50,7 +50,7 @@ def extract_zip_flatten( if dry_run: # GitHub archives contain a single top-level directory that must be - # flattened (stripped) during extraction — mirror what the Python code does. + # flattened (stripped) during extraction - mirror what the Python code does. print(f"_ztmp=$(mktemp -d)") print(f"unzip -o {shlex.quote(zip_path)} -d \"$_ztmp\"") print(f"cp -a \"$_ztmp\"/*/* {shlex.quote(dest_dir)}/") diff --git a/util/bootstrap/src/recipes/builder.py b/util/bootstrap/src/recipes/builder.py index ea465f78a8..9890f2fcd6 100644 --- a/util/bootstrap/src/recipes/builder.py +++ b/util/bootstrap/src/recipes/builder.py @@ -17,7 +17,7 @@ import os import shutil -from typing import Optional, Dict, Any +from typing import Optional, Dict, List, Any from ..core.platform import is_windows from ..core.filesystem import ensure_dir, remove_dir @@ -194,6 +194,7 @@ def run_cmake_recipe_step( cflags: str = "", cxxflags: str = "", ldflags: str = "", + extra_cmake_options: Optional[List[str]] = None, force: bool = False, dry_run: bool = False, verbose: bool = False, @@ -288,9 +289,15 @@ def run_cmake_recipe_step( else: opts.append(extra_opts) - # Merge sanitizer flags with user-provided flags - merged_c_flags = (san_c_flags + " " + cflags).strip() - merged_cxx_flags = (san_cxx_flags + " " + cxxflags).strip() + # Suppress warnings for dependency builds (not our code). + # Prepend so user-provided flags can override (last flag wins). + # We use is_windows() as a proxy for MSVC because MrDocs only + # builds with MSVC on Windows (no MinGW/Clang-CL support). + suppress_warnings = "/w" if is_windows() else "-w" + + # Merge: suppress-warnings + sanitizer flags + user-provided flags + merged_c_flags = (suppress_warnings + " " + san_c_flags + " " + cflags).strip() + merged_cxx_flags = (suppress_warnings + " " + san_cxx_flags + " " + cxxflags).strip() merged_ld_flags = (san_ld_flags + " " + ldflags).strip() if merged_c_flags: @@ -301,6 +308,11 @@ def run_cmake_recipe_step( opts.append(f"-DCMAKE_EXE_LINKER_FLAGS_INIT={merged_ld_flags}") opts.append(f"-DCMAKE_SHARED_LINKER_FLAGS_INIT={merged_ld_flags}") + # Append extra CMake options (e.g. -DLLVM_ENABLE_RUNTIMES=) + # without modifying the recipe definition + if extra_cmake_options: + opts.extend(extra_cmake_options) + ensure_dir(build_dir, dry_run=dry_run, ui=ui) # Configure @@ -418,6 +430,7 @@ def build_recipe( cflags: str = "", cxxflags: str = "", ldflags: str = "", + extra_cmake_options: Optional[List[str]] = None, force: bool = False, dry_run: bool = False, verbose: bool = False, @@ -441,6 +454,8 @@ def build_recipe( cflags: Extra C compiler flags. cxxflags: Extra C++ compiler flags. ldflags: Extra linker flags. + extra_cmake_options: Additional CMake options appended at build + time without modifying the recipe definition. force: If True, clean before building. dry_run: If True, only print what would be done. verbose: If True, show verbose output. @@ -458,6 +473,7 @@ def build_recipe( recipe, raw_step, source_dir, third_party_src_dir, preset, cc, cxx, build_dir_opt, install_dir_opt, sanitizer, cflags, cxxflags, ldflags, + extra_cmake_options, force, dry_run, verbose, debug, env, ui ) elif step_type == "command": diff --git a/util/bootstrap/src/recipes/fetcher.py b/util/bootstrap/src/recipes/fetcher.py index 02d4b5c02e..59963d4283 100644 --- a/util/bootstrap/src/recipes/fetcher.py +++ b/util/bootstrap/src/recipes/fetcher.py @@ -69,7 +69,7 @@ def is_recipe_up_to_date( cflags: str = "", cxxflags: str = "", ldflags: str = "", -) -> bool: +) -> str: """ Check if a recipe is already built and up to date. @@ -84,62 +84,220 @@ def is_recipe_up_to_date( ldflags: Extra linker flags. Returns: - True if the recipe is up to date. + Empty string if the recipe is up to date, otherwise a reason + explaining why the stamp doesn't match. """ stamp_path = recipe_stamp_path(recipe) if not os.path.exists(stamp_path): - return False + return "no stamp file found" try: with open(stamp_path, "r", encoding="utf-8") as f: data = json.load(f) except Exception: - return False - if data.get("version") != recipe.version or data.get("ref") != resolved_ref: - return False - # If the stamp has a content hash, verify it matches the current - # recipe and runtime parameters. Stamps without a hash (from older - # bootstrap versions) pass this check. - stored_hash = data.get("content_hash") - if stored_hash is not None: - current_hash = _recipe_content_hash( - recipe, sanitizer, cc, cxx, cflags, cxxflags, ldflags - ) - if stored_hash != current_hash: - return False - return True - - -def _recipe_content_hash( - recipe: Recipe, + return "stamp file is corrupt or unreadable" + if data.get("ref") != resolved_ref: + return f"ref changed: {data.get('ref')!r} -> {resolved_ref!r}" + # Check if the recipe definition changed. + # New format stores individual fields ("recipe" dict); old format + # stores an opaque hash ("recipe_hash" string). Both are accepted. + stored_recipe = data.get("recipe") + if stored_recipe is not None: + current_recipe = _recipe_fields(recipe) + for key in set(list(stored_recipe.keys()) + list(current_recipe.keys())): + old_val = stored_recipe.get(key) + new_val = current_recipe.get(key) + if old_val != new_val: + diff = _field_diff(old_val, new_val) + return f"recipe {key} changed: {diff}" + elif data.get("recipe_hash") is not None: + # Old format with opaque hash — verify it still matches + current_hash = _recipe_hash(recipe) + if data["recipe_hash"] != current_hash: + return f"recipe changed (hash {data['recipe_hash']!r} -> {current_hash!r})" + # Check if the platform changed (OS, version, architecture). + stored_platform = data.get("platform") + if stored_platform is not None: + current_platform = _platform_info() + for key in set(list(stored_platform.keys()) + list(current_platform.keys())): + old_val = stored_platform.get(key) + new_val = current_platform.get(key) + if old_val != new_val: + return f"platform {key} changed: {old_val!r} -> {new_val!r}" + # Check runtime build parameters (sanitizer, compiler, flags). + # Stamps from older versions may have "content_hash" or + # "build_params" with recipe fields mixed in -- treat as stale. + stored_params = data.get("build_params") + if stored_params is None: + # Old-format stamps (with content_hash instead of build_params) + # can't be compared field-by-field. Treat as stale so the stamp + # gets rewritten in the new format. + has_keys = [k for k in data if k not in ("name", "version", "ref")] + return f"stamp uses old format (has: {has_keys})" + current_params = _build_params(sanitizer, cc, cxx, cflags, cxxflags, ldflags) + for key in set(list(stored_params.keys()) + list(current_params.keys())): + old_val = stored_params.get(key) + new_val = current_params.get(key) + if old_val != new_val: + return f"{key} changed: {_field_diff(old_val, new_val)}" + return "" + + +def _field_diff(old_val, new_val) -> str: + """Produce a human-readable diff between two field values. + + For JSON-encoded strings, parse them and show only the parts + that differ. For simple values, show old -> new. + """ + if old_val is None: + return f"added: {new_val!r}" + if new_val is None: + return f"removed: {old_val!r}" + # Try parsing as JSON for a deeper diff + try: + old_parsed = json.loads(old_val) if isinstance(old_val, str) else old_val + new_parsed = json.loads(new_val) if isinstance(new_val, str) else new_val + return _value_diff(old_parsed, new_parsed) + except (json.JSONDecodeError, TypeError): + pass + return f"{old_val!r} -> {new_val!r}" + + +def _value_diff(old, new, indent=2) -> str: + """Recursively diff two parsed values, showing only what changed.""" + pad = " " * indent + if old == new: + return f"{pad}(unchanged)" + if isinstance(old, dict) and isinstance(new, dict): + diffs = [] + for k in sorted(set(list(old.keys()) + list(new.keys()))): + ov = old.get(k) + nv = new.get(k) + if ov == nv: + continue + if ov is None: + diffs.append(f"{pad}{k}: added {nv!r}") + elif nv is None: + diffs.append(f"{pad}{k}: removed {ov!r}") + elif isinstance(ov, (dict, list)) and isinstance(nv, (dict, list)): + diffs.append(f"{pad}{k}:") + diffs.append(_value_diff(ov, nv, indent + 2)) + else: + diffs.append(f"{pad}{k}: {ov!r} -> {nv!r}") + return "\n".join(diffs) + if isinstance(old, list) and isinstance(new, list): + if len(old) == len(new): + # Same-length lists: diff element by element + diffs = [] + for i, (ov, nv) in enumerate(zip(old, new)): + if ov != nv: + diffs.append(f"{pad}[{i}]:") + diffs.append(_value_diff(ov, nv, indent + 2)) + return "\n".join(diffs) if diffs else f"{pad}(unchanged)" + # Different-length lists: show added/removed + only_old = [x for x in old if x not in new] + only_new = [x for x in new if x not in old] + parts = [] + if only_old: + parts.append(f"{pad}removed: {only_old!r}") + if only_new: + parts.append(f"{pad}added: {only_new!r}") + return "\n".join(parts) if parts else f"{pad}{old!r} -> {new!r}" + return f"{pad}{old!r} -> {new!r}" + + +def _recipe_fields(recipe: Recipe) -> dict: + """ + Collect the recipe definition fields that affect the build. + + Covers every field from the recipe file. Computed fields + (source_dir, build_dir, install_dir) are excluded since they + depend on the environment. Values are JSON-serialized so + complex types (lists, dicts) can be compared as strings. + """ + import dataclasses + return { + "name": recipe.name, + "version": recipe.version, + "source": json.dumps(dataclasses.asdict(recipe.source), sort_keys=True), + "dependencies": json.dumps(recipe.dependencies, sort_keys=True), + "build": json.dumps(recipe.build, sort_keys=True), + "build_type": recipe.build_type, + "tags": json.dumps(recipe.tags, sort_keys=True) if recipe.tags else "[]", + "install_scope": recipe.install_scope, + "package_root_var": recipe.package_root_var, + } + + +def _recipe_hash(recipe: Recipe) -> str: + """Compute a hash of the recipe fields for backward compatibility + with old-format stamps that stored 'recipe_hash'.""" + import hashlib + content = json.dumps(_recipe_fields(recipe), sort_keys=True) + return hashlib.sha256(content.encode()).hexdigest()[:16] + + +def _platform_info() -> dict: + """ + Collect platform information that affects binary compatibility. + + A change in OS, OS version, or architecture means cached binaries + may not work. + """ + import platform + info = { + "os": platform.system(), + "arch": platform.machine(), + } + # Get OS version where available + if platform.system() == "Linux": + try: + import distro + info["os_version"] = distro.version() + except ImportError: + # Fall back to reading /etc/os-release directly + try: + with open("/etc/os-release") as f: + for line in f: + if line.startswith("VERSION_ID="): + info["os_version"] = line.split("=", 1)[1].strip().strip('"') + break + except OSError: + pass + elif platform.system() == "Darwin": + info["os_version"] = platform.mac_ver()[0] + elif platform.system() == "Windows": + info["os_version"] = platform.version() + return info + + +def _build_params( sanitizer: str = "", cc: str = "", cxx: str = "", cflags: str = "", cxxflags: str = "", ldflags: str = "", -) -> str: +) -> dict: """ - Compute a hash of everything that affects the build output. + Collect runtime build parameters that affect the build output. - Covers both the recipe definition and runtime parameters like - sanitizer, compiler, and flags. Any change invalidates the stamp. + Only includes non-empty values. These are compared field by field + so the mismatch reason identifies exactly what changed. """ - import hashlib - content = json.dumps({ - "version": recipe.version, - "source_url": recipe.source.url, - "source_ref": recipe.source.commit or recipe.source.tag or recipe.source.branch or recipe.source.ref or "", - "build": recipe.build, - "build_type": recipe.build_type, - "tags": recipe.tags, - "sanitizer": sanitizer, - "cc": cc, - "cxx": cxx, - "cflags": cflags, - "cxxflags": cxxflags, - "ldflags": ldflags, - }, sort_keys=True) - return hashlib.sha256(content.encode()).hexdigest()[:16] + params = {} + if sanitizer: + params["sanitizer"] = sanitizer + if cc: + params["cc"] = cc + if cxx: + params["cxx"] = cxx + if cflags: + params["cflags"] = cflags + if cxxflags: + params["cxxflags"] = cxxflags + if ldflags: + params["ldflags"] = ldflags + return params def write_recipe_stamp( @@ -177,12 +335,16 @@ def write_recipe_stamp( "name": recipe.name, "version": recipe.version, "ref": resolved_ref, - "content_hash": _recipe_content_hash( - recipe, sanitizer, cc, cxx, cflags, cxxflags, ldflags - ), + "recipe": _recipe_fields(recipe), + "platform": _platform_info(), + "build_params": _build_params(sanitizer, cc, cxx, cflags, cxxflags, ldflags), } ensure_dir(recipe.install_dir, dry_run=dry_run, ui=ui) - write_text(stamp, json.dumps(payload, indent=2), dry_run=dry_run, ui=ui) + content = json.dumps(payload, indent=2) + write_text(stamp, content, dry_run=dry_run, ui=ui) + if not dry_run: + ui.info(f"Stamp written to {stamp}") + ui.info(f"Stamp contents:\n{content}") def download_file( @@ -255,7 +417,7 @@ def fetch_recipe_source( if clean and os.path.exists(dest): remove_dir(dest, dry_run=dry_run, ui=ui) - if not dry_run and not force and is_recipe_up_to_date(recipe, resolved_ref): + if not dry_run and not force and not is_recipe_up_to_date(recipe, resolved_ref): ui.ok(f"[{recipe.name}] already up to date ({resolved_ref or 'HEAD'}).") return resolved_ref diff --git a/util/bootstrap/tests/test_archive_fetcher_loader.py b/util/bootstrap/tests/test_archive_fetcher_loader.py index 2a678edf5f..9e55a4a16d 100644 --- a/util/bootstrap/tests/test_archive_fetcher_loader.py +++ b/util/bootstrap/tests/test_archive_fetcher_loader.py @@ -30,7 +30,11 @@ build_archive_url, recipe_stamp_path, is_recipe_up_to_date, - _recipe_content_hash, + _build_params, + _field_diff, + _recipe_fields, + _recipe_hash, + _platform_info, write_recipe_stamp, download_file, fetch_recipe_source, @@ -209,25 +213,35 @@ class TestIsRecipeUpToDate(unittest.TestCase): def test_no_stamp_file(self): """Missing stamp file means not up to date.""" r = _make_recipe(install_dir="/nonexistent") - self.assertFalse(is_recipe_up_to_date(r, "abc")) + self.assertNotEqual(is_recipe_up_to_date(r, "abc"), "") def test_matching_stamp(self): - """Stamp with matching version and ref should be up to date.""" + """Stamp with matching fields should be up to date.""" with tempfile.TemporaryDirectory() as td: r = _make_recipe(install_dir=td) stamp = recipe_stamp_path(r) - payload = {"version": "1.0", "ref": "abc123"} + payload = { + "version": "1.0", "ref": "abc123", + "recipe": _recipe_fields(r), + "platform": _platform_info(), + "build_params": {}, + } with open(stamp, "w") as f: json.dump(payload, f) - self.assertTrue(is_recipe_up_to_date(r, "abc123")) + self.assertEqual(is_recipe_up_to_date(r, "abc123"), "") def test_version_mismatch(self): + """Version change is caught by recipe field comparison.""" with tempfile.TemporaryDirectory() as td: r = _make_recipe(install_dir=td) + old_recipe = _recipe_fields(r) + old_recipe["version"] = "2.0" stamp = recipe_stamp_path(r) with open(stamp, "w") as f: - json.dump({"version": "2.0", "ref": "abc"}, f) - self.assertFalse(is_recipe_up_to_date(r, "abc")) + json.dump({"version": "2.0", "ref": "abc", + "recipe": old_recipe, + "build_params": {}}, f) + self.assertIn("recipe version changed", is_recipe_up_to_date(r, "abc")) def test_ref_mismatch(self): with tempfile.TemporaryDirectory() as td: @@ -235,7 +249,7 @@ def test_ref_mismatch(self): stamp = recipe_stamp_path(r) with open(stamp, "w") as f: json.dump({"version": "1.0", "ref": "old"}, f) - self.assertFalse(is_recipe_up_to_date(r, "new")) + self.assertIn("ref changed", is_recipe_up_to_date(r, "new")) def test_corrupt_stamp(self): with tempfile.TemporaryDirectory() as td: @@ -243,40 +257,252 @@ def test_corrupt_stamp(self): stamp = recipe_stamp_path(r) with open(stamp, "w") as f: f.write("not json") - self.assertFalse(is_recipe_up_to_date(r, "abc")) + self.assertIn("corrupt", is_recipe_up_to_date(r, "abc")) - def test_content_hash_match(self): - """Stamp with valid content_hash should be up to date.""" + def test_build_params_match(self): + """Stamp with matching build_params should be up to date.""" with tempfile.TemporaryDirectory() as td: r = _make_recipe(install_dir=td) - h = _recipe_content_hash(r, sanitizer="", cc="/usr/bin/gcc") + params = _build_params(sanitizer="", cc="/usr/bin/gcc") stamp = recipe_stamp_path(r) with open(stamp, "w") as f: - json.dump({"version": "1.0", "ref": "abc", "content_hash": h}, f) - self.assertTrue(is_recipe_up_to_date(r, "abc", cc="/usr/bin/gcc")) + json.dump({"version": "1.0", "ref": "abc", + "recipe": _recipe_fields(r), + "platform": _platform_info(), + "build_params": params}, f) + self.assertEqual(is_recipe_up_to_date(r, "abc", cc="/usr/bin/gcc"), "") + + def test_build_params_mismatch(self): + """Stamp with different build_params should report which field changed.""" + with tempfile.TemporaryDirectory() as td: + r = _make_recipe(install_dir=td) + params = _build_params(cc="old-gcc") + stamp = recipe_stamp_path(r) + with open(stamp, "w") as f: + json.dump({"version": "1.0", "ref": "abc", + "recipe": _recipe_fields(r), + "build_params": params}, f) + self.assertIn("cc changed", is_recipe_up_to_date(r, "abc", cc="new-gcc")) - def test_content_hash_mismatch(self): - """Stamp with wrong content_hash should not be up to date.""" + def test_recipe_changed(self): + """Stamp with different recipe fields should report which field changed.""" with tempfile.TemporaryDirectory() as td: r = _make_recipe(install_dir=td) + old_recipe = _recipe_fields(r) + old_recipe["build_type"] = "Debug" stamp = recipe_stamp_path(r) with open(stamp, "w") as f: - json.dump({"version": "1.0", "ref": "abc", "content_hash": "wrong"}, f) - self.assertFalse(is_recipe_up_to_date(r, "abc")) + json.dump({"version": "1.0", "ref": "abc", + "recipe": old_recipe, + "build_params": {}}, f) + self.assertIn("recipe build_type changed", is_recipe_up_to_date(r, "abc")) + def test_old_recipe_hash_matching(self): + """Stamp with old recipe_hash format should pass if hash matches.""" + with tempfile.TemporaryDirectory() as td: + r = _make_recipe(install_dir=td) + h = _recipe_hash(r) + stamp = recipe_stamp_path(r) + with open(stamp, "w") as f: + json.dump({"version": "1.0", "ref": "abc", + "recipe_hash": h, + "build_params": {}}, f) + self.assertEqual(is_recipe_up_to_date(r, "abc"), "") -class TestRecipeContentHash(unittest.TestCase): + def test_old_recipe_hash_mismatch(self): + """Stamp with old recipe_hash format should report change if hash differs.""" + with tempfile.TemporaryDirectory() as td: + r = _make_recipe(install_dir=td) + stamp = recipe_stamp_path(r) + with open(stamp, "w") as f: + json.dump({"version": "1.0", "ref": "abc", + "recipe_hash": "stale_hash", + "build_params": {}}, f) + self.assertIn("recipe changed", is_recipe_up_to_date(r, "abc")) + + def test_platform_mismatch(self): + """Stamp from a different platform should trigger rebuild.""" + with tempfile.TemporaryDirectory() as td: + r = _make_recipe(install_dir=td) + stamp = recipe_stamp_path(r) + with open(stamp, "w") as f: + json.dump({"version": "1.0", "ref": "abc", + "recipe": _recipe_fields(r), + "platform": {"os": "FakeOS", "arch": "z80", "os_version": "1.0"}, + "build_params": {}}, f) + result = is_recipe_up_to_date(r, "abc") + self.assertIn("platform", result) + self.assertIn("changed", result) + + def test_old_content_hash_triggers_rebuild(self): + """Stamp with old content_hash format should trigger rebuild.""" + with tempfile.TemporaryDirectory() as td: + r = _make_recipe(install_dir=td) + stamp = recipe_stamp_path(r) + with open(stamp, "w") as f: + json.dump({"version": "1.0", "ref": "abc", "content_hash": "old"}, f) + self.assertIn("old format", is_recipe_up_to_date(r, "abc")) + + +class TestBuildParams(unittest.TestCase): def test_deterministic(self): - r = _make_recipe() - h1 = _recipe_content_hash(r, cc="gcc") - h2 = _recipe_content_hash(r, cc="gcc") - self.assertEqual(h1, h2) + p1 = _build_params(cc="gcc") + p2 = _build_params(cc="gcc") + self.assertEqual(p1, p2) def test_different_params(self): + p1 = _build_params(cc="gcc") + p2 = _build_params(cc="clang") + self.assertNotEqual(p1, p2) + + def test_empty_when_no_params(self): + self.assertEqual(_build_params(), {}) + + +class TestFieldDiff(unittest.TestCase): + def test_simple_change(self): + result = _field_diff("old", "new") + self.assertIn("old", result) + self.assertIn("new", result) + + def test_added(self): + result = _field_diff(None, "value") + self.assertIn("added", result) + + def test_removed(self): + result = _field_diff("value", None) + self.assertIn("removed", result) + + def test_dict_diff(self): + old = json.dumps({"a": 1, "b": 2}) + new = json.dumps({"a": 1, "b": 3}) + result = _field_diff(old, new) + self.assertIn("b", result) + self.assertNotIn("a:", result) + + def test_list_same_length_diff(self): + old = json.dumps([{"x": 1, "y": 2}]) + new = json.dumps([{"x": 1, "y": 3}]) + result = _field_diff(old, new) + self.assertIn("y", result) + self.assertNotIn("x:", result) + + def test_list_different_length(self): + old = json.dumps(["a", "b", "c"]) + new = json.dumps(["a", "b"]) + result = _field_diff(old, new) + self.assertIn("removed", result) + self.assertIn("c", result) + + def test_non_json_fallback(self): + result = _field_diff("plain old", "plain new") + self.assertIn("plain old", result) + self.assertIn("plain new", result) + + +class TestValueDiff(unittest.TestCase): + """Direct tests for _value_diff covering recursion and edge cases.""" + + def test_equal_values_unchanged(self): + """Equal values should report unchanged.""" + from src.recipes.fetcher import _value_diff + result = _value_diff({"a": 1}, {"a": 1}) + self.assertIn("unchanged", result) + + def test_dict_added_key(self): + """A key only in 'new' should appear as added.""" + from src.recipes.fetcher import _value_diff + result = _value_diff({"a": 1}, {"a": 1, "b": 2}) + self.assertIn("added", result) + self.assertIn("b", result) + + def test_dict_removed_key(self): + """A key only in 'old' should appear as removed.""" + from src.recipes.fetcher import _value_diff + result = _value_diff({"a": 1, "b": 2}, {"a": 1}) + self.assertIn("removed", result) + self.assertIn("b", result) + + def test_nested_dict_recursion(self): + """Nested dict diffs should recurse and indent.""" + from src.recipes.fetcher import _value_diff + old = {"outer": {"inner": "old_val"}} + new = {"outer": {"inner": "new_val"}} + result = _value_diff(old, new) + self.assertIn("outer", result) + self.assertIn("inner", result) + self.assertIn("old_val", result) + self.assertIn("new_val", result) + + def test_nested_list_in_dict(self): + """A list-valued key changing should recurse into list diff.""" + from src.recipes.fetcher import _value_diff + old = {"items": [1, 2, 3]} + new = {"items": [1, 2, 4]} + result = _value_diff(old, new) + self.assertIn("items", result) + self.assertIn("3", result) + self.assertIn("4", result) + + def test_list_added_only(self): + """List that only grows should report only the added entries.""" + from src.recipes.fetcher import _value_diff + result = _value_diff([1, 2], [1, 2, 3]) + self.assertIn("added", result) + self.assertIn("3", result) + + def test_list_same_length_all_unchanged(self): + """Same-length lists with all elements equal should report unchanged.""" + from src.recipes.fetcher import _value_diff + result = _value_diff([1, 2, 3], [1, 2, 3]) + self.assertIn("unchanged", result) + + def test_mixed_types_fall_through(self): + """Mismatched container types should fall through to scalar diff.""" + from src.recipes.fetcher import _value_diff + result = _value_diff({"a": 1}, [1, 2]) + self.assertIn("->", result) + + +class TestRecipeFields(unittest.TestCase): + def test_fields_from_recipe(self): + r = _make_recipe() + fields = _recipe_fields(r) + self.assertEqual(fields["name"], "test-lib") + self.assertEqual(fields["version"], "1.0") + self.assertEqual(fields["build_type"], "Release") + self.assertIn("source", fields) + self.assertIn("build", fields) + + def test_deterministic(self): r = _make_recipe() - h1 = _recipe_content_hash(r, cc="gcc") - h2 = _recipe_content_hash(r, cc="clang") - self.assertNotEqual(h1, h2) + f1 = _recipe_fields(r) + f2 = _recipe_fields(r) + self.assertEqual(f1, f2) + + +class TestPlatformInfo(unittest.TestCase): + def test_has_required_keys(self): + info = _platform_info() + self.assertIn("os", info) + self.assertIn("arch", info) + + def test_os_is_known(self): + info = _platform_info() + self.assertIn(info["os"], ("Linux", "Darwin", "Windows")) + + +class TestRecipeHash(unittest.TestCase): + def test_deterministic(self): + r = _make_recipe() + h1 = _recipe_hash(r) + h2 = _recipe_hash(r) + self.assertEqual(h1, h2) + + def test_different_recipe(self): + r1 = _make_recipe(version="1.0") + r2 = _make_recipe(version="2.0") + self.assertNotEqual(_recipe_hash(r1), _recipe_hash(r2)) class TestWriteRecipeStamp(unittest.TestCase): @@ -291,7 +517,8 @@ def test_writes_stamp_file(self): data = json.load(f) self.assertEqual(data["name"], "test-lib") self.assertEqual(data["ref"], "abc123") - self.assertIn("content_hash", data) + self.assertIn("recipe", data) + self.assertIn("build_params", data) def test_dry_run_does_not_write(self): """Dry-run should not create stamp file.""" @@ -339,7 +566,7 @@ def test_up_to_date_skips(self): source_dir="/tmp/src", ) ui = TextUI() - with patch("src.recipes.fetcher.is_recipe_up_to_date", return_value=True): + with patch("src.recipes.fetcher.is_recipe_up_to_date", return_value=""): ref = fetch_recipe_source(r, "/tmp", ui=ui) self.assertEqual(ref, "abc") @@ -353,7 +580,7 @@ def test_source_exists_skips_download(self): source_dir=src_dir, ) ui = TextUI() - with patch("src.recipes.fetcher.is_recipe_up_to_date", return_value=False): + with patch("src.recipes.fetcher.is_recipe_up_to_date", return_value="stale"): ref = fetch_recipe_source(r, td, ui=ui) self.assertEqual(ref, "abc") @@ -361,7 +588,7 @@ def test_source_exists_skips_download(self): @patch("src.recipes.fetcher.download_file") @patch("src.recipes.fetcher.remove_dir") @patch("src.recipes.fetcher.ensure_dir") - @patch("src.recipes.fetcher.is_recipe_up_to_date", return_value=False) + @patch("src.recipes.fetcher.is_recipe_up_to_date", return_value="stale") def test_archive_fetch_zip(self, mock_uptodate, mock_ensure, mock_remove, mock_dl, mock_extract): """GitHub archive should use download + extract_zip_flatten.""" with tempfile.TemporaryDirectory() as td: @@ -379,7 +606,7 @@ def test_archive_fetch_zip(self, mock_uptodate, mock_ensure, mock_remove, mock_d @patch("src.recipes.fetcher.run_cmd") @patch("src.recipes.fetcher.ensure_dir") - @patch("src.recipes.fetcher.is_recipe_up_to_date", return_value=False) + @patch("src.recipes.fetcher.is_recipe_up_to_date", return_value="stale") def test_git_clone_fallback(self, mock_uptodate, mock_ensure, mock_run): """Non-GitHub URL should fall back to git clone.""" with tempfile.TemporaryDirectory() as td: diff --git a/util/bootstrap/tests/test_builder.py b/util/bootstrap/tests/test_builder.py index 1df6802cfb..da0ddc025c 100644 --- a/util/bootstrap/tests/test_builder.py +++ b/util/bootstrap/tests/test_builder.py @@ -53,10 +53,10 @@ def test_cflags_in_configure(self, mock_which, mock_ensure, mock_run): ) configure_call = mock_run.call_args_list[0] cmd = configure_call[0][0] - self.assertTrue( - any("-DCMAKE_C_FLAGS_INIT=-gz=zstd" in arg for arg in cmd), - f"CMAKE_C_FLAGS_INIT not found in configure command: {cmd}" - ) + c_flags = [a for a in cmd if "CMAKE_C_FLAGS_INIT" in a] + self.assertTrue(len(c_flags) > 0, f"CMAKE_C_FLAGS_INIT not found: {cmd}") + self.assertIn("-gz=zstd", c_flags[0], f"User cflags missing: {c_flags[0]}") + self.assertIn("-w", c_flags[0], f"Warning suppression missing: {c_flags[0]}") @patch("src.recipes.builder.run_cmd") @patch("src.recipes.builder.ensure_dir") @@ -71,10 +71,10 @@ def test_cxxflags_in_configure(self, mock_which, mock_ensure, mock_run): ) configure_call = mock_run.call_args_list[0] cmd = configure_call[0][0] - self.assertTrue( - any("-DCMAKE_CXX_FLAGS_INIT=-gz=zstd -O2" in arg for arg in cmd), - f"CMAKE_CXX_FLAGS_INIT not found in configure command: {cmd}" - ) + cxx_flags = [a for a in cmd if "CMAKE_CXX_FLAGS_INIT" in a] + self.assertTrue(len(cxx_flags) > 0, f"CMAKE_CXX_FLAGS_INIT not found: {cmd}") + self.assertIn("-gz=zstd -O2", cxx_flags[0], f"User cxxflags missing: {cxx_flags[0]}") + self.assertIn("-w", cxx_flags[0], f"Warning suppression missing: {cxx_flags[0]}") @patch("src.recipes.builder.run_cmd") @patch("src.recipes.builder.ensure_dir") @@ -101,8 +101,8 @@ def test_ldflags_in_configure(self, mock_which, mock_ensure, mock_run): @patch("src.recipes.builder.run_cmd") @patch("src.recipes.builder.ensure_dir") @patch("shutil.which", return_value="/usr/bin/cmake") - def test_no_flags_when_empty(self, mock_which, mock_ensure, mock_run): - """No FLAGS_INIT args when no user flags or sanitizer.""" + def test_only_warning_suppression_when_no_user_flags(self, mock_which, mock_ensure, mock_run): + """Only -w warning suppression in FLAGS_INIT when no user flags or sanitizer.""" recipe = _make_recipe() step = {"type": "cmake"} run_cmake_recipe_step( @@ -110,9 +110,16 @@ def test_no_flags_when_empty(self, mock_which, mock_ensure, mock_run): ) configure_call = mock_run.call_args_list[0] cmd = configure_call[0][0] + c_flags = [a for a in cmd if "CMAKE_C_FLAGS_INIT" in a] + cxx_flags = [a for a in cmd if "CMAKE_CXX_FLAGS_INIT" in a] + self.assertTrue(len(c_flags) > 0, f"CMAKE_C_FLAGS_INIT expected with -w: {cmd}") + self.assertEqual(c_flags[0], "-DCMAKE_C_FLAGS_INIT=-w") + self.assertTrue(len(cxx_flags) > 0, f"CMAKE_CXX_FLAGS_INIT expected with -w: {cmd}") + self.assertEqual(cxx_flags[0], "-DCMAKE_CXX_FLAGS_INIT=-w") + # No linker flags when no user ldflags self.assertFalse( - any("FLAGS_INIT" in arg for arg in cmd), - f"Unexpected FLAGS_INIT in command: {cmd}" + any("LINKER_FLAGS_INIT" in arg for arg in cmd), + f"Unexpected LINKER_FLAGS_INIT: {cmd}" ) @patch("src.recipes.builder.run_cmd") @@ -382,5 +389,51 @@ def test_flags_passed_to_cmake_step(self, mock_step): self.assertIn("-fuse-ld=lld", all_args, "ldflags not passed through") +class TestExtraCmakeOptions(unittest.TestCase): + """Test that extra_cmake_options are appended to the configure command.""" + + @patch("src.recipes.builder.run_cmd") + @patch("src.recipes.builder.ensure_dir") + @patch("shutil.which", return_value="/usr/bin/cmake") + def test_extra_options_appended_to_configure(self, mock_which, mock_ensure, mock_run): + """extra_cmake_options should appear in the configure command.""" + recipe = _make_recipe() + step = {"type": "cmake"} + run_cmake_recipe_step( + recipe, step, "/src", "/third-party", "preset", + extra_cmake_options=["-DLLVM_ENABLE_RUNTIMES=", "-DFOO=bar"], + ) + configure_call = mock_run.call_args_list[0] + cmd = configure_call[0][0] + self.assertIn("-DLLVM_ENABLE_RUNTIMES=", cmd) + self.assertIn("-DFOO=bar", cmd) + + @patch("src.recipes.builder.run_cmd") + @patch("src.recipes.builder.ensure_dir") + @patch("shutil.which", return_value="/usr/bin/cmake") + def test_no_extra_options_means_no_extra_flags(self, mock_which, mock_ensure, mock_run): + """No extra_cmake_options should leave the command untouched.""" + recipe = _make_recipe() + step = {"type": "cmake"} + run_cmake_recipe_step( + recipe, step, "/src", "/third-party", "preset", + ) + configure_call = mock_run.call_args_list[0] + cmd = configure_call[0][0] + self.assertFalse(any("LLVM_ENABLE_RUNTIMES" in arg for arg in cmd)) + + @patch("src.recipes.builder.run_cmake_recipe_step") + def test_extra_options_passed_through_build_recipe(self, mock_step): + """build_recipe should forward extra_cmake_options to the cmake step.""" + recipe = _make_recipe() + build_recipe( + recipe, "/src", "/third-party", "preset", + extra_cmake_options=["-DLLVM_ENABLE_RUNTIMES="], + ) + mock_step.assert_called_once() + all_args = list(mock_step.call_args[0]) + list(mock_step.call_args[1].values()) + self.assertIn(["-DLLVM_ENABLE_RUNTIMES="], all_args) + + if __name__ == "__main__": unittest.main() diff --git a/util/bootstrap/tests/test_cache_dir.py b/util/bootstrap/tests/test_cache_dir.py index 08dd677f10..6ae04e63a0 100644 --- a/util/bootstrap/tests/test_cache_dir.py +++ b/util/bootstrap/tests/test_cache_dir.py @@ -155,9 +155,9 @@ def test_stamp_makes_recipe_up_to_date(self): recipe = make_recipe(name="llvm", version="abc123", install_dir=install_dir) resolved_ref = "abc123def456" - self.assertFalse(is_recipe_up_to_date(recipe, resolved_ref)) + self.assertNotEqual(is_recipe_up_to_date(recipe, resolved_ref), "") write_recipe_stamp(recipe, resolved_ref) - self.assertTrue(is_recipe_up_to_date(recipe, resolved_ref)) + self.assertEqual(is_recipe_up_to_date(recipe, resolved_ref), "") def test_stamp_version_mismatch(self): """Stamp with different version should not be up to date.""" @@ -165,7 +165,7 @@ def test_stamp_version_mismatch(self): recipe = make_recipe(name="llvm", version="abc123", install_dir=install_dir) write_recipe_stamp(recipe, "old_ref") - self.assertFalse(is_recipe_up_to_date(recipe, "new_ref")) + self.assertNotEqual(is_recipe_up_to_date(recipe, "new_ref"), "") class TestLegacyCacheDetection(unittest.TestCase): @@ -197,7 +197,7 @@ def test_nonempty_dir_without_stamp_is_legacy(self): resolved_ref = "abc123" # Should NOT be up to date (no stamp) - self.assertFalse(is_recipe_up_to_date(recipe, resolved_ref)) + self.assertNotEqual(is_recipe_up_to_date(recipe, resolved_ref), "") # But directory is non-empty self.assertTrue(os.path.isdir(install_dir)) self.assertTrue(len(os.listdir(install_dir)) > 0) @@ -213,13 +213,13 @@ def test_legacy_cache_gets_stamp_after_write(self): resolved_ref = "abc123def456" # Initially no stamp - self.assertFalse(is_recipe_up_to_date(recipe, resolved_ref)) + self.assertNotEqual(is_recipe_up_to_date(recipe, resolved_ref), "") # Write stamp (simulating what installer does for legacy caches) write_recipe_stamp(recipe, resolved_ref) # Now it should be up to date - self.assertTrue(is_recipe_up_to_date(recipe, resolved_ref)) + self.assertEqual(is_recipe_up_to_date(recipe, resolved_ref), "") def test_ci_compatible_cache_path(self): """Cache dir structure matches CI: //.""" diff --git a/util/bootstrap/tests/test_installer.py b/util/bootstrap/tests/test_installer.py index 050c5f90ce..fd065ce5ff 100644 --- a/util/bootstrap/tests/test_installer.py +++ b/util/bootstrap/tests/test_installer.py @@ -273,6 +273,33 @@ def test_writes_sanitizer_flag(self): content = f.read() self.assertIn("BOOTSTRAP_LDFLAGS=-fsanitize=address", content) + def test_writes_bootstrap_rebuilt(self): + """When recipes are rebuilt, BOOTSTRAP_REBUILT lists them.""" + with tempfile.TemporaryDirectory() as tmp: + env_path = os.path.join(tmp, "env.txt") + inst = _make_installer() + inst.options.env_file = env_path + inst.options.dry_run = False + inst.rebuilt_recipes = ["llvm", "lua"] + inst.write_env_file() + with open(env_path) as f: + content = f.read() + self.assertIn("BOOTSTRAP_REBUILT=llvm,lua", content) + + def test_omits_bootstrap_rebuilt_when_empty(self): + """No rebuilds means no BOOTSTRAP_REBUILT line.""" + with tempfile.TemporaryDirectory() as tmp: + env_path = os.path.join(tmp, "env.txt") + inst = _make_installer() + inst.options.env_file = env_path + inst.options.dry_run = False + inst.package_roots = {"FOO": "bar"} + # rebuilt_recipes is empty by default + inst.write_env_file() + with open(env_path) as f: + content = f.read() + self.assertNotIn("BOOTSTRAP_REBUILT", content) + def test_dry_run_prints_to_stdout(self): inst = _make_installer() inst.options.env_file = "/tmp/test_env.txt" @@ -507,7 +534,7 @@ def test_install_dependencies_no_recipes_raises(self, mock_load): with self.assertRaises(RuntimeError): inst.install_dependencies() - @patch("src.installer.is_recipe_up_to_date", return_value=True) + @patch("src.installer.is_recipe_up_to_date", return_value="") @patch("src.installer.topo_sort_recipes", side_effect=lambda x: x) @patch("src.installer.load_recipe_files") def test_install_dependencies_skips_up_to_date(self, mock_load, mock_topo, mock_uptodate): @@ -555,6 +582,77 @@ def test_install_dependencies_cache_dir_overrides_install_dir( self.assertEqual(recipe.install_dir, "/cache/llvm") + @patch("src.installer.write_recipe_stamp") + @patch("src.installer.build_recipe") + @patch("src.installer.apply_recipe_patches") + @patch("src.installer.fetch_recipe_source") + @patch("src.installer.topo_sort_recipes", side_effect=lambda x: x) + @patch("src.installer.load_recipe_files") + def test_install_dependencies_tracks_rebuilt_recipes( + self, mock_load, mock_topo, mock_fetch, mock_patch, mock_build, mock_stamp + ): + """Rebuilt recipes should be appended to inst.rebuilt_recipes.""" + r1 = self._make_recipe("llvm") + r2 = self._make_recipe("lua", "5.4") + mock_load.return_value = [r1, r2] + + inst = _make_installer() + inst.install_dependencies() + + self.assertEqual(inst.rebuilt_recipes, ["llvm", "lua"]) + + @patch("src.installer.build_libcxx_runtimes") + @patch("src.installer.needs_libcxx_runtimes", return_value=True) + @patch("src.installer.libcxx_runtime_flags", + return_value={"cxxflags": "-isystem /opt/llvm/include/c++/v1", + "ldflags": "-L/opt/llvm/lib -lc++"}) + @patch("src.installer.write_recipe_stamp") + @patch("src.installer.build_recipe") + @patch("src.installer.apply_recipe_patches") + @patch("src.installer.fetch_recipe_source") + @patch("src.installer.topo_sort_recipes", side_effect=lambda x: x) + @patch("src.installer.load_recipe_files") + def test_install_dependencies_disables_main_llvm_runtimes_when_libcxx_built( + self, mock_load, mock_topo, mock_fetch, mock_patch, mock_build, mock_stamp, + mock_flags, mock_needs, mock_libcxx + ): + """LLVM + clang + ASan/MSan should pass -DLLVM_ENABLE_RUNTIMES= to the + main LLVM build via extra_cmake_options, so the instrumented libc++ + we just built isn't overwritten.""" + recipe = self._make_recipe("llvm") + mock_load.return_value = [recipe] + + inst = _make_installer(sanitizer="address") + inst.compiler_info = {"CMAKE_CXX_COMPILER_ID": "Clang"} + inst.install_dependencies() + + mock_libcxx.assert_called_once() + # Find the extra_cmake_options arg in the build_recipe call + # (positional, between ldflags and force). + build_args = mock_build.call_args[0] + self.assertIn(["-DLLVM_ENABLE_RUNTIMES="], build_args) + + @patch("src.installer.needs_libcxx_runtimes", return_value=False) + @patch("src.installer.write_recipe_stamp") + @patch("src.installer.build_recipe") + @patch("src.installer.apply_recipe_patches") + @patch("src.installer.fetch_recipe_source") + @patch("src.installer.topo_sort_recipes", side_effect=lambda x: x) + @patch("src.installer.load_recipe_files") + def test_install_dependencies_no_extra_cmake_options_without_libcxx( + self, mock_load, mock_topo, mock_fetch, mock_patch, mock_build, mock_stamp, + mock_needs + ): + """Without libc++ runtimes, build_recipe receives None for extra_cmake_options.""" + recipe = self._make_recipe("llvm") + mock_load.return_value = [recipe] + + inst = _make_installer() + inst.install_dependencies() + + build_args = mock_build.call_args[0] + self.assertIn(None, build_args) + class TestCreatePresets(unittest.TestCase): """Tests for create_presets with mocked create_cmake_presets.""" diff --git a/util/bootstrap/tests/test_main.py b/util/bootstrap/tests/test_main.py index 2f389a9807..30bd1c6011 100644 --- a/util/bootstrap/tests/test_main.py +++ b/util/bootstrap/tests/test_main.py @@ -342,6 +342,39 @@ def test_supports_emoji_default(self): with patch.dict(os.environ, env, clear=True): self.assertTrue(TextUI._supports_emoji()) + def test_supports_emoji_ascii_encoding_disabled(self): + """ASCII (and similar narrow) stdout encodings should disable emoji.""" + env = os.environ.copy() + env.pop("BOOTSTRAP_PLAIN", None) + for encoding in ("ascii", "ASCII", "charmap", "cp1252", "latin-1", "iso-8859-1"): + mock_stdout = MagicMock() + mock_stdout.encoding = encoding + with patch.dict(os.environ, env, clear=True), \ + patch("src.core.ui.sys.stdout", mock_stdout): + self.assertFalse( + TextUI._supports_emoji(), + f"expected emoji disabled for encoding {encoding!r}" + ) + + def test_supports_emoji_utf8_encoding_enabled(self): + """UTF-8 stdout encoding should keep emoji enabled.""" + env = os.environ.copy() + env.pop("BOOTSTRAP_PLAIN", None) + mock_stdout = MagicMock() + mock_stdout.encoding = "utf-8" + with patch.dict(os.environ, env, clear=True), \ + patch("src.core.ui.sys.stdout", mock_stdout): + self.assertTrue(TextUI._supports_emoji()) + + def test_supports_emoji_no_encoding_attribute(self): + """Missing encoding attribute should not crash; default to enabled.""" + env = os.environ.copy() + env.pop("BOOTSTRAP_PLAIN", None) + mock_stdout = object() # no .encoding attribute + with patch.dict(os.environ, env, clear=True), \ + patch("src.core.ui.sys.stdout", mock_stdout): + self.assertTrue(TextUI._supports_emoji()) + class TestTextUIFmtColor(unittest.TestCase): """Test _fmt with color enabled.""" @@ -354,6 +387,10 @@ def setUp(self): self.ui.base_path = None self.ui.base_token = "." self.ui.dry_run = False + self.ui._ci = False + self.ui._ci_group_open = False + self.ui._ci_group_title = "" + self.ui._ci_group_start = 0.0 def test_fmt_with_color(self): """_fmt with color should wrap text in ANSI codes.""" @@ -414,6 +451,10 @@ def setUp(self): self.ui.base_path = None self.ui.base_token = "." self.ui.dry_run = False + self.ui._ci = False + self.ui._ci_group_open = False + self.ui._ci_group_title = "" + self.ui._ci_group_start = 0.0 @patch("sys.stdout", new_callable=io.StringIO) def test_subsection_non_plain_has_underline(self, mock_out): @@ -661,6 +702,10 @@ def setUp(self): self.ui.base_path = None self.ui.base_token = "." self.ui.dry_run = False + self.ui._ci = False + self.ui._ci_group_open = False + self.ui._ci_group_title = "" + self.ui._ci_group_start = 0.0 @patch("sys.stdout", new_callable=io.StringIO) def test_checklist_unicode_marks(self, mock_out): @@ -755,5 +800,78 @@ def test_error_block_multiple_tips(self, mock_err): self.assertIn("tip3", output) +class TestTextUICIGroups(unittest.TestCase): + """Test CI group emission.""" + + def _make_ci_ui(self): + ui = TextUI.__new__(TextUI) + ui.color_enabled = False + ui.emoji_enabled = False + ui.max_path = 50 + ui.base_path = None + ui.base_token = "." + ui.dry_run = False + ui._ci = True + ui._ci_group_open = False + ui._ci_group_title = "" + ui._ci_group_start = 0.0 + return ui + + @patch("sys.stdout", new_callable=io.StringIO) + def test_start_group_emits_group_command(self, mock_out): + ui = self._make_ci_ui() + ui.start_group("My Group") + self.assertIn("::group::My Group", mock_out.getvalue()) + self.assertTrue(ui._ci_group_open) + + @patch("sys.stdout", new_callable=io.StringIO) + def test_end_group_emits_endgroup(self, mock_out): + ui = self._make_ci_ui() + ui.start_group("Test") + mock_out.truncate(0) + mock_out.seek(0) + ui.end_group() + output = mock_out.getvalue() + self.assertIn("::endgroup::", output) + self.assertIn("completed in", output) + self.assertFalse(ui._ci_group_open) + + @patch("sys.stdout", new_callable=io.StringIO) + def test_end_group_noop_when_no_group(self, mock_out): + ui = self._make_ci_ui() + ui.end_group() + self.assertEqual(mock_out.getvalue(), "") + + @patch("sys.stdout", new_callable=io.StringIO) + def test_start_group_closes_previous(self, mock_out): + ui = self._make_ci_ui() + ui.start_group("First") + ui.start_group("Second") + output = mock_out.getvalue() + self.assertIn("::group::First", output) + self.assertIn("::endgroup::", output) + self.assertIn("::group::Second", output) + + @patch("sys.stdout", new_callable=io.StringIO) + def test_no_groups_outside_ci(self, mock_out): + ui = self._make_ci_ui() + ui._ci = False + ui.start_group("Should Not Appear") + self.assertNotIn("::group::", mock_out.getvalue()) + self.assertFalse(ui._ci_group_open) + + @patch("sys.stdout", new_callable=io.StringIO) + def test_section_starts_group_in_ci(self, mock_out): + ui = self._make_ci_ui() + ui.section("My Section") + self.assertIn("::group::My Section", mock_out.getvalue()) + + @patch("sys.stdout", new_callable=io.StringIO) + def test_subsection_starts_group_in_ci(self, mock_out): + ui = self._make_ci_ui() + ui.subsection("My Sub") + self.assertIn("::group::My Sub", mock_out.getvalue()) + + if __name__ == "__main__": unittest.main() diff --git a/util/danger/README.md b/util/danger/README.md index e994120d98..af27537cb1 100644 --- a/util/danger/README.md +++ b/util/danger/README.md @@ -33,10 +33,9 @@ npm --prefix util/danger run danger:ci # run Danger in CI mode (requires Git - Scopes reflect the MrDocs tree: `source`, `tests`, `golden-tests`, `docs`, `ci`, `build`, `tooling`, `third-party`, `other`. - Conventional commit types allowed: `feat, fix, docs, style, refactor, perf, test, build, ci, chore, revert`. - Non-test commit size notice triggers at 2000 lines of churn (tests and golden fixtures ignored) and is informational. +- Feature PRs (`feat:` commits, `feat` PR title, or `feature` label) without any `docs/` change get a light warning. Opt out with the `no-docs-needed` label or a `[skip danger docs]` marker in the PR body. ## Updating rules - Edit `logic.ts`, refresh `fixtures/` as needed, and run `npm test`. - Keep warnings human-readable; prefer `warn()` over `fail()` until the team decides otherwise. - -> Note: The Danger.js rules for MrDocs are still experimental, so some warnings may be rough or occasionally fire as false positives—feedback is welcome. diff --git a/util/danger/format.test.ts b/util/danger/format.test.ts index 24b3b7fb62..e9996eeca3 100644 --- a/util/danger/format.test.ts +++ b/util/danger/format.test.ts @@ -25,7 +25,6 @@ describe("renderDangerReport", () => { const output = renderDangerReport(result); - expect(output.startsWith("> 🚧 Danger.js checks for MrDocs")).toBe(true); expect(output).toContain("## ⚠️ Warnings"); expect((output.match(/> \[!WARNING\]/g) || []).length).toBe(2); expect(output).toContain("## 🧾 Changes by Scope"); @@ -59,7 +58,6 @@ describe("renderDangerReport", () => { expect(output).toMatch(/\|\s*\*\*Total\*\*\s*\|\s*100%\s*\|\s*\*\*6\*\*\s*\|\s*5\s*\|\s*1\s*\|\s*\*\*2\*\*\s*\|/); // No highlights section when no golden tests changed expect(output).not.toContain("## ✨ Highlights"); - expect(output.trim().startsWith("> 🚧 Danger.js checks for MrDocs")).toBe(true); }); it("treats removed files as positive file deltas", () => { diff --git a/util/danger/format.ts b/util/danger/format.ts index 447be15bec..cb935fcdff 100644 --- a/util/danger/format.ts +++ b/util/danger/format.ts @@ -9,8 +9,6 @@ // import { formatChurn, scopeDisplayOrder, type DangerResult, type ScopeReport, type ScopeTotals } from "./logic"; -const notice = "> 🚧 Danger.js checks for MrDocs are experimental; expect some rough edges while we tune the rules."; - const scopeLabels: Record = { source: "Source", tests: "Unit Tests", @@ -243,7 +241,6 @@ function renderTopChanges(summary: ScopeReport): string { */ export function renderDangerReport(result: DangerResult): string { const sections = [ - notice, renderWarnings(result.warnings), renderInfos(result.infos), renderHighlights(result.summary.highlights), diff --git a/util/danger/logic.test.ts b/util/danger/logic.test.ts index 9f3368ca65..fedebde6d1 100644 --- a/util/danger/logic.test.ts +++ b/util/danger/logic.test.ts @@ -122,4 +122,85 @@ describe("starterChecks", () => { expect(warnings.some((message) => message.includes("Source changed"))).toBe(false); }); + + // Warns when a feat commit ships without any documentation update. + it("warns when a feat commit ships without docs", () => { + const inputs: DangerInputs = { + files: [], + commits: [], + prBody: "Adds a shiny new generator option.\n\nTesting: ran golden tests locally.", + prTitle: "feat: shiny option", + labels: [], + }; + + const summary = summarizeScopes([ + { filename: "src/lib/Gen/option.cpp", additions: 30, deletions: 1 }, + { filename: "src/test/option.cpp", additions: 20, deletions: 0 }, + ]); + const parsed = validateCommits([{ sha: "3", message: "feat: shiny option" }]).parsed; + const warnings = basicChecks(inputs, summary, parsed); + + expect(warnings.some((message) => message.includes("does not update any documentation"))).toBe(true); + }); + + // Stays quiet when a feat commit also touches docs. + it("does not warn when a feat commit updates docs", () => { + const inputs: DangerInputs = { + files: [], + commits: [], + prBody: "Adds a shiny new generator option.\n\nTesting: ran golden tests.", + prTitle: "feat: shiny option", + labels: [], + }; + + const summary = summarizeScopes([ + { filename: "src/lib/Gen/option.cpp", additions: 30, deletions: 1 }, + { filename: "src/test/option.cpp", additions: 20, deletions: 0 }, + { filename: "docs/modules/ROOT/pages/options.adoc", additions: 12, deletions: 0 }, + ]); + const parsed = validateCommits([{ sha: "4", message: "feat: shiny option" }]).parsed; + const warnings = basicChecks(inputs, summary, parsed); + + expect(warnings.some((message) => message.includes("does not update any documentation"))).toBe(false); + }); + + // Honors explicit opt-out labels. + it("respects no-docs-needed label", () => { + const inputs: DangerInputs = { + files: [], + commits: [], + prBody: "Adds an internal-only feature flag.\n\nTesting: covered by existing suites.", + prTitle: "feat: internal flag", + labels: ["no-docs-needed"], + }; + + const summary = summarizeScopes([ + { filename: "src/lib/internal.cpp", additions: 5, deletions: 0 }, + { filename: "src/test/internal.cpp", additions: 5, deletions: 0 }, + ]); + const parsed = validateCommits([{ sha: "5", message: "feat: internal flag" }]).parsed; + const warnings = basicChecks(inputs, summary, parsed); + + expect(warnings.some((message) => message.includes("does not update any documentation"))).toBe(false); + }); + + // Stays quiet for non-feature commits even without docs. + it("does not warn for fix commits without docs", () => { + const inputs: DangerInputs = { + files: [], + commits: [], + prBody: "Fixes off-by-one.\n\nTesting: added a regression unit test.", + prTitle: "fix: off-by-one", + labels: [], + }; + + const summary = summarizeScopes([ + { filename: "src/lib/loop.cpp", additions: 2, deletions: 2 }, + { filename: "src/test/loop.cpp", additions: 8, deletions: 0 }, + ]); + const parsed = validateCommits([{ sha: "6", message: "fix: off-by-one" }]).parsed; + const warnings = basicChecks(inputs, summary, parsed); + + expect(warnings.some((message) => message.includes("does not update any documentation"))).toBe(false); + }); }); diff --git a/util/danger/logic.ts b/util/danger/logic.ts index a9fb9e1437..4b79445768 100644 --- a/util/danger/logic.ts +++ b/util/danger/logic.ts @@ -142,6 +142,8 @@ const scopeFormat = /^[a-z0-9._/-]+$/i; const typeSet = new Set(allowedTypes); const skipTestLabels = new Set(["no-tests-needed", "skip-tests", "tests-not-required"]); const skipTestMarkers = ["[skip danger tests]", "[danger skip tests]"]; +const skipDocsLabels = new Set(["no-docs-needed", "skip-docs", "docs-not-required"]); +const skipDocsMarkers = ["[skip danger docs]", "[danger skip docs]"]; const nonTestCommitLimit = 2000; /** @@ -515,6 +517,21 @@ function hasSkipTests(prBody: string, labels: string[]): boolean { return skipTestMarkers.some((marker) => body.includes(marker)); } +/** + * Check for explicit signals to skip the feature-without-docs warning. + * + * @param prBody pull request body text. + * @param labels labels applied to the pull request. + * @returns true when skip markers or labels are present. + */ +function hasSkipDocs(prBody: string, labels: string[]): boolean { + if (labels.some((label) => skipDocsLabels.has(label))) { + return true; + } + const body = prBody.toLowerCase(); + return skipDocsMarkers.some((marker) => body.includes(marker)); +} + /** * Additional hygiene checks around PR description, test coverage signals, and coherence. * @@ -557,6 +574,19 @@ export function basicChecks(input: DangerInputs, scopes: ScopeReport, parsedComm warnings.push("Source changed but no tests or fixtures were updated."); } + const featureSignal = + commitTypes.has("feat") || + /^feat[(:]/i.test(input.prTitle || "") || + input.labels.some((label) => /(^|[-_ ])feature([-_ ]|$)/i.test(label)); + + const skipDocs = hasSkipDocs(input.prBody || "", input.labels); + if (featureSignal && !skipDocs && scopes.totals.docs.files === 0) { + // === New feature without docs updates warnings === + warnings.push( + "This PR looks like it adds a new feature but does not update any documentation. Please document the new functionality under `docs/`, or add a `no-docs-needed` label / `[skip danger docs]` marker if not applicable.", + ); + } + const totalFiles = Object.values(scopes.totals).reduce((sum, scope) => sum + scope.files, 0); const nonDocFiles = totalFiles - scopes.totals.docs.files; const testFiles = scopes.totals.tests.files + scopes.totals["golden-tests"].files;