ENT-14443: Added cfengine test, a one-stop command to lint/build/deploy/run a policy-set

.coveragerc

@@ -1,5 +1,5 @@
 [run]
 parallel = True
-source = src
+source = ${PROJECT_ROOT}/src
 concurrency = multiprocessing
 sigterm = True

Makefile

@@ -23,7 +23,9 @@ install:
 	pipx install --force --editable .
 
 
-export COVERAGE_PROCESS_START = $(PWD)/.coveragerc
+coverage: export COVERAGE_PROCESS_START = $(PWD)/.coveragerc
+coverage: export COVERAGE_FILE = $(PWD)/.coverage
+coverage: export PROJECT_ROOT = $(PWD)
 coverage:
 	uv run coverage erase
 	uv run coverage run --parallel-mode -m pytest

pyproject.toml

@@ -43,7 +43,7 @@ cfengine = "cfengine_cli.main:main"
 license-files = [] # Workaround bug in setuptools https://github.com/astral-sh/uv/issues/9513
 
 [tool.setuptools.package-data]
-cfengine_cli = ["*.json"] # syntax-description.json
+cfengine_cli = ["*.json", "docker/test-agent/Dockerfile"] # syntax-description.json, cfengine test's container
 
 [tool.pyright]
 include = ["src"]

src/cfengine_cli/cfengine_wrapper/cfengine_commands.py

@@ -22,6 +22,7 @@
 )
 
 from cfengine_cli.utils import UserError
+from cfengine_cli.container import run_in_container
 from cfengine_cli.cfengine_wrapper.cfengine_objects import (
     Executable,
     ensure_default_agent_flags,
@@ -271,10 +272,7 @@ def deploy(
 
     # TODO/WOULD be nice: Deploy without run (CFE-4704: https://northerntech.atlassian.net/browse/CFE-4704)
     if hubs:
-        # cf-remote functions use "localhost" (not "local" as it is here)
-        deploy_targets = [
-            "localhost" if location == "local" else location for location in hubs
-        ]
+        deploy_targets = [location for location in hubs]
         error = deploy_command(deploy_targets, masterfiles)
     else:
         return deploy_command(hubs, masterfiles)
@@ -287,6 +285,14 @@ def deploy(
     return error
 
 
+def test() -> int:
+    rc = build_command()
+    if rc != 0:
+        return rc
+
+    return run_in_container("out/masterfiles")
+
+
 def show(target: list[str] | None = None) -> int:
     if target == [] or target is None:
         return show_command(False)

src/cfengine_cli/cfengine_wrapper/cfengine_objects.py

@@ -27,22 +27,22 @@ def ensure_default_agent_flags(command: str) -> str:
 class Executable:
     """
     A single binary (cf-agent or cf-hub) at a known location -- either
-    "local" or a remote host identifier ("user@ip"). Knows its own path
+    localhost or a remote host identifier ("user@ip"). Knows its own path
     and how to run a command against itself, whether that means a local
     subprocess or an SSH call via cf-remote.
     """
 
     def __init__(self, name: str, location: str, path: str, aliases=None) -> None:
         self.name = name  # "cf-agent" / "cf-hub"
-        self.location = location  # "local" or "user@ip"
+        self.location = location  # "localhost" or "user@ip"
         self.path = path  # absolute path to the binary at that location
         self.aliases = (
             aliases or []
         )  # friendly names from cf-remote's saved state, e.g. "hub", "local", "remote"
 
     @property
     def is_local(self) -> bool:
-        return self.location == "local"
+        return self.location == "localhost"
 
     @property
     def label(self) -> str:
@@ -102,7 +102,7 @@ class Installation:
 
     @property
     def is_local(self) -> bool:
-        return self.location == "local"
+        return self.location == "localhost"
 
     @property
     def aliases(self) -> list:

src/cfengine_cli/cfengine_wrapper/cfengine_utils.py

@@ -111,16 +111,16 @@ def _hosts_with_info(role_filter=None):
 
 def _identities(binary_name: str) -> Iterator[_Id]:
     """local + every known host as (location, aliases), without connecting."""
-    yield _Id("local", ["localhost"])
+    yield _Id("localhost", ["localhost", "local"])
     for host, aliases in _known_hosts(None if binary_name == "cf-agent" else "hub"):
         yield _Id(host, aliases)
 
 
 def _resolve(binary_name: str, ident: _Id) -> Executable | None:
     """Connect (if remote) and build an Executable, or None if unavailable."""
-    if ident.location == "local":
+    if ident.location == "localhost":
         path = _find_local_path(binary_name)
-        return Executable(binary_name, "local", path) if path else None
+        return Executable(binary_name, "localhost", path) if path else None
     data = _host_info(ident.location)
     if not data:
         return None

src/cfengine_cli/commands.py

@@ -12,6 +12,7 @@
 from cfengine_cli.initialize_project import init_policy_module, init_promise_type
 from cf_remote.paths import cf_remote_dir
 from cfbs.commands import init_command
+from cfengine_cli.cfengine_wrapper.cfengine_commands import test as _cfengine_test
 from pydantic import ValidationError
 
 
@@ -47,6 +48,17 @@ def lint(files, strict, syntax_path) -> int:
     return errors
 
 
+def test(files, strict) -> int:
+    errors = _lint(files, strict, None)
+    if errors != 0:
+        plural = "error" if errors == 1 else "errors"
+        print(f"Lint failed, {errors} {plural} in total. Skipping build/deploy/run.")
+        return errors
+
+    print("Lint passed, no errors found.")
+    return _cfengine_test()
+
+
 def dev(subcommand, args) -> int:
     return dispatch_dev_subcommand(subcommand, args)
 

src/cfengine_cli/container.py

@@ -0,0 +1,50 @@
+import os
+import shutil
+import subprocess
+
+from cfengine_cli.utils import UserError
+
+_DOCKERFILE_DIR = os.path.join(os.path.dirname(__file__), "docker", "test-agent")
+_IMAGE_TAG = "cfengine-cli-test-agent:latest"
+
+
+def require_docker() -> None:
+    if shutil.which("docker") is None:
+        raise UserError(
+            "`cfengine test` runs the built policy set inside a Docker container -- install Docker to use it."
+        )
+
+
+def _ensure_image_built() -> None:
+    result = subprocess.run(
+        ["docker", "build", "-q", "-t", _IMAGE_TAG, _DOCKERFILE_DIR]
+    )
+    if result.returncode != 0:
+        raise UserError("Failed to build the cfengine-test Docker image.")
+
+
+def run_in_container(masterfiles_dir: str) -> int:
+    """
+    Runs cf-agent against the given built masterfiles directory inside a container
+    """
+    require_docker()
+    _ensure_image_built()
+
+    abs_masterfiles = os.path.abspath(masterfiles_dir)
+    result = subprocess.run(
+        [
+            "docker",
+            "run",
+            "--rm",
+            "-v",
+            f"{abs_masterfiles}:/mnt/masterfiles:ro",
+            _IMAGE_TAG,
+            "sh",
+            "-c",
+            "rm -rf /var/cfengine/inputs "
+            "&& cp -r /mnt/masterfiles /var/cfengine/inputs "
+            "&& /var/cfengine/bin/cf-agent -KIf update.cf "
+            "&& /var/cfengine/bin/cf-agent -KI",
+        ]
+    )
+    return result.returncode

src/cfengine_cli/docker/test-agent/Dockerfile

@@ -0,0 +1,17 @@
+# Install cfengine community (client)
+# `cfengine test` never needs a real hub, given that it is only supposed
+# to check and execute a policy set.
+FROM debian:12-slim
+ENV PATH="/root/.local/bin:${PATH}"
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        python3 pipx sudo
+RUN pipx install cfengine --force
+# RUN cfengine install --hub localhost --edition community
+# Does not work since localhost resolves to `local` in cfengine cli
+RUN cfengine install --clients localhost --edition community
+
+# The package promise module wrapper (and other Python-backed modules) look
+# for an interpreter here
+RUN ln -s "$(command -v python3)" /var/cfengine/bin/cfengine-selected-python

src/cfengine_cli/main.py

@@ -88,6 +88,24 @@ def _get_arg_parser():
     )
     lnt.add_argument("files", nargs="*", help="Files to lint")
 
+    tst = subp.add_parser(
+        "test",
+        help="Lint, then build/run a cfbs project in a throwaway container as one non-interactive check",
+        description="A convenience wrapper chaining `cfengine lint`, `cfengine build`, `cfengine deploy` and `cfengine run` "
+        "to quickly test a module or policy-set end-to-end.",
+    )
+    tst.add_argument(
+        "--strict",
+        type=str,
+        default="yes",
+        help="Strict mode for linting. Default=yes, checks for undefined promise types, bundles, bodies, functions",
+    )
+    tst.add_argument(
+        "files",
+        nargs="*",
+        help="Files/Folder to lint (default is . (the entire project) )",
+    )
+
     dev_parser = subp.add_parser(
         "dev", help="Utilities intended for developers / maintainers of CFEngine"
     )
@@ -245,6 +263,11 @@ def run_command_with_args(args) -> int:
             (args.strict.lower() in ("y", "ye", "yes")),
             args.syntax_description,
         )
+    if args.command == "test":
+        return commands.test(
+            args.files,
+            (args.strict.lower() in ("y", "ye", "yes")),
+        )
     if args.command == "report":
         return cfengine_commands.report(
             target=args.hub,
@@ -376,7 +399,7 @@ def validate_args(args):
     if "hub" in args and args.hub:
         log.debug(f"validate_args, hubs in args, args.hub='{args.hub}'")
         if args.hub in ["local", "localhost"]:
-            args.hub = ["local"]
+            args.hub = ["localhost"]
         else:
             args.hub = resolve_hosts(args.hub)
 

tests/run-shell-tests.sh

@@ -2,6 +2,7 @@
 
 set -e
 # set -x
+export CFBS_USER_AGENT=CI # this user agent will be excluded from the build modules statistics
 
 echo "These tests expect cfengine CLI to be installed globally or in venv"
 
@@ -15,6 +16,11 @@ ls -al tests/shell/00*.sh
 rm -rf tmp
 mkdir -p tmp
 
+export GIT_CONFIG_GLOBAL="$(pwd)/tmp/gitconfig"
+export GIT_CONFIG_SYSTEM=/dev/null
+git config --global user.name "test-runner[bot]"
+git config --global user.email "test_runner@bot"
+
 echo "Run shell tests:"
 for file in tests/shell/*.sh; do
   bash $file

tests/shell/006-test-fail.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -e
+set -x
+
+# Setup: create a temp directory for test files
+tmpdir=$(mktemp -d)
+trap "rm -rf $tmpdir" EXIT
+
+cfengine test --help
+
+# A file with a lint error should make `cfengine test` fail on the lint step
+# and not go through to build/deploy/run.
+printf 'bundle agent main\n{\n  reports:\n      "missing semicolon"\n}\n' > "$tmpdir/bad.cf"
+
+output_file=$(mktemp)
+trap "rm -rf $tmpdir $output_file" EXIT
+
+if cfengine test "$tmpdir/bad.cf" > "$output_file" 2>&1; then
+	cat "$output_file"
+	echo "FAIL: expected cfengine test to fail on a lint error"
+	exit 1
+fi
+cat "$output_file"
+grep -q "Lint failed" "$output_file"
+grep -q "Skipping build/deploy/run" "$output_file"

tests/shell/007-test-container.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -e
+set -x
+
+# Setup: create a temp directory for test files
+tmpdir=$(mktemp -d)
+trap "rm -rf $tmpdir" EXIT
+
+mkdir $tmpdir/test-module
+cd $tmpdir/test-module
+
+cfengine init --policy-module --with-input --non-interactive
+cfengine test
+
+