[build] add code coverage to provider_integration workflow

.github/workflows/rust.yml

@@ -84,19 +84,21 @@ jobs:
       env:
         OPENSSL_DIR: ${{ github.workspace }}/${{ env.OPENSSL_INSTALL }}
         LD_LIBRARY_PATH: ${{ github.workspace }}/${{ env.OPENSSL_INSTALL }}/lib
-      run: cargo xtask precheck --coverage
+      run: cargo xtask precheck --coverage --skip-integration
       continue-on-error: true
-    - name: Upload coverage report
-      uses: actions/upload-artifact@v7
-      with:
-        name: coverage-report-ubuntu
-        path: ./target/reports/
     - name: Generate nextest report
       id: nextest-report
       run: cargo xtask precheck --nextest-report
     - name: Code Coverage Summary Report
+      id: coverage-report
       if: ${{ steps.test-mock-coverage.outcome == 'success' }}
       run: cargo xtask precheck --coverage-report
+    - name: Upload coverage report
+      if: ${{ steps.coverage-report.outcome == 'success' }}
+      uses: actions/upload-artifact@v7
+      with:
+        name: coverage-report-ubuntu
+        path: ./target/reports/
     - name: Fail if any tests failed
       if: ${{ steps.test-mock-coverage.outcome == 'failure' || steps.test-mock-table-4.outcome == 'failure' || steps.test-mock-table-64.outcome == 'failure' }}
       run: |
@@ -145,6 +147,8 @@ jobs:
     - name: Build azihsm with OpenSSL ${{ env.OPENSSL_VERSION }}
       env:
         OPENSSL_DIR: ${{ github.workspace }}/${{ env.OPENSSL_INSTALL }}
+        RUSTFLAGS: -C instrument-coverage
+        LLVM_PROFILE_FILE: ${{ github.workspace }}/target/llvm-cov-target/azihsm-sdk-%p-%14m.profraw
       run: |
         cargo build -p azihsm_api_native --features mock
         cargo build -p azihsm_ossl_provider --features mock
@@ -177,21 +181,21 @@ jobs:
           -algorithm EC -pkeyopt group:P-384 \
           -outform DER -out /dev/null -text
 
-    - name: Run CLI integration tests
+    - name: Run CLI integration tests with code coverage
       env:
         OPENSSL_BIN: ${{ github.workspace }}/${{ env.OPENSSL_INSTALL }}/bin/openssl
         OPENSSL_LIB: ${{ github.workspace }}/${{ env.OPENSSL_INSTALL }}/lib:${{ github.workspace }}/target/debug
         PROPQUERY: "?provider=azihsm"
         AZIHSM_CREDENTIALS_ID: "70fcf730b8764238b8358010ce8a3f76"
         AZIHSM_CREDENTIALS_PIN: "db3dc77fc22e430080d41b31b6f04800"
-      run: cargo xtask precheck --nextest --package provider-integration-tests-cli --features integration --profile ci-provider-integration
+      run: cargo xtask precheck --coverage --package provider-integration-tests-cli --features integration --profile ci-provider-integration
 
-    - name: Run C API integration tests
+    - name: Run C API integration tests with code coverage
       env:
         OPENSSL_DIR: ${{ github.workspace }}/${{ env.OPENSSL_INSTALL }}
         AZIHSM_CREDENTIALS_ID: "70fcf730b8764238b8358010ce8a3f76"
         AZIHSM_CREDENTIALS_PIN: "db3dc77fc22e430080d41b31b6f04800"
-      run: cargo xtask precheck --nextest --package provider-integration-tests-capi --features integration --profile ci-provider-integration
+      run: cargo xtask precheck --coverage --package provider-integration-tests-capi --features integration --profile ci-provider-integration
 
     - name: Install libazihsm_api_native for NGINX tests
       run: |
@@ -208,18 +212,27 @@ jobs:
         sudo apt-get update
         sudo apt-get install -y --no-install-recommends nginx
 
-    - name: NGINX integration tests
+    - name: NGINX integration tests with code coverage
       env:
         OPENSSL_BIN: ${{ github.workspace }}/${{ env.OPENSSL_INSTALL }}/bin/openssl
         OPENSSL_LIB: ${{ github.workspace }}/${{ env.OPENSSL_INSTALL }}/lib
         AZIHSM_CREDENTIALS_ID: "70fcf730b8764238b8358010ce8a3f76"
         AZIHSM_CREDENTIALS_PIN: "db3dc77fc22e430080d41b31b6f04800"
       run: |
-        cargo xtask precheck --nextest \
+        cargo xtask precheck --coverage \
           --package provider-integration-tests-nginx \
           --features integration \
           --profile ci-provider-integration
 
+    - name: Code Coverage Summary Report
+      run: cargo xtask precheck --coverage-report --no-default-native --additional-obj-paths ${{ github.workspace }}/target/debug/libazihsm_api_native.so --additional-obj-paths /usr/lib/libazihsm_api_native.so
+
+    - name: Upload coverage report
+      uses: actions/upload-artifact@v7
+      with:
+        name: coverage-report-provider-integration
+        path: ./target/reports/
+
   build_windows:
 
     runs-on: windows-latest
@@ -257,19 +270,21 @@ jobs:
       run: cargo xtask precheck --clippy
     - name: Run all mock tests with code coverage
       id: test-mock-coverage
-      run: cargo xtask precheck --coverage
+      run: cargo xtask precheck --coverage --skip-integration
       continue-on-error: true
-    - name: Upload coverage report
-      uses: actions/upload-artifact@v7
-      with:
-        name: coverage-report-windows
-        path: ./target/reports/
     - name: Generate nextest report
       id: nextest-report
       run: cargo xtask precheck --nextest-report
     - name: Code Coverage Summary Report
+      id: coverage-report
       if: ${{ steps.test-mock-coverage.outcome == 'success' }}
       run: cargo xtask precheck --coverage-report
+    - name: Upload coverage report
+      if: ${{ steps.coverage-report.outcome == 'success' }}
+      uses: actions/upload-artifact@v7
+      with:
+        name: coverage-report-windows
+        path: ./target/reports/
     - name: Fail if any tests failed
       if: ${{ steps.test-mock-coverage.outcome == 'failure' }}
       run: |

xtask/README.md

@@ -59,13 +59,23 @@ cargo xtask copyright --fix
 
 ### coverage
 
-Build and run all tests with code coverage enabled. Generates a cobertura XML, JSON, and HTML report in [reporoot]/target/reports.
+Build and run all tests with code coverage enabled.
 
 ```bash
-# Build/run tests with code coverage and generate reports
+# Build/run tests with code coverage
 cargo xtask coverage
 ```
 
+### coverage-report
+
+Using coverage data created in coverage xtask, generates a cobertura XML, JSON, and HTML report in [reporoot]/target/reports. 
+Also outputs a markdown line coverage summary to stdout if run locally or GITHUB_STEP_SUMMARY if run in GitHub Actions.
+
+```bash
+# Generate code coverage reports
+cargo xtask coverage-report
+```
+
 ## Command Details
 
 - **precheck**: Combines setup, copyright, audit, fmt, clippy, and nextest stages for comprehensive validation

xtask/src/coverage.rs

@@ -15,123 +15,108 @@ use crate::XtaskCtx;
 /// Xtask to run code coverage
 #[derive(Parser)]
 #[clap(about = "Run code coverage using cargo llvm-cov")]
-pub struct Coverage {}
+pub struct Coverage {
+    /// Features to include in nextest run
+    #[clap(long)]
+    pub features: Option<String>,
+
+    /// Package argument to run nextest command with
+    #[clap(long)]
+    pub package: Option<String>,
+
+    /// Whether to include --no-default-features
+    #[clap(long)]
+    pub no_default_features: bool,
+
+    /// Test filterset (see https://nexte.st/docs/filtersets)
+    #[clap(long, short = 'E')]
+    pub filterset: Option<String>,
+
+    /// The nextest profile to use
+    #[clap(long)]
+    pub profile: Option<String>,
+
+    /// Crates to exclude from nextest (e.g. crates with heavyweight build scripts)
+    #[clap(long)]
+    pub exclude: Vec<String>,
+}
 
 impl Xtask for Coverage {
-    fn run(self, ctx: XtaskCtx) -> anyhow::Result<()> {
+    fn run(self, _ctx: XtaskCtx) -> anyhow::Result<()> {
         log::trace!("running code coverage");
 
         let sh = xshell::Shell::new()?;
 
         // Check cargo-llvm-cov version
         cmd!(sh, "cargo llvm-cov --version").quiet().run()?;
 
-        // Run tests with coverage
-        log::info!("Building all tests and running them with coverage");
-        cmd!(
-            sh,
-            "cargo llvm-cov nextest --no-report --no-fail-fast --features mock --profile ci-mock --workspace --exclude provider-integration-tests-cli --exclude provider-integration-tests-capi"
-        )
-        .run()?;
-
-        // Run resiliency fault-injection tests with coverage
-        log::info!("Building resiliency fault-injection tests and running them with coverage");
-        cmd!(
-            sh,
-            "cargo llvm-cov nextest --no-report --no-fail-fast -E test(resiliency::fault_injection::) --features mock,res-test --package azihsm_api_tests --profile ci-mock-res"
-        )
-        .run()?;
-
-        // Check for/create reports directory
-        let reports_dir = ctx.root.join("target").join("reports");
-        if !reports_dir.exists() {
-            log::info!("Creating reports directory at {}", reports_dir.display());
-            std::fs::create_dir_all(&reports_dir)?;
+        // convert xtask parameters into cargo command arguments
+        let mut command_args = Vec::new();
+        let mut features_vec = Vec::new();
+        if self.features.is_some() {
+            features_vec.push(self.features.unwrap_or_default());
         }
-
-        // Find path to azihsm_api_native object file
-        let build_dir = ctx
-            .root
-            .join("target")
-            .join("llvm-cov-target")
-            .join("debug")
-            .join("build");
-        let mut native_obj_path = None;
-        if build_dir.exists() {
-            for entry in std::fs::read_dir(&build_dir)? {
-                let entry = entry?;
-                let path = entry.path();
-                if path.is_dir()
-                    && path
-                        .file_name()
-                        .and_then(|s| s.to_str())
-                        .map(|s| s.starts_with("azihsm_api_tests-"))
-                        .unwrap_or(false)
-                {
-                    // check if directory contains 'out' subdirectory to see if it's the cmake build directory
-                    if path.join("out").is_dir() {
-                        log::info!("Found cmake build directory at: {}", path.display());
-                        #[cfg(target_os = "windows")]
-                        {
-                            native_obj_path =
-                                Some(path.join("out").join("build").join("azihsm_api_native.dll"));
-                        }
-                        #[cfg(not(target_os = "windows"))]
-                        {
-                            native_obj_path = Some(
-                                path.join("out")
-                                    .join("build")
-                                    .join("libazihsm_api_native.so"),
-                            );
-                        }
-                        break;
-                    }
-                }
-            }
-        } else {
-            log::warn!(
-                "Cargo build-script directory not found at expected path: {}. Coverage reports may be incomplete.",
-                build_dir.display()
-            );
+        let features_val;
+        if !features_vec.is_empty() {
+            command_args.push("--features");
+            features_val = features_vec.join(",");
+            command_args.push(&features_val);
         }
-
-        // set LLVM_COV_FLAGS to include azihsm_api_native object file in coverage reports
-        if let Some(native_obj_path) = native_obj_path {
-            if native_obj_path.is_file() {
-                let path_str = native_obj_path.to_string_lossy();
-                let new_flags = match std::env::var("LLVM_COV_FLAGS") {
-                    Ok(existing) if !existing.trim().is_empty() => {
-                        format!("{existing} -object {path_str}")
-                    }
-                    _ => format!("-object {path_str}"),
-                };
-                sh.set_var("LLVM_COV_FLAGS", new_flags);
-            } else {
-                log::warn!("Could not find azihsm_api_native object at expected path: {}. Coverage reports may be incomplete.", native_obj_path.display());
+        let package_val = self.package.clone().unwrap_or_default();
+        if self.package.is_some() {
+            command_args.push("--package");
+            command_args.push(&package_val);
+        }
+        if self.no_default_features {
+            command_args.push("--no-default-features");
+        }
+        let filterset_val = self.filterset.clone().unwrap_or_default();
+        if self.filterset.is_some() {
+            command_args.push("--filterset");
+            command_args.push(&filterset_val);
+        }
+        let profile_val = self.profile.clone().unwrap_or_default();
+        if self.profile.is_some() {
+            command_args.push("--profile");
+            command_args.push(&profile_val);
+        }
+        let exclude_vals: Vec<String>;
+        if self.package.is_none() {
+            command_args.push("--workspace");
+            if !self.exclude.is_empty() {
+                exclude_vals = self
+                    .exclude
+                    .iter()
+                    .flat_map(|c| ["--exclude".to_string(), c.clone()])
+                    .collect();
+                for val in &exclude_vals {
+                    command_args.push(val);
+                }
             }
-        } else {
-            log::warn!("Could not find cmake build directory or azihsm_api_native object. Coverage reports may be incomplete.");
         }
 
-        // Generate cobertura report
-        log::info!("Generating cobertura report");
-        cmd!(
-            sh,
-            "cargo llvm-cov report --cobertura --output-path ./target/reports/cobertura_sdk.xml --ignore-filename-regex xtask*"
-        ).run()?;
-
-        // Generate json report
-        log::info!("Generating json report");
+        // Run tests with coverage
+        log::info!("Building all tests and running them with coverage");
         cmd!(
             sh,
-            "cargo llvm-cov report --json --summary-only --output-path ./target/reports/sdk-cov.json --ignore-filename-regex xtask*"
-        ).run()?;
-
-        // Generate HTML report
-        log::info!("Generating HTML report");
-        cmd!(sh, " cargo llvm-cov report --html --output-dir ./target/reports/sdk-cov/ --ignore-filename-regex xtask*").run()?;
+            "cargo llvm-cov nextest --no-report --no-fail-fast {command_args...}"
+        )
+        .run()?;
 
         log::info!("Code coverage completed successfully");
         Ok(())
     }
 }
+
+impl From<crate::nextest::Nextest> for Coverage {
+    fn from(nextest: crate::nextest::Nextest) -> Self {
+        Self {
+            features: nextest.features,
+            package: nextest.package,
+            no_default_features: nextest.no_default_features,
+            filterset: nextest.filterset,
+            profile: nextest.profile,
+            exclude: nextest.exclude,
+        }
+    }
+}