🐛 fix(ci): make mutation failures decisive

.config/nextest.toml

@@ -9,6 +9,10 @@ status-level = "slow"
 [profile.ci.junit]
 path = "junit.xml"
 
+[profile.mutation]
+inherits = "ci"
+fail-fast = true
+
 [test-groups]
 containers = { max-threads = 1 }
 e2e-servers = { max-threads = 4 }

crates/peryx-driver/src/jobs/metrics.rs

@@ -46,7 +46,7 @@ impl JobMetrics {
 
     pub(crate) fn started(&self, kind: &'static str) {
         self.with(kind, |counters| {
-            counters.started += 1;
+            counters.started = counters.started.saturating_add(1);
             counters.running += 1;
         });
     }

crates/peryx-driver/src/serving.rs

@@ -70,8 +70,6 @@ pub trait EcosystemRegistration: Send + Sync {
 }
 
 pub trait EcosystemAuth: Send + Sync {
-    fn fields(&self) -> &'static [&'static str];
-    fn defaults(&self) -> toml::Table;
     /// # Errors
     /// Returns the ecosystem's configuration error.
     fn validate(&self, config: PluginAuthConfig<'_>) -> Result<(), String>;

crates/peryx-driver/tests/unit/state/build/tests.rs

@@ -20,10 +20,18 @@ fn test_rate_limit_constructor_keeps_default_runtime_controls() {
     let state = AppState::with_rate_limits(meta, blobs, 60, Vec::new(), RateLimitConfig::default(), []);
 
     assert_eq!(state.serving.max_stale_secs, DEFAULT_MAX_STALE_SECS);
-    assert_eq!(
-        state.serving.cache.hot.policy().max_capacity(),
-        Some(DEFAULT_HOT_CACHE_BYTES)
-    );
+    assert_eq!(DEFAULT_HOT_CACHE_BYTES, 268_435_456);
+    assert_eq!(state.serving.cache.hot.policy().max_capacity(), Some(268_435_456));
+}
+
+#[test]
+fn test_default_clock_stamps_operations_with_unix_time() {
+    let (_dir, meta, blobs) = stores();
+    let state = AppState::new(meta.clone(), blobs, 60, Vec::new());
+
+    state.serving.claim_admitted_write("clock");
+
+    assert!(meta.operation_outcome("clock").unwrap().unwrap().updated_at_unix > 1_700_000_000);
 }
 
 #[test]

crates/peryx-driver/tests/unit/state/describe/tests.rs

@@ -6,6 +6,7 @@ use peryx_identity::{Action, Glob, Grant, IndexAcl, NamedToken};
 use peryx_index::{Index, IndexKind};
 use peryx_policy::Policy;
 use peryx_upstream::{NamedUpstream, UpstreamClient, UpstreamRouter};
+use rstest::rstest;
 
 fn writer_acl(secret: impl Into<String>) -> IndexAcl {
     IndexAcl {
@@ -87,13 +88,38 @@ fn test_describe_indexes_preserves_input_order() {
     );
 }
 
-#[test]
-fn test_hosted_index_reports_volatile_deletes_when_writable_and_volatile() {
-    let indexes = vec![index("store", IndexKind::Hosted { volatile: true }, writer_acl("s"))];
-    let described = describe_index(&indexes, 0);
-    assert_eq!(described.kind, "hosted");
-    assert!(described.volatile_deletes);
-    assert!(described.precedence.is_empty());
+#[rstest]
+#[case::hosted_read_only_stable(false, false, false, false)]
+#[case::hosted_read_only_volatile(false, false, true, false)]
+#[case::hosted_writable_stable(false, true, false, false)]
+#[case::hosted_writable_volatile(false, true, true, true)]
+#[case::virtual_read_only_stable(true, false, false, false)]
+#[case::virtual_read_only_volatile(true, false, true, false)]
+#[case::virtual_writable_stable(true, true, false, false)]
+#[case::virtual_writable_volatile(true, true, true, true)]
+fn test_volatile_deletes_require_a_writable_volatile_target(
+    #[case] virtual_index: bool,
+    #[case] writable: bool,
+    #[case] volatile: bool,
+    #[case] expected: bool,
+) {
+    let acl = if writable { writer_acl("s") } else { IndexAcl::default() };
+    let mut indexes = vec![index("store", IndexKind::Hosted { volatile }, acl)];
+    let position = if virtual_index {
+        indexes.push(index(
+            "virtual",
+            IndexKind::Virtual {
+                layers: vec![0],
+                write_target: Some(0),
+            },
+            IndexAcl::default(),
+        ));
+        1
+    } else {
+        0
+    };
+
+    assert_eq!(describe_index(&indexes, position).volatile_deletes, expected);
 }
 
 #[test]
@@ -133,7 +159,6 @@ fn test_virtual_upload_target_drives_uploads_and_volatile_deletes() {
     ];
     let described = describe_index(&indexes, 1);
     assert!(described.uploads);
-    assert!(described.volatile_deletes);
     assert_eq!(described.upload_to.as_deref(), Some("store"));
     assert_eq!(described.precedence, vec![member("store", "hosted")]);
 }

crates/peryx-driver/tests/unit/state/operation/tests.rs

@@ -22,14 +22,14 @@ fn test_claim_admitted_write_records_a_pending_write_with_a_retention_deadline()
     state.claim_admitted_write("op");
     let stored = meta.operation_outcome("op").unwrap().unwrap();
     assert_eq!(stored.state, OperationState::Pending);
-    assert_eq!(stored.expiry_unix, Some(NOW + OPERATION_RETENTION_SECS));
+    assert_eq!(stored.expiry_unix, Some(NOW + 86_400));
 }
 
 #[test]
 fn test_claim_admitted_write_that_never_finalizes_expires() {
     let (_dir, state, meta) = state();
     state.claim_admitted_write("op");
-    let health = meta.operation_outcome_health(NOW + OPERATION_RETENTION_SECS).unwrap();
+    let health = meta.operation_outcome_health(NOW + 86_400).unwrap();
     assert_eq!(
         health.expired, 1,
         "an unfinalized write reads expired past its deadline"

crates/peryx-driver/tests/unit/state/registry/tests.rs

@@ -1,5 +1,6 @@
 use std::collections::HashMap;
-use std::sync::Arc;
+use std::sync::{Arc, Mutex};
+use std::time::Duration;
 
 use async_trait::async_trait;
 use axum::Router;
@@ -11,7 +12,9 @@ use peryx_core::{Ecosystem, Lexicon};
 use peryx_identity::IndexAcl;
 use peryx_index::{Index, IndexKind};
 use peryx_policy::Policy;
-use peryx_search::EmptyIndexer;
+use peryx_search::{
+    ContentSource, EmptyIndexer, IndexerCtx, SearchDocument, SearchDocumentProvider, SearchError, SearchParams,
+};
 use peryx_storage::blob::{BlobDurability, BlobStore};
 use peryx_storage::meta::MetaStore;
 
@@ -35,6 +38,25 @@ struct ReplacementDriver;
 
 struct Drainer;
 
+struct MutableDocs(Arc<Mutex<String>>);
+
+impl SearchDocumentProvider for MutableDocs {
+    fn documents(&self, _ctx: &IndexerCtx<'_>) -> Result<Vec<SearchDocument>, SearchError> {
+        let text = self.0.lock().unwrap().clone();
+        Ok(vec![SearchDocument {
+            display_label: "package".to_owned(),
+            resource_key: "package".to_owned(),
+            route: "root".to_owned(),
+            index: "root".to_owned(),
+            ecosystem: "indexed".to_owned(),
+            source: ContentSource::Cached,
+            available_locally: false,
+            summary: None,
+            text,
+        }])
+    }
+}
+
 #[async_trait]
 impl peryx_ha::AuthorityDrainer for Drainer {
     async fn drain(
@@ -940,3 +962,49 @@ fn test_registration_defaults_and_read_only_mutation_are_observable() {
     );
     assert!(shared.read_only);
 }
+
+#[test]
+fn test_read_only_retry_interval_is_observable() {
+    let (_dir, mut state) = state();
+
+    state.set_read_only_retry_after(Some(Duration::from_secs(17))).unwrap();
+
+    assert_eq!(state.serving.read_only_retry_after(), Some(Duration::from_secs(17)));
+}
+
+#[test]
+fn test_search_epoch_refreshes_the_published_index() {
+    let (_dir, mut state) = state();
+    let text = Arc::new(Mutex::new("old".to_owned()));
+    state.register_lexicon(Ecosystem::new("indexed"), &Lexicon::NEUTRAL);
+    state
+        .register_protocol(
+            ProtocolDriver::Indexed(Arc::new(IndexedDriver)),
+            Arc::new(MutableDocs(text.clone())),
+        )
+        .unwrap();
+    let state = Arc::new(state);
+    let services = crate::http_services::HttpDomainServices::for_state(&state);
+    assert_eq!(
+        services.search().search(SearchParams::default(), None).unwrap().total,
+        1
+    );
+    *text.lock().unwrap() = "new".to_owned();
+
+    state.serving.bump_search_epoch();
+
+    assert_eq!(
+        services
+            .search()
+            .search(
+                SearchParams {
+                    query: "new".to_owned(),
+                    ..SearchParams::default()
+                },
+                None,
+            )
+            .unwrap()
+            .total,
+        1
+    );
+}

crates/peryx-ecosystem-oci/src/lib.rs

@@ -174,14 +174,6 @@ impl EcosystemConfig for OciPlugin {
 }
 
 impl EcosystemAuth for OciPlugin {
-    fn fields(&self) -> &'static [&'static str] {
-        &[]
-    }
-
-    fn defaults(&self) -> toml::Table {
-        toml::Table::new()
-    }
-
     fn validate(&self, config: PluginAuthConfig<'_>) -> Result<(), String> {
         if config.signing_key_configured
             && config.token_ttl_secs < 60
@@ -282,7 +274,7 @@ pub fn registration() -> peryx_plugin_registry::PluginRegistration {
         rate_limit_principal: Some(&OciPlugin),
         client_discovery: Some(&OciPlugin),
         openapi: &OciPlugin,
-        auth: Some(&OciPlugin),
+        auth: Some(peryx_plugin_registry::PluginAuthRegistration::Shared(&OciPlugin)),
         browse: Some(&OciPlugin),
         snippets: None,
         metadata_migration: None,

crates/peryx-ecosystem-oci/tests/unit/tests/plugin_contract_tests.rs

@@ -11,6 +11,7 @@ use rstest::rstest;
 use utoipa::openapi::PathsBuilder;
 
 use crate::{ECOSYSTEM, OciPlugin, registration};
+use peryx_plugin_registry::PluginAuthRegistration;
 
 fn state() -> (tempfile::TempDir, AppState) {
     let dir = tempfile::tempdir().unwrap();
@@ -32,6 +33,7 @@ fn plugin_exposes_its_contract() {
     let plugin = OciPlugin;
 
     assert_eq!(plugin.ecosystem(), ECOSYSTEM);
+    assert_eq!(plugin.absolute_prefixes(), &["/v2/"]);
     let protocol = plugin.driver();
     let driver = protocol.absolute().unwrap();
     assert_eq!(driver.prefixes(), &["/v2/"]);
@@ -75,10 +77,7 @@ fn plugin_exposes_its_contract() {
 
 #[test]
 fn plugin_auth_uses_only_shared_settings() {
-    assert_eq!(
-        (OciPlugin.fields(), OciPlugin.defaults()),
-        (&[] as &[&str], toml::Table::new())
-    );
+    assert!(matches!(registration().auth, Some(PluginAuthRegistration::Shared(_))));
 }
 
 #[test]

crates/peryx-ecosystem-pypi/src/lib.rs

@@ -206,14 +206,6 @@ impl EcosystemRegistration for PypiPlugin {
 
 #[cfg(feature = "serving")]
 impl EcosystemAuth for PypiPlugin {
-    fn fields(&self) -> &'static [&'static str] {
-        trusted_publishing::AUTH_FIELDS
-    }
-
-    fn defaults(&self) -> toml::Table {
-        trusted_publishing::auth_defaults()
-    }
-
     fn validate(&self, config: peryx_driver::serving::PluginAuthConfig<'_>) -> Result<(), String> {
         trusted_publishing::validate(config)
     }
@@ -392,7 +384,11 @@ pub fn registration() -> peryx_plugin_registry::PluginRegistration {
         rate_limit_principal: Some(&PypiPlugin),
         client_discovery: Some(&PypiPlugin),
         openapi: &PypiPlugin,
-        auth: Some(&PypiPlugin),
+        auth: Some(peryx_plugin_registry::PluginAuthRegistration::Extension {
+            auth: &PypiPlugin,
+            fields: trusted_publishing::AUTH_FIELDS,
+            defaults: trusted_publishing::auth_defaults,
+        }),
         browse: Some(&PypiPlugin),
         snippets: Some(&PypiPlugin),
         metadata_migration: Some(Arc::new(PypiPlugin)),

crates/peryx-ecosystem-pypi/tests/unit/plugin_contract_tests.rs

@@ -15,6 +15,7 @@ use peryx_driver::serving::{
 use peryx_identity::IndexAcl;
 use peryx_index::{Index, IndexKind};
 use peryx_plugin_registry::OperatorJobOptions;
+use peryx_plugin_registry::PluginAuthRegistration;
 use peryx_policy::{
     Policy, RetentionClass, RetentionConfig, RetentionDecision, RetentionOutcome, RetentionPolicy, RetentionSelector,
     RetentionVisibility,
@@ -89,13 +90,10 @@ fn plugin_exposes_identity_defaults_and_driver() {
 
 #[test]
 fn plugin_exposes_trusted_publishing_auth_configuration() {
-    let plugin = PypiPlugin;
-
-    assert_eq!(plugin.fields(), &["oidc_audience", "trusted_publisher"]);
-    assert_eq!(
-        plugin.defaults(),
-        toml::Table::from_iter([("oidc_audience".to_owned(), toml::Value::String("peryx".to_owned()),)])
-    );
+    assert!(matches!(
+        registration().auth,
+        Some(PluginAuthRegistration::Extension { .. })
+    ));
 }
 
 #[test]