From 0dc72ce448c05adbb2db1a6f924dbbf72d11b319 Mon Sep 17 00:00:00 2001 From: Beinan Date: Fri, 20 Nov 2026 14:45:25 +0000 Subject: [PATCH] feat(ingest): observe narrow compaction remap and metadata preparation --- .../src/annotation_publication/compaction.rs | 38 +++++++++++-------- .../src/materializer/compaction.rs | 4 +- .../src/materializer/observation.rs | 9 ++++- docs/design/annotation-application.md | 9 +++++ 4 files changed, 42 insertions(+), 18 deletions(-) diff --git a/crates/lance-context-ingest/src/annotation_publication/compaction.rs b/crates/lance-context-ingest/src/annotation_publication/compaction.rs index 044a2a7d..ffc44517 100644 --- a/crates/lance-context-ingest/src/annotation_publication/compaction.rs +++ b/crates/lance-context-ingest/src/annotation_publication/compaction.rs @@ -1,5 +1,6 @@ //! Large narrow rewrites use the existing native preparation/reference encoding. use super::*; +use crate::materializer::observation::{Counters, Phase}; use lance::dataset::optimize::RewriteResult; use lance_context_core::annotation_staging::RESULTS_PROPERTY; @@ -9,23 +10,30 @@ pub(crate) async fn descriptors( selection: &Selection, storage: &Storage, maximum: u64, + phases: &Arc, ) -> Result { - let mut transaction = Transaction::new_from_version( - base.version_id(), - lance_context_core::native_compaction::prepare_rewrite( - base, - result, - metadata::MAX_NATIVE_BYTES, + let rewrite = phases + .measure( + Phase::AnnotationCompactionRemap, + lance_context_core::native_compaction::prepare_rewrite( + base, + result, + metadata::MAX_NATIVE_BYTES, + ), ) - .await?, - ); - transaction.uuid = selection.pending.plan.transaction.to_string(); - transaction.transaction_properties = base - .read_transaction() - .await? - .and_then(|t| t.transaction_properties); - validate(base, &transaction, selection).await?; - persist_transaction(base, &transaction, selection, storage, maximum).await + .await?; + phases + .measure(Phase::AnnotationCompactionMetadata, async { + let mut transaction = Transaction::new_from_version(base.version_id(), rewrite); + transaction.uuid = selection.pending.plan.transaction.to_string(); + transaction.transaction_properties = base + .read_transaction() + .await? + .and_then(|t| t.transaction_properties); + validate(base, &transaction, selection).await?; + persist_transaction(base, &transaction, selection, storage, maximum).await + }) + .await } pub(super) async fn validate( diff --git a/crates/lance-context-ingest/src/materializer/compaction.rs b/crates/lance-context-ingest/src/materializer/compaction.rs index 12dbc4de..741d718a 100644 --- a/crates/lance-context-ingest/src/materializer/compaction.rs +++ b/crates/lance-context-ingest/src/materializer/compaction.rs @@ -281,7 +281,7 @@ pub(super) async fn prepare( if result.row_addrs.is_some() { return if annotations { crate::annotation_publication::compaction::descriptors( - &dataset, result, &selection, &storage, maximum, + &dataset, result, &selection, &storage, maximum, &phases, ) .await } else { @@ -291,7 +291,7 @@ pub(super) async fn prepare( let bytes = serde_json::to_vec(&result)?; if annotations && bytes.len() as u64 > maximum { crate::annotation_publication::compaction::descriptors( - &dataset, result, &selection, &storage, maximum, + &dataset, result, &selection, &storage, maximum, &phases, ) .await } else { diff --git a/crates/lance-context-ingest/src/materializer/observation.rs b/crates/lance-context-ingest/src/materializer/observation.rs index a85b15ac..5884630a 100644 --- a/crates/lance-context-ingest/src/materializer/observation.rs +++ b/crates/lance-context-ingest/src/materializer/observation.rs @@ -95,8 +95,12 @@ pub enum Phase { AnnotationMaintenancePrepare, /// Resolving, installing and finishing the prepared maintenance publication. AnnotationMaintenancePublish, + /// Validating a narrow rewrite and preparing its native index remaps. + AnnotationCompactionRemap, + /// Validating and persisting narrow rewrite metadata after index remapping. + AnnotationCompactionMetadata, } -const NAMES: [&str; 51] = [ +const NAMES: [&str; 53] = [ "claim", "acquire", "state", @@ -148,6 +152,8 @@ const NAMES: [&str; 51] = [ "annotation_maintenance_select", "annotation_maintenance_prepare", "annotation_maintenance_publish", + "annotation_compaction_remap", + "annotation_compaction_metadata", ]; pub const PHASE_COUNT: usize = NAMES.len(); @@ -175,6 +181,7 @@ impl Default for Counters { pub struct Observation { pub phase: &'static str, pub started: u64, + /// Finished calls, including the failed subset below; excludes cancellation. pub completed: u64, pub failed: u64, pub cancelled: u64, diff --git a/docs/design/annotation-application.md b/docs/design/annotation-application.md index c522a0a0..5e002db2 100644 --- a/docs/design/annotation-application.md +++ b/docs/design/annotation-application.md @@ -324,6 +324,15 @@ catalog publication. They distinguish actual native work from the parent `compact` scheduling checks. Each child records one admitted native call, not one fragment, cloud request or successfully published version. +After a narrow native rewrite, `annotation_compaction_remap` measures rewrite +validation and native index remapping. `annotation_compaction_metadata` then +measures transaction assembly, validation and descriptor persistence. These +steps are outside `native_compaction`; an already completed file rewrite can +still be waiting on index reads or metadata writes before publication. These +phases cover narrow native descriptor preparation, not body-table maintenance. +For all phase counters, `completed` includes `failed`; cancellations are counted +separately. Fields are sampled independently and are not an atomic progress view. + Annotation commit resolution reports four nested phases. `annotation_commit_base` loads the fixed predecessor; `annotation_commit_publish` validates and publishes the prepared target when it