From d90e56ebeb3498738d1c93aab2500bd0dfdfc007 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Mon, 7 Sep 2026 21:51:44 -0700 Subject: [PATCH 01/11] Core: Add v4 write-direction TrackedFile adapters Add reusable write-direction wrappers that present a legacy DataFile, DeleteFile, or ManifestFile as a v4 TrackedFile row for manifest serialization. Content stats are served by a reusable map-backed view over the file's stat maps. Generated-by: Cursor Grok 4.6 Co-authored-by: Cursor --- .../java/org/apache/iceberg/ManifestFile.java | 24 + .../apache/iceberg/GenericManifestFile.java | 64 +- .../apache/iceberg/MapBackedContentStats.java | 254 +++++++ .../org/apache/iceberg/TableMetadata.java | 1 + .../apache/iceberg/TrackedFileAdapters.java | 616 +++++++++++++++- .../org/apache/iceberg/TrackedFileStruct.java | 40 +- .../iceberg/TestMapBackedContentStats.java | 348 +++++++++ .../iceberg/TestTrackedFileAdapters.java | 662 +++++++++++++++++- 8 files changed, 1960 insertions(+), 49 deletions(-) create mode 100644 core/src/main/java/org/apache/iceberg/MapBackedContentStats.java create mode 100644 core/src/test/java/org/apache/iceberg/TestMapBackedContentStats.java diff --git a/api/src/main/java/org/apache/iceberg/ManifestFile.java b/api/src/main/java/org/apache/iceberg/ManifestFile.java index cfedb78a7cee..4d26c6e4d90d 100644 --- a/api/src/main/java/org/apache/iceberg/ManifestFile.java +++ b/api/src/main/java/org/apache/iceberg/ManifestFile.java @@ -29,6 +29,9 @@ public interface ManifestFile { int PARTITION_SUMMARIES_ELEMENT_ID = 508; + /** Format version for pre-v4 manifest files. */ + int LEGACY_FORMAT_VERSION = 0; + Types.NestedField PATH = required(500, "manifest_path", Types.StringType.get(), "Location URI with FS scheme"); Types.NestedField LENGTH = @@ -186,6 +189,22 @@ default boolean hasDeletedFiles() { /** Returns the total number of rows in all files with status DELETED in the manifest file. */ Long deletedRowsCount(); + /** + * Returns the number of files with status REPLACED in the manifest file, or null for pre-v4 + * manifests. + */ + default Integer replacedFilesCount() { + return null; + } + + /** + * Returns the total number of rows in all files with status REPLACED in the manifest file, or + * null for pre-v4 manifests. + */ + default Long replacedRowsCount() { + return null; + } + /** * Returns a list of {@link PartitionFieldSummary partition field summaries}. * @@ -215,6 +234,11 @@ default ManifestBitmap manifestDeletionVector() { return null; } + /** Returns the number of entries in the manifest file, or null for pre-v4 manifests. */ + default Long recordCount() { + return null; + } + /** Returns the writer format version of the manifest, or 0 if it predates format tracking. */ default int formatVersion() { return 0; diff --git a/core/src/main/java/org/apache/iceberg/GenericManifestFile.java b/core/src/main/java/org/apache/iceberg/GenericManifestFile.java index 9624484ffe0c..f03578eb9851 100644 --- a/core/src/main/java/org/apache/iceberg/GenericManifestFile.java +++ b/core/src/main/java/org/apache/iceberg/GenericManifestFile.java @@ -60,6 +60,8 @@ public class GenericManifestFile extends SupportsIndexProjection private PartitionFieldSummary[] partitions = null; private byte[] keyMetadata = null; private Long firstRowId = null; + private Long recordCount = null; + private int formatVersion = LEGACY_FORMAT_VERSION; /** Used by Avro reflection to instantiate this class when reading manifest files. */ public GenericManifestFile(Schema avroSchema) { @@ -94,7 +96,7 @@ public GenericManifestFile(Schema avroSchema) { this.firstRowId = null; } - /** Adjust the arg order to avoid conflict with the public constructor below */ + /** Pre-v4 constructor. */ GenericManifestFile( String path, long length, @@ -130,6 +132,50 @@ public GenericManifestFile(Schema avroSchema) { this.partitions = partitions == null ? null : partitions.toArray(new PartitionFieldSummary[0]); this.keyMetadata = ByteBuffers.toByteArray(keyMetadata); this.firstRowId = firstRowId; + this.recordCount = null; + this.formatVersion = LEGACY_FORMAT_VERSION; + } + + /** v4+ constructor variant that accepts recordCount and formatVersion. */ + GenericManifestFile( + String path, + long length, + int specId, + ManifestContent content, + long sequenceNumber, + long minSequenceNumber, + Long snapshotId, + List partitions, + ByteBuffer keyMetadata, + Integer addedFilesCount, + Long addedRowsCount, + Integer existingFilesCount, + Long existingRowsCount, + Integer deletedFilesCount, + Long deletedRowsCount, + Long firstRowId, + Long recordCount, + int formatVersion) { + super(ManifestFile.schema().columns().size()); + this.avroSchema = AVRO_SCHEMA; + this.manifestPath = path; + this.length = length; + this.specId = specId; + this.content = content; + this.sequenceNumber = sequenceNumber; + this.minSequenceNumber = minSequenceNumber; + this.snapshotId = snapshotId; + this.addedFilesCount = addedFilesCount; + this.addedRowsCount = addedRowsCount; + this.existingFilesCount = existingFilesCount; + this.existingRowsCount = existingRowsCount; + this.deletedFilesCount = deletedFilesCount; + this.deletedRowsCount = deletedRowsCount; + this.partitions = partitions == null ? null : partitions.toArray(new PartitionFieldSummary[0]); + this.keyMetadata = ByteBuffers.toByteArray(keyMetadata); + this.firstRowId = firstRowId; + this.recordCount = recordCount; + this.formatVersion = formatVersion; } /** @@ -172,6 +218,8 @@ private GenericManifestFile(GenericManifestFile toCopy) { ? null : Arrays.copyOf(toCopy.keyMetadata, toCopy.keyMetadata.length); this.firstRowId = toCopy.firstRowId; + this.recordCount = toCopy.recordCount; + this.formatVersion = toCopy.formatVersion; } /** Constructor for Java serialization. */ @@ -272,6 +320,16 @@ public Long firstRowId() { return firstRowId; } + @Override + public Long recordCount() { + return recordCount; + } + + @Override + public int formatVersion() { + return formatVersion; + } + @Override public int size() { return ManifestFile.schema().columns().size(); @@ -468,7 +526,9 @@ private CopyBuilder(ManifestFile toCopy) { toCopy.existingRowsCount(), toCopy.deletedFilesCount(), toCopy.deletedRowsCount(), - toCopy.firstRowId()); + toCopy.firstRowId(), + toCopy.recordCount(), + toCopy.formatVersion()); } } diff --git a/core/src/main/java/org/apache/iceberg/MapBackedContentStats.java b/core/src/main/java/org/apache/iceberg/MapBackedContentStats.java new file mode 100644 index 000000000000..5bb55bb5cc75 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/MapBackedContentStats.java @@ -0,0 +1,254 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.iceberg.relocated.com.google.common.collect.Iterables; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.types.Conversions; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; + +/** Reusable {@link ContentStats} view over a legacy {@link ContentFile}'s stat maps. */ +class MapBackedContentStats implements ContentStats, StructLike { + private final Types.StructType struct; + private final int[] posToId; + private final Map> statsById; + + private Map valueCounts; + private Map nullValueCounts; + private Map nanValueCounts; + private Map lowerBounds; + private Map upperBounds; + + MapBackedContentStats(Schema tableSchema, MetricsConfig metricsConfig) { + this.struct = StatsUtil.statsWriteSchema(tableSchema, metricsConfig); + List fields = struct.fields(); + this.posToId = new int[fields.size()]; + this.statsById = Maps.newHashMapWithExpectedSize(fields.size()); + for (int i = 0; i < fields.size(); i += 1) { + Types.NestedField field = fields.get(i); + int fieldId = StatsUtil.toFieldId(field.fieldId()); + posToId[i] = fieldId; + statsById.put( + fieldId, + new MapBackedFieldStats<>( + field.type().asStructType(), fieldId, tableSchema.findType(fieldId))); + } + } + + MapBackedContentStats wrap(ContentFile file) { + this.valueCounts = file.valueCounts(); + this.nullValueCounts = file.nullValueCounts(); + this.nanValueCounts = file.nanValueCounts(); + this.lowerBounds = file.lowerBounds(); + this.upperBounds = file.upperBounds(); + return this; + } + + private boolean hasStats(int id) { + return containsId(valueCounts, id) + || containsId(nullValueCounts, id) + || containsId(nanValueCounts, id) + || containsId(lowerBounds, id) + || containsId(upperBounds, id); + } + + private static boolean containsId(Map map, int id) { + return map != null && map.containsKey(id); + } + + @Override + public Iterable> fieldStats() { + return Iterables.transform( + Iterables.filter(statsById.entrySet(), entry -> hasStats(entry.getKey())), + Map.Entry::getValue); + } + + @Override + @SuppressWarnings("unchecked") + public FieldStats statsFor(int fieldId) { + return hasStats(fieldId) ? (FieldStats) statsById.get(fieldId) : null; + } + + @Override + public Types.StructType type() { + return struct; + } + + @Override + public int size() { + return struct.fields().size(); + } + + @Override + public T get(int pos, Class javaClass) { + int id = posToId[pos]; + return javaClass.cast(hasStats(id) ? statsById.get(id) : null); + } + + @Override + public void set(int pos, T value) { + throw new UnsupportedOperationException( + "Reusable content stats wrapper does not support set()"); + } + + @Override + public ContentStats copy() { + throw new UnsupportedOperationException( + "Reusable content stats wrapper does not support copy(); materialize via a writer instead"); + } + + @Override + public ContentStats copy(Set fieldIds) { + throw new UnsupportedOperationException( + "Reusable content stats wrapper does not support copy(); materialize via a writer instead"); + } + + /** Reusable {@link FieldStats} view over one field's entries in a {@link ContentFile}'s maps. */ + private class MapBackedFieldStats implements FieldStats, StructLike { + private final Types.StructType struct; + private final int fieldId; + private final Type boundType; + private final int[] posToOffset; + + MapBackedFieldStats(Types.StructType struct, int fieldId, Type boundType) { + this.struct = struct; + this.fieldId = fieldId; + this.boundType = boundType; + this.posToOffset = posToOffset(struct); + } + + @Override + public int fieldId() { + return fieldId; + } + + @Override + public Types.StructType type() { + return struct; + } + + @Override + @SuppressWarnings("unchecked") + public T lowerBound() { + ByteBuffer buf = lowerBounds == null ? null : lowerBounds.get(fieldId); + return buf == null ? null : (T) Conversions.fromByteBuffer(boundType, buf); + } + + @Override + @SuppressWarnings("unchecked") + public T upperBound() { + ByteBuffer buf = upperBounds == null ? null : upperBounds.get(fieldId); + return buf == null ? null : (T) Conversions.fromByteBuffer(boundType, buf); + } + + @Override + public boolean tightBounds() { + return false; + } + + @Override + public boolean hasValueCount() { + return count(valueCounts) != null; + } + + @Override + public long valueCount() { + return count(valueCounts); + } + + @Override + public boolean hasNullValueCount() { + return count(nullValueCounts) != null; + } + + @Override + public long nullValueCount() { + return count(nullValueCounts); + } + + @Override + public boolean hasNanValueCount() { + return count(nanValueCounts) != null; + } + + @Override + public long nanValueCount() { + return count(nanValueCounts); + } + + @Override + public Integer avgValueSizeInBytes() { + return null; + } + + private Long count(Map counts) { + return counts == null ? null : counts.get(fieldId); + } + + @Override + public int size() { + return struct.fields().size(); + } + + @Override + public C get(int pos, Class javaClass) { + return javaClass.cast(getOffset(posToOffset[pos])); + } + + private Object getOffset(int offset) { + return switch (offset) { + case StatsUtil.LOWER_BOUND_OFFSET -> lowerBound(); + case StatsUtil.UPPER_BOUND_OFFSET -> upperBound(); + case StatsUtil.TIGHT_BOUNDS_OFFSET -> tightBounds(); + case StatsUtil.VALUE_COUNT_OFFSET -> count(valueCounts); + case StatsUtil.NULL_VALUE_COUNT_OFFSET -> count(nullValueCounts); + case StatsUtil.NAN_VALUE_COUNT_OFFSET -> count(nanValueCounts); + case StatsUtil.AVG_VALUE_SIZE_OFFSET -> null; + default -> throw new UnsupportedOperationException("Unsupported stats offset: " + offset); + }; + } + + @Override + public void set(int pos, C value) { + throw new UnsupportedOperationException( + "Reusable field stats wrapper does not support set()"); + } + + @Override + public FieldStats copy() { + throw new UnsupportedOperationException( + "Reusable field stats wrapper does not support copy(); materialize via a writer instead"); + } + + private static int[] posToOffset(Types.StructType struct) { + List fields = struct.fields(); + int[] offsets = new int[fields.size()]; + for (int i = 0; i < offsets.length; i += 1) { + offsets[i] = StatsUtil.statOffset(fields.get(i).fieldId()); + } + + return offsets; + } + } +} diff --git a/core/src/main/java/org/apache/iceberg/TableMetadata.java b/core/src/main/java/org/apache/iceberg/TableMetadata.java index a43d4145907a..cd48ac2e6fa7 100644 --- a/core/src/main/java/org/apache/iceberg/TableMetadata.java +++ b/core/src/main/java/org/apache/iceberg/TableMetadata.java @@ -59,6 +59,7 @@ public class TableMetadata implements Serializable { static final int MIN_FORMAT_VERSION_ROW_LINEAGE = 3; static final int MIN_FORMAT_VERSION_PARQUET_MANIFESTS = 4; static final int MIN_FORMAT_VERSION_OPTIONAL_LOCATION = 4; + static final int MIN_FORMAT_VERSION_ADAPTIVE_MANIFEST_TREE = 4; static final int INITIAL_SPEC_ID = 0; static final int INITIAL_SORT_ORDER_ID = 1; static final int INITIAL_SCHEMA_ID = 0; diff --git a/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java b/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java index 5acb03b9089c..6f3eca37581a 100644 --- a/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java +++ b/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java @@ -24,10 +24,20 @@ import java.util.Map; import java.util.Set; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.StructProjection; -/** Adapts {@link TrackedFile} entries to their read APIs, for example {@link DataFile}. */ +/** + * Adapts between the {@link TrackedFile} row and the {@link DataFile} / {@link DeleteFile} / {@link + * ManifestFile} APIs in both directions. + */ class TrackedFileAdapters { + private static final int TRACKED_FILE_FIELD_COUNT = + TrackedFile.schema(Types.StructType.of(), Types.StructType.of()).asStruct().fields().size(); + + private static final int MANIFEST_INFO_FIELD_COUNT = ManifestInfo.schema().fields().size(); + private TrackedFileAdapters() {} static DataFile asDataFile(TrackedFile file, Map specsById) { @@ -63,6 +73,49 @@ static ManifestFile asManifestFile(TrackedFile file) { return new TrackedManifestFile(file); } + /** + * Returns a reusable wrapper that presents a {@link DataFile} as a {@link TrackedFile} row. + * + * @param formatVersion the target table's format version (must be 4+) + * @param tableSchema table schema used to build {@link ContentStats} from the file's stats + * @param metricsConfig metrics config used to prune the content stats schema + * @param partitionType target partition struct type; use one spec's partition type for a + * single-spec manifest, or the union across live specs for a multi-spec manifest + */ + static DataTrackedFile forDataFile( + int formatVersion, + Schema tableSchema, + MetricsConfig metricsConfig, + Types.StructType partitionType) { + return new DataTrackedFile(formatVersion, tableSchema, metricsConfig, partitionType); + } + + /** + * Returns a reusable wrapper that presents an equality {@link DeleteFile} as a {@link + * TrackedFile} row. + * + * @param formatVersion the target table's format version (must be 4+) + * @param tableSchema table schema used to build {@link ContentStats} from the file's stats + * @param metricsConfig metrics config used to prune the content stats schema + * @param partitionType target partition struct type; use one spec's partition type for a + * single-spec manifest, or the union across live specs for a multi-spec manifest + */ + static EqualityDeleteTrackedFile forEqualityDeleteFile( + int formatVersion, + Schema tableSchema, + MetricsConfig metricsConfig, + Types.StructType partitionType) { + return new EqualityDeleteTrackedFile(formatVersion, tableSchema, metricsConfig, partitionType); + } + + /** + * Returns a reusable wrapper that presents a {@link ManifestFile} as a {@link TrackedFile} leaf + * manifest row. + */ + static ManifestTrackedFile forManifestReference() { + return new ManifestTrackedFile(); + } + /** Shared base for data and delete file adapters. */ private abstract static class TrackedFileAdapter> implements ContentFile, Serializable { @@ -529,6 +582,515 @@ public ManifestFile copy() { } } + /** Shared base for content-file (DATA / EQUALITY_DELETES) write-direction wrappers. */ + abstract static class ContentTrackedFile> + implements TrackedFile, StructLike { + private final int formatVersion; + private final Types.StructType partitionType; + private final MapBackedContentStats statsWrapper; + + private Tracking tracking; + private F file; + private StructProjection partition; + private ContentStats stats; + + ContentTrackedFile( + int formatVersion, + Schema tableSchema, + MetricsConfig metricsConfig, + Types.StructType partitionType) { + Preconditions.checkArgument( + formatVersion >= TableMetadata.MIN_FORMAT_VERSION_ADAPTIVE_MANIFEST_TREE, + "Invalid format version for adaptive manifest tree: %s (must be >= %s)", + formatVersion, + TableMetadata.MIN_FORMAT_VERSION_ADAPTIVE_MANIFEST_TREE); + Preconditions.checkArgument(tableSchema != null, "Invalid table schema: null"); + Preconditions.checkArgument(metricsConfig != null, "Invalid metrics config: null"); + Preconditions.checkArgument(partitionType != null, "Invalid partition type: null"); + this.formatVersion = formatVersion; + this.partitionType = partitionType; + this.statsWrapper = new MapBackedContentStats(tableSchema, metricsConfig); + } + + void wrapWithTracking(F newFile, Tracking newTracking) { + Preconditions.checkArgument(newFile != null, "Invalid file: null"); + Preconditions.checkArgument(newTracking != null, "Invalid tracking: null"); + validateContent(newFile); + + this.file = newFile; + this.partition = projectPartition(newFile, partitionType); + this.stats = statsWrapper.wrap(newFile); + this.tracking = newTracking; + } + + /** Content-type-specific validation of the wrapped file. */ + abstract void validateContent(F newFile); + + protected F file() { + return file; + } + + @Override + public Tracking tracking() { + return tracking; + } + + @Override + public int formatVersion() { + return formatVersion; + } + + @Override + public String location() { + return file.location(); + } + + @Override + public FileFormat fileFormat() { + return file.format(); + } + + @Override + public long recordCount() { + return file.recordCount(); + } + + @Override + public long fileSizeInBytes() { + return file.fileSizeInBytes(); + } + + @Override + public Integer specId() { + return file.specId(); + } + + @Override + public StructLike partition() { + return partition; + } + + @Override + public ContentStats contentStats() { + return stats; + } + + @Override + public Integer sortOrderId() { + return null; + } + + @Override + public DeletionVector deletionVector() { + return null; + } + + @Override + public ManifestInfo manifestInfo() { + return null; + } + + @Override + public ByteBuffer keyMetadata() { + return file.keyMetadata(); + } + + @Override + public List splitOffsets() { + return file.splitOffsets(); + } + + @Override + public List equalityIds() { + return null; + } + + @Override + public TrackedFile copy() { + throw new UnsupportedOperationException( + "Reusable content-file wrapper does not support copy(); materialize via a writer instead"); + } + + @Override + public TrackedFile copyWithStats(Set requestedColumnIds) { + throw new UnsupportedOperationException( + "Reusable content-file wrapper does not support copyWithStats()"); + } + + @Override + public int size() { + return TRACKED_FILE_FIELD_COUNT; + } + + @Override + public T get(int pos, Class javaClass) { + return javaClass.cast(TrackedFileStruct.getByPos(this, pos)); + } + + @Override + public void set(int pos, T value) { + throw new UnsupportedOperationException( + "Reusable content-file wrapper does not support set()"); + } + } + + /** Wraps a {@link DataFile} as a {@link TrackedFile} row. */ + static class DataTrackedFile extends ContentTrackedFile { + DataTrackedFile( + int formatVersion, + Schema tableSchema, + MetricsConfig metricsConfig, + Types.StructType partitionType) { + super(formatVersion, tableSchema, metricsConfig, partitionType); + } + + /** Re-points this wrapper at {@code newFile} in place. */ + public DataTrackedFile wrap(DataFile newFile, Tracking tracking) { + wrapWithTracking(newFile, tracking); + return this; + } + + @Override + void validateContent(DataFile newFile) { + Preconditions.checkArgument( + newFile.content() == FileContent.DATA, + "Invalid content for data file: %s", + newFile.content()); + } + + @Override + public FileContent contentType() { + return FileContent.DATA; + } + + @Override + public Integer sortOrderId() { + return file().sortOrderId(); + } + } + + /** Wraps an equality {@link DeleteFile} as a {@link TrackedFile} row. */ + static class EqualityDeleteTrackedFile extends ContentTrackedFile { + EqualityDeleteTrackedFile( + int formatVersion, + Schema tableSchema, + MetricsConfig metricsConfig, + Types.StructType partitionType) { + super(formatVersion, tableSchema, metricsConfig, partitionType); + } + + /** Re-points this wrapper at {@code newFile} in place. */ + public EqualityDeleteTrackedFile wrap(DeleteFile newFile, Tracking tracking) { + wrapWithTracking(newFile, tracking); + return this; + } + + @Override + void validateContent(DeleteFile newFile) { + Preconditions.checkArgument( + newFile.content() == FileContent.EQUALITY_DELETES, + "Invalid content for delete file: %s", + newFile.content()); + } + + @Override + public FileContent contentType() { + return FileContent.EQUALITY_DELETES; + } + + @Override + public List equalityIds() { + return file().equalityFieldIds(); + } + } + + /** Wraps a {@link ManifestFile} as a v4+ leaf manifest row. */ + static class ManifestTrackedFile implements TrackedFile, StructLike { + private final WrappedManifestInfo manifestInfo = new WrappedManifestInfo(); + private Tracking tracking; + private ManifestFile manifest; + private long recordCount; + private FileContent contentType; + + ManifestTrackedFile() {} + + /** + * Re-points this wrapper at {@code newManifest} in place. + * + * @param newManifest manifest file being referenced; must carry an assigned {@code + * sequence_number} and {@code min_sequence_number} + * @param status entry status for the reference + * @param firstRowId first-row-id resolved by the caller for a DATA manifest reference, or null + * for a DELETE manifest reference + */ + public ManifestTrackedFile wrap(ManifestFile newManifest, EntryStatus status, Long firstRowId) { + Preconditions.checkArgument(newManifest != null, "Invalid manifest file: null"); + Preconditions.checkArgument(status != null, "Invalid status: null"); + int formatVersion = newManifest.formatVersion(); + Preconditions.checkArgument( + formatVersion == ManifestFile.LEGACY_FORMAT_VERSION + || formatVersion >= TableMetadata.MIN_FORMAT_VERSION_ADAPTIVE_MANIFEST_TREE, + "Invalid manifest format_version: %s (must be %s for pre-v4 or >= %s for v4+)", + formatVersion, + ManifestFile.LEGACY_FORMAT_VERSION, + TableMetadata.MIN_FORMAT_VERSION_ADAPTIVE_MANIFEST_TREE); + Long manifestSnapshotId = newManifest.snapshotId(); + Preconditions.checkArgument(manifestSnapshotId != null, "Invalid manifest snapshot id: null"); + long manifestSeq = newManifest.sequenceNumber(); + Preconditions.checkArgument( + manifestSeq != ManifestWriter.UNASSIGNED_SEQ, + "Invalid manifest reference %s: sequence_number is unassigned", + newManifest.path()); + Preconditions.checkArgument( + newManifest.minSequenceNumber() != ManifestWriter.UNASSIGNED_SEQ, + "Invalid manifest reference %s: min_sequence_number is unassigned", + newManifest.path()); + Preconditions.checkArgument( + firstRowId == null || newManifest.content() == ManifestContent.DATA, + "firstRowId is only valid for DATA manifests, but content is %s", + newManifest.content()); + + this.manifest = newManifest; + this.contentType = + newManifest.content() == ManifestContent.DATA + ? FileContent.DATA_MANIFEST + : FileContent.DELETE_MANIFEST; + this.recordCount = resolveRecordCount(newManifest); + this.tracking = + new TrackingStruct( + status, manifestSnapshotId, manifestSeq, manifestSeq, null, firstRowId, null, null); + this.manifestInfo.wrap(newManifest); + return this; + } + + @Override + public Tracking tracking() { + return tracking; + } + + @Override + public FileContent contentType() { + return contentType; + } + + @Override + public int formatVersion() { + return manifest.formatVersion(); + } + + @Override + public String location() { + return manifest.path(); + } + + @Override + public FileFormat fileFormat() { + return FileFormat.fromFileName(manifest.path()); + } + + @Override + public long recordCount() { + return recordCount; + } + + @Override + public long fileSizeInBytes() { + return manifest.length(); + } + + @Override + public Integer specId() { + return manifest.partitionSpecId(); + } + + @Override + public StructLike partition() { + return null; + } + + @Override + public ContentStats contentStats() { + return null; + } + + @Override + public Integer sortOrderId() { + return null; + } + + @Override + public DeletionVector deletionVector() { + return null; + } + + @Override + public ManifestInfo manifestInfo() { + return manifestInfo; + } + + @Override + public ByteBuffer keyMetadata() { + return manifest.keyMetadata(); + } + + @Override + public List splitOffsets() { + return null; + } + + @Override + public List equalityIds() { + return null; + } + + @Override + public TrackedFile copy() { + throw new UnsupportedOperationException( + "Reusable manifest-reference wrapper does not support copy()"); + } + + @Override + public TrackedFile copyWithStats(Set requestedColumnIds) { + throw new UnsupportedOperationException( + "Reusable manifest-reference wrapper does not support copyWithStats()"); + } + + @Override + public int size() { + return TRACKED_FILE_FIELD_COUNT; + } + + @Override + public T get(int pos, Class javaClass) { + return javaClass.cast(TrackedFileStruct.getByPos(this, pos)); + } + + @Override + public void set(int pos, T value) { + throw new UnsupportedOperationException( + "Reusable manifest-reference wrapper does not support set()"); + } + } + + /** Reusable {@link ManifestInfo} view over a {@link ManifestFile}'s counts. */ + private static class WrappedManifestInfo implements ManifestInfo, StructLike { + private ManifestFile manifest; + + void wrap(ManifestFile newManifest) { + this.manifest = newManifest; + } + + @Override + public int addedFilesCount() { + return zeroIfNull(manifest.addedFilesCount()); + } + + @Override + public int existingFilesCount() { + return zeroIfNull(manifest.existingFilesCount()); + } + + @Override + public int deletedFilesCount() { + return zeroIfNull(manifest.deletedFilesCount()); + } + + // TODO: GenericManifestFile doesn't yet plumb REPLACED aggregates, so replacedFilesCount() and + // replacedRowsCount() resolve to 0 today. Wire the aggregation from leaf-manifest writers up to + // the manifest_list entry in a follow-up. + @Override + public int replacedFilesCount() { + return zeroIfNull(manifest.replacedFilesCount()); + } + + @Override + public int modifiedFilesCount() { + return 0; + } + + @Override + public long addedRowsCount() { + return zeroIfNull(manifest.addedRowsCount()); + } + + @Override + public long existingRowsCount() { + return zeroIfNull(manifest.existingRowsCount()); + } + + @Override + public long deletedRowsCount() { + return zeroIfNull(manifest.deletedRowsCount()); + } + + @Override + public long replacedRowsCount() { + return zeroIfNull(manifest.replacedRowsCount()); + } + + @Override + public long modifiedRowsCount() { + return 0L; + } + + @Override + public long minSequenceNumber() { + return manifest.minSequenceNumber(); + } + + @Override + public ManifestBitmap manifestDeletionVector() { + return manifest.manifestDeletionVector(); + } + + @Override + public ManifestInfo copy() { + throw new UnsupportedOperationException( + "Reusable manifest-info wrapper does not support copy()"); + } + + @Override + public int size() { + return MANIFEST_INFO_FIELD_COUNT; + } + + @Override + public T get(int pos, Class javaClass) { + Object value = + switch (pos) { + case 0 -> addedFilesCount(); + case 1 -> existingFilesCount(); + case 2 -> deletedFilesCount(); + case 3 -> replacedFilesCount(); + case 4 -> modifiedFilesCount(); + case 5 -> addedRowsCount(); + case 6 -> existingRowsCount(); + case 7 -> deletedRowsCount(); + case 8 -> replacedRowsCount(); + case 9 -> modifiedRowsCount(); + case 10 -> minSequenceNumber(); + case 11 -> + manifestDeletionVector() != null ? manifestDeletionVector().buffer() : null; + default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos); + }; + return javaClass.cast(value); + } + + @Override + public void set(int pos, T value) { + throw new UnsupportedOperationException( + "Reusable manifest-info wrapper does not support set()"); + } + + private static int zeroIfNull(Integer value) { + return value != null ? value : 0; + } + + private static long zeroIfNull(Long value) { + return value != null ? value : 0L; + } + } + private static int resolveSpecId(TrackedFile file, Map specsById) { Integer specId = file.specId(); if (specId != null) { @@ -547,4 +1109,56 @@ private static int resolveSpecId(TrackedFile file, Map s throw new IllegalArgumentException( "Cannot find unpartitioned spec in specs: " + specsById.keySet()); } + + /** + * Projects the file's per-spec partition tuple into the target partition schema by field ID. + * Fields present in the target but not in the file's spec land as null. + */ + private static StructProjection projectPartition( + ContentFile file, Types.StructType partitionType) { + StructLike partition = file.partition(); + Types.StructType sourceType; + if (partition instanceof PartitionData) { + sourceType = ((PartitionData) partition).getPartitionType(); + } else if (partition == null || partition.size() == 0) { + sourceType = Types.StructType.of(); + } else { + throw new IllegalArgumentException( + String.format( + "Cannot project partition for %s: partition type is unavailable for %s", + file.location(), partition)); + } + + return StructProjection.createAllowMissing(sourceType, partitionType).wrap(partition); + } + + /** + * Resolves record_count for a manifest-reference row. v4+ manifests carry a persisted + * record_count; pre-v4 manifests sum the per-status file counts. + */ + private static long resolveRecordCount(ManifestFile manifest) { + if (manifest.formatVersion() >= TableMetadata.MIN_FORMAT_VERSION_ADAPTIVE_MANIFEST_TREE) { + Long persisted = manifest.recordCount(); + Preconditions.checkArgument( + persisted != null, + "Invalid v4 manifest reference for %s: record_count must be set by the writer", + manifest.path()); + return persisted; + } + + long total = 0L; + if (manifest.addedFilesCount() != null) { + total += manifest.addedFilesCount(); + } + + if (manifest.existingFilesCount() != null) { + total += manifest.existingFilesCount(); + } + + if (manifest.deletedFilesCount() != null) { + total += manifest.deletedFilesCount(); + } + + return total; + } } diff --git a/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java b/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java index 7a333968247d..42f394a81a91 100644 --- a/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java +++ b/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java @@ -267,27 +267,31 @@ public TrackedFile copyWithStats(Set requestedColumnIds) { @Override protected T internalGet(int pos, Class javaClass) { - return javaClass.cast(getByPos(pos)); + return javaClass.cast(getByPos(this, pos)); } - private Object getByPos(int pos) { + /** + * Returns the value at the given position in TrackedFile schema field order, shared with the + * write-direction wrappers so that the position-to-field mapping is defined in one place. + */ + static Object getByPos(TrackedFile file, int pos) { return switch (pos) { - case 0 -> tracking; - case 1 -> contentType != null ? contentType.id() : null; - case 2 -> formatVersion; - case 3 -> location; - case 4 -> fileFormat != null ? fileFormat.toString() : null; - case 5 -> recordCount; - case 6 -> fileSizeInBytes; - case 7 -> specId; - case 8 -> partitionData; - case 9 -> contentStats; - case 10 -> sortOrderId; - case 11 -> deletionVector; - case 12 -> manifestInfo; - case 13 -> keyMetadata(); - case 14 -> splitOffsets(); - case 15 -> equalityIds(); + case 0 -> file.tracking(); + case 1 -> file.contentType() != null ? file.contentType().id() : null; + case 2 -> file.formatVersion(); + case 3 -> file.location(); + case 4 -> file.fileFormat() != null ? file.fileFormat().toString() : null; + case 5 -> file.recordCount(); + case 6 -> file.fileSizeInBytes(); + case 7 -> file.specId(); + case 8 -> file.partition(); + case 9 -> file.contentStats(); + case 10 -> file.sortOrderId(); + case 11 -> file.deletionVector(); + case 12 -> file.manifestInfo(); + case 13 -> file.keyMetadata(); + case 14 -> file.splitOffsets(); + case 15 -> file.equalityIds(); default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos); }; } diff --git a/core/src/test/java/org/apache/iceberg/TestMapBackedContentStats.java b/core/src/test/java/org/apache/iceberg/TestMapBackedContentStats.java new file mode 100644 index 000000000000..5324eb8dd527 --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/TestMapBackedContentStats.java @@ -0,0 +1,348 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.nio.ByteBuffer; +import java.util.Comparator; +import java.util.Map; +import java.util.Set; +import org.apache.iceberg.TestHelpers.Row; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.types.Comparators; +import org.apache.iceberg.types.Conversions; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; +import org.junit.jupiter.api.Test; + +class TestMapBackedContentStats { + + // field 5 (flag) is intentionally left out of every stat map to exercise the absent-field path. + private static final Schema SCHEMA = + new Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.optional(2, "score", Types.FloatType.get()), + Types.NestedField.optional(3, "ts", Types.LongType.get()), + Types.NestedField.optional(4, "name", Types.StringType.get()), + Types.NestedField.optional(5, "flag", Types.BooleanType.get())); + + private static final MetricsConfig METRICS_CONFIG = + MetricsConfig.from(ImmutableMap.of(), SCHEMA, SortOrder.unsorted()); + + private static final PartitionData EMPTY_PARTITION = + new PartitionData(PartitionSpec.unpartitioned().partitionType()); + + /** + * A file with stats on fields 1-4 (int, float, long, string) but none on field 5 (boolean). Field + * 3 (ts) intentionally has no value_count entry to exercise the absent-count throw path. + */ + private static final DataFile FILE_WITH_STATS = + dataFile( + ImmutableMap.of(1, 100L, 2, 100L, 4, 100L), + ImmutableMap.of(2, 5L, 3, 1L, 4, 2L), + ImmutableMap.of(2, 3L), + ImmutableMap.of( + 1, buf(Types.IntegerType.get(), 1), + 2, buf(Types.FloatType.get(), 1.5f), + 3, buf(Types.LongType.get(), 100L), + 4, buf(Types.StringType.get(), "aaa")), + ImmutableMap.of( + 1, buf(Types.IntegerType.get(), 1000), + 2, buf(Types.FloatType.get(), 9.5f), + 3, buf(Types.LongType.get(), 999L), + 4, buf(Types.StringType.get(), "zzz"))); + + @Test + void boundDecodingPerType() { + MapBackedContentStats stats = + new MapBackedContentStats(SCHEMA, METRICS_CONFIG).wrap(FILE_WITH_STATS); + + FieldStats id = stats.statsFor(1); + assertThat(id.lowerBound()).isInstanceOf(Integer.class).isEqualTo(1); + assertThat(id.upperBound()).isInstanceOf(Integer.class).isEqualTo(1000); + + FieldStats score = stats.statsFor(2); + assertThat(score.lowerBound()).isInstanceOf(Float.class).isEqualTo(1.5f); + assertThat(score.upperBound()).isInstanceOf(Float.class).isEqualTo(9.5f); + + FieldStats ts = stats.statsFor(3); + assertThat(ts.lowerBound()).isInstanceOf(Long.class).isEqualTo(100L); + assertThat(ts.upperBound()).isInstanceOf(Long.class).isEqualTo(999L); + + FieldStats name = stats.statsFor(4); + assertThat(name.lowerBound()).isInstanceOf(CharSequence.class); + assertThat(name.lowerBound().toString()).isEqualTo("aaa"); + assertThat(name.upperBound()).isInstanceOf(CharSequence.class); + assertThat(name.upperBound().toString()).isEqualTo("zzz"); + } + + @Test + void missingBoundsDecodeToNull() { + // value counts present but no lower/upper bound entries for the field + DataFile file = + dataFile( + ImmutableMap.of(1, 100L), + ImmutableMap.of(), + ImmutableMap.of(), + ImmutableMap.of(), + ImmutableMap.of()); + MapBackedContentStats stats = new MapBackedContentStats(SCHEMA, METRICS_CONFIG).wrap(file); + + FieldStats id = stats.statsFor(1); + assertThat(id.lowerBound()).isNull(); + assertThat(id.upperBound()).isNull(); + assertThat(id.valueCount()).isEqualTo(100L); + } + + @Test + void countsAndDefaults() { + MapBackedContentStats stats = + new MapBackedContentStats(SCHEMA, METRICS_CONFIG).wrap(FILE_WITH_STATS); + + FieldStats id = stats.statsFor(1); + assertThat(id.hasValueCount()).isTrue(); + assertThat(id.valueCount()).isEqualTo(100L); + // id tracks no null or nan count; callers must check presence first, since the getters throw + // when the count is absent (as with FieldStatsStruct) + assertThat(id.hasNullValueCount()).isFalse(); + assertThat(id.hasNanValueCount()).isFalse(); + // primitive unbox of an absent Long count throws the JVM's helpful NPE + assertThatThrownBy(id::nullValueCount) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("Long.longValue()"); + assertThatThrownBy(id::nanValueCount) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("Long.longValue()"); + // the wrapper never tracks tight bounds or avg value size on the write path + assertThat(id.tightBounds()).isFalse(); + assertThat(id.avgValueSizeInBytes()).isNull(); + + FieldStats score = stats.statsFor(2); + assertThat(score.hasNullValueCount()).isTrue(); + assertThat(score.nullValueCount()).isEqualTo(5L); + assertThat(score.hasNanValueCount()).isTrue(); + assertThat(score.nanValueCount()).isEqualTo(3L); + + // ts has other stats but no value_count entry — statsFor is non-null, valueCount() must throw + FieldStats ts = stats.statsFor(3); + assertThat(ts.hasValueCount()).isFalse(); + assertThatThrownBy(ts::valueCount) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("Long.longValue()"); + assertThat(ts.hasNullValueCount()).isTrue(); + assertThat(ts.nullValueCount()).isEqualTo(1L); + } + + @Test + void countsOnlyColumnOmitsBounds() { + // configure the float column (score) as counts-only; its stats sub-struct must drop bounds + MetricsConfig countsConfig = + MetricsConfig.from( + ImmutableMap.of("write.metadata.metrics.column.score", "counts"), + SCHEMA, + SortOrder.unsorted()); + MapBackedContentStats stats = + new MapBackedContentStats(SCHEMA, countsConfig).wrap(FILE_WITH_STATS); + + // counts mode drops lower_bound/upper_bound/tight_bounds; only the counts remain + FieldStats score = stats.statsFor(2); + assertThat(score.type().fields()) + .extracting(Types.NestedField::name) + .containsExactly("value_count", "null_value_count", "nan_value_count"); + + // a column left at the default mode still carries bounds + assertThat(stats.statsFor(1).type().fields()) + .extracting(Types.NestedField::name) + .contains("lower_bound", "upper_bound"); + } + + @Test + void countsOnlyStructLike() { + // the pruned counts-only struct must expose its counts in the right positions via StructLike + MetricsConfig countsConfig = + MetricsConfig.from( + ImmutableMap.of("write.metadata.metrics.column.score", "counts"), + SCHEMA, + SortOrder.unsorted()); + MapBackedContentStats stats = + new MapBackedContentStats(SCHEMA, countsConfig).wrap(FILE_WITH_STATS); + + FieldStats score = stats.statsFor(2); + // struct order: value_count, null_value_count, nan_value_count (bounds pruned in counts mode) + Row expected = Row.of(100L, 5L, 3L); + Comparator comparator = Comparators.forType(score.type()); + assertThat(comparator.compare((StructLike) score, expected)).isEqualTo(0); + } + + @Test + void absentFieldHasNoStats() { + MapBackedContentStats stats = + new MapBackedContentStats(SCHEMA, METRICS_CONFIG).wrap(FILE_WITH_STATS); + + // field 5 (flag) has no entry in any stat map + assertThat(stats.statsFor(5)).isNull(); + assertThat(stats.fieldStats()) + .extracting(FieldStats::fieldId) + .containsExactlyInAnyOrder(1, 2, 3, 4); + } + + @Test + void typeMatchesStatsReadSchema() { + MapBackedContentStats stats = new MapBackedContentStats(SCHEMA, METRICS_CONFIG); + assertThat(stats.type()) + .isEqualTo(StatsUtil.statsReadSchema(SCHEMA, ImmutableList.of(1, 2, 3, 4, 5))); + } + + @Test + void defaultStructLike() { + // the default (bound-bearing) struct for an optional float exposes all six fields via + // StructLike + MapBackedContentStats stats = + new MapBackedContentStats(SCHEMA, METRICS_CONFIG).wrap(FILE_WITH_STATS); + + FieldStats score = stats.statsFor(2); + // struct order: lower_bound, upper_bound, tight_bounds, value_count, null_value_count, + // nan_value_count + Row expected = Row.of(1.5f, 9.5f, false, 100L, 5L, 3L); + Comparator comparator = Comparators.forType(score.type()); + assertThat(comparator.compare((StructLike) score, expected)).isEqualTo(0); + } + + @Test + void contentStructLikeGetReturnsChildrenOrNull() { + MapBackedContentStats stats = + new MapBackedContentStats(SCHEMA, METRICS_CONFIG).wrap(FILE_WITH_STATS); + + Set nullFieldIds = Sets.newHashSet(); + for (int pos = 0; pos < stats.size(); pos += 1) { + FieldStats child = stats.get(pos, FieldStats.class); + int fieldId = StatsUtil.toFieldId(stats.type().fields().get(pos).fieldId()); + if (child == null) { + nullFieldIds.add(fieldId); + } else { + assertThat(child.fieldId()).isEqualTo(fieldId); + assertThat(stats.statsFor(fieldId)).isSameAs(child); + } + } + + // only field 5 (flag) lacks stats + assertThat(nullFieldIds).containsExactly(5); + } + + @Test + void copyNotSupported() { + MapBackedContentStats stats = + new MapBackedContentStats(SCHEMA, METRICS_CONFIG).wrap(FILE_WITH_STATS); + + // the reusable wrapper is serialized directly; snapshots must be materialized via a writer + assertThatThrownBy(stats::copy) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("does not support copy()"); + + assertThatThrownBy(() -> stats.copy(ImmutableSet.of(1))) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("does not support copy()"); + } + + @Test + void setNotSupported() { + MapBackedContentStats stats = + new MapBackedContentStats(SCHEMA, METRICS_CONFIG).wrap(FILE_WITH_STATS); + + assertThatThrownBy(() -> stats.set(0, null)) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("does not support set()"); + } + + @Test + void fieldStatsCopyNotSupported() { + // the reusable field wrapper is serialized directly; snapshots must be materialized via a + // writer + MapBackedContentStats stats = + new MapBackedContentStats(SCHEMA, METRICS_CONFIG).wrap(FILE_WITH_STATS); + + assertThatThrownBy(stats.statsFor(1)::copy) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("does not support copy()"); + } + + @Test + void fieldStatsSetNotSupported() { + MapBackedContentStats stats = + new MapBackedContentStats(SCHEMA, METRICS_CONFIG).wrap(FILE_WITH_STATS); + + assertThatThrownBy(() -> ((StructLike) stats.statsFor(1)).set(0, 5)) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("does not support set()"); + } + + @Test + void reuseRebindsBounds() { + MapBackedContentStats stats = new MapBackedContentStats(SCHEMA, METRICS_CONFIG); + + stats.wrap(FILE_WITH_STATS); + assertThat(stats.statsFor(1).lowerBound()).isEqualTo(1); + + DataFile file2 = + dataFile( + ImmutableMap.of(1, 50L), + ImmutableMap.of(), + ImmutableMap.of(), + ImmutableMap.of(1, buf(Types.IntegerType.get(), 500)), + ImmutableMap.of(1, buf(Types.IntegerType.get(), 5000))); + stats.wrap(file2); + + assertThat(stats.statsFor(1).lowerBound()).isEqualTo(500); + assertThat(stats.statsFor(1).upperBound()).isEqualTo(5000); + assertThat(stats.statsFor(1).valueCount()).isEqualTo(50L); + // fields that had stats in file1 but not file2 are now absent + assertThat(stats.statsFor(2)).isNull(); + } + + private static ByteBuffer buf(Type type, Object value) { + return Conversions.toByteBuffer(type, value); + } + + private static DataFile dataFile( + Map valueCounts, + Map nullValueCounts, + Map nanValueCounts, + Map lowerBounds, + Map upperBounds) { + Metrics metrics = + new Metrics( + 100L, null, valueCounts, nullValueCounts, nanValueCounts, lowerBounds, upperBounds); + return new GenericDataFile( + PartitionSpec.unpartitioned().specId(), + "s3://bucket/data/file.parquet", + FileFormat.PARQUET, + EMPTY_PARTITION, + 1024L, + metrics, + null, + ImmutableList.of(0L), + null, + null); + } +} diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java index a7bfe777c147..e641097de65d 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java @@ -25,10 +25,12 @@ import static org.mockito.Mockito.when; import java.nio.ByteBuffer; +import java.util.List; import java.util.Map; import org.apache.iceberg.mumbling.MumblingTestUtil; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.types.Comparators; import org.apache.iceberg.types.Conversions; import org.apache.iceberg.types.Types; import org.junit.jupiter.api.Test; @@ -46,7 +48,6 @@ class TestTrackedFileAdapters { // Tracking values that the delegation tests validate. private static final long MANIFEST_POS = 3L; - private static final long SNAPSHOT_ID = 42L; private static final long DATA_SEQUENCE_NUMBER = 10L; private static final long FILE_SEQUENCE_NUMBER = 11L; private static final long FIRST_ROW_ID = 1000L; @@ -55,8 +56,6 @@ class TestTrackedFileAdapters { private static final Map UNPARTITIONED = ImmutableMap.of(UNPARTITIONED_SPEC_ID, PartitionSpec.unpartitioned()); - private static final ByteBuffer KEY_METADATA = ByteBuffer.wrap(new byte[] {1, 2, 3}); - private static final Schema PARTITION_SCHEMA = new Schema(Types.NestedField.required(1, "category", Types.StringType.get())); private static final int PARTITIONED_SPEC_ID = 1; @@ -73,29 +72,29 @@ class TestTrackedFileAdapters { private static final Schema TABLE_SCHEMA = new Schema( - optional(1, "id", Types.IntegerType.get()), - optional(2, "score", Types.FloatType.get()), - optional(3, "geom", Types.GeometryType.crs84())); + optional(1, "id", Types.IntegerType.get()), optional(2, "score", Types.FloatType.get())); private static final Types.StructType CONTENT_STATS_TYPE = - StatsUtil.statsReadSchema(TABLE_SCHEMA, ImmutableList.of(1, 2, 3)); + StatsUtil.statsReadSchema(TABLE_SCHEMA, ImmutableList.of(1, 2)); private static final FieldStats ID_STATS = StatsTestUtil.mockFieldStats( CONTENT_STATS_TYPE.fieldType("id").asStructType(), 1, 1, 1000, 100L, 5L, null); private static final FieldStats SCORE_STATS = StatsTestUtil.mockFieldStats( CONTENT_STATS_TYPE.fieldType("score").asStructType(), 2, 1.0f, 100.0f, 100L, 10L, 3L); - private static final FieldStats GEOM_STATS = - StatsTestUtil.mockFieldStats( - CONTENT_STATS_TYPE.fieldType("geom").asStructType(), 3, null, null, 100L, 20L, null, 12); private static final ContentStatsStruct CONTENT_STATS = new ContentStatsStruct(CONTENT_STATS_TYPE); static { CONTENT_STATS.setStats(1, ID_STATS); CONTENT_STATS.setStats(2, SCORE_STATS); - CONTENT_STATS.setStats(3, GEOM_STATS); } + private static final MetricsConfig METRICS_CONFIG = + MetricsConfig.from(ImmutableMap.of(), TABLE_SCHEMA, SortOrder.unsorted()); + private static final PartitionSpec UNPARTITIONED_SPEC = PartitionSpec.unpartitioned(); + private static final Types.StructType PARTITION_TYPE = UNPARTITIONED_SPEC.partitionType(); + private static final long SNAPSHOT_ID = 42L; + private static final Tracking MANIFEST_TRACKING = new TrackingStruct( EntryStatus.ADDED, @@ -108,6 +107,7 @@ class TestTrackedFileAdapters { null); // replacedPositions private static final ByteBuffer MANIFEST_KEY_METADATA = ByteBuffer.wrap(new byte[] {7, 8, 9}); + private static final byte[] MANIFEST_DV = MumblingTestUtil.onlyFirstBitSetBytes(); private static final ManifestInfo MANIFEST_INFO = ManifestInfoStruct.builder() @@ -125,6 +125,71 @@ class TestTrackedFileAdapters { .dv(ByteBuffer.wrap(MumblingTestUtil.onlyFirstBitSetBytes())) .build(); + private static final long DATA_SEQ = 7L; + private static final long FILE_SEQ = 11L; + private static final String DATA_PATH = "s3://bucket/data/file.parquet"; + private static final String DELETE_PATH = "s3://bucket/data/eq-delete.parquet"; + private static final String MANIFEST_PATH = "s3://bucket/metadata/manifest.parquet"; + + private static final Metrics METRICS_WITH_BOUNDS = + new Metrics( + 100L, + ImmutableMap.of(1, 16L, 2, 64L), + ImmutableMap.of(1, 100L, 2, 100L), + ImmutableMap.of(1, 0L, 2, 5L), + ImmutableMap.of(), + ImmutableMap.of(1, Conversions.toByteBuffer(Types.IntegerType.get(), 1)), + ImmutableMap.of(1, Conversions.toByteBuffer(Types.IntegerType.get(), 1000))); + + private static final Tracking ADDED_TRACKING = TrackingBuilder.added(SNAPSHOT_ID).build(); + private static final Tracking EXISTING_TRACKING = + new TrackingStruct( + EntryStatus.EXISTING, SNAPSHOT_ID, DATA_SEQ, FILE_SEQ, null, null, null, null); + private static final Tracking DELETED_TRACKING = + new TrackingStruct( + EntryStatus.DELETED, SNAPSHOT_ID, DATA_SEQ, FILE_SEQ, null, null, null, null); + + private static final DataFile DATA_FILE = + new GenericDataFile( + UNPARTITIONED_SPEC.specId(), + DATA_PATH, + FileFormat.PARQUET, + PartitionData.EMPTY, + 1024L, + new Metrics(100L, null, null, null, null), + null, + ImmutableList.of(0L), + null, + null); + private static final DataFile DATA_FILE_WITH_METRICS = + new GenericDataFile( + UNPARTITIONED_SPEC.specId(), + DATA_PATH, + FileFormat.PARQUET, + PartitionData.EMPTY, + 1024L, + METRICS_WITH_BOUNDS, + null, + ImmutableList.of(0L), + null, + null); + private static final DeleteFile EQUALITY_DELETE_FILE = + new GenericDeleteFile( + UNPARTITIONED_SPEC.specId(), + FileContent.EQUALITY_DELETES, + DELETE_PATH, + FileFormat.PARQUET, + PartitionData.EMPTY, + 512L, + new Metrics(50L, null, null, null, null), + new int[] {1}, + null, + null, + null, + null, + null, + null); + @Test void dataFileAdapterDelegation() { TrackingStruct tracking = @@ -140,7 +205,6 @@ void dataFileAdapterDelegation() { tracking.set(MANIFEST_LOCATION_ORDINAL, MANIFEST_LOCATION); tracking.set(MANIFEST_POSITION_ORDINAL, MANIFEST_POS); - DeletionVector dv = mock(DeletionVector.class); TrackedFile file = new TrackedFileStruct( tracking, @@ -154,7 +218,7 @@ void dataFileAdapterDelegation() { PARTITION, CONTENT_STATS, 3, - dv, + null, null, ByteBuffer.wrap(new byte[] {1, 2, 3}), ImmutableList.of(50L, 100L), @@ -177,15 +241,11 @@ void dataFileAdapterDelegation() { assertThat(dataFile.keyMetadata()).isEqualTo(ByteBuffer.wrap(new byte[] {1, 2, 3})); assertThat(dataFile.splitOffsets()).containsExactly(50L, 100L); assertThat(dataFile.manifestLocation()).isEqualTo(MANIFEST_LOCATION); - assertThat(dataFile.deletionVector()).isSameAs(dv); assertThat(dataFile.equalityFieldIds()).isNull(); assertThat(dataFile.columnSizes()).isNull(); - assertThat(dataFile.valueCounts()) - .containsOnly(Map.entry(1, 100L), Map.entry(2, 100L), Map.entry(3, 100L)); - assertThat(dataFile.nullValueCounts()) - .containsOnly(Map.entry(1, 5L), Map.entry(2, 10L), Map.entry(3, 20L)); + assertThat(dataFile.valueCounts()).containsOnly(Map.entry(1, 100L), Map.entry(2, 100L)); + assertThat(dataFile.nullValueCounts()).containsOnly(Map.entry(1, 5L), Map.entry(2, 10L)); assertThat(dataFile.nanValueCounts()).containsOnly(Map.entry(2, 3L)); - assertThat(dataFile.avgValueSizes()).containsOnly(Map.entry(3, 12)); assertThat(dataFile.lowerBounds()) .containsOnly( Map.entry(1, Conversions.toByteBuffer(Types.IntegerType.get(), 1)), @@ -260,12 +320,9 @@ void equalityDeleteFileAdapterDelegation() { assertThat(deleteFile.manifestLocation()).isEqualTo(MANIFEST_LOCATION); assertThat(deleteFile.equalityFieldIds()).containsExactly(1, 2, 3); assertThat(deleteFile.columnSizes()).isNull(); - assertThat(deleteFile.valueCounts()) - .containsOnly(Map.entry(1, 100L), Map.entry(2, 100L), Map.entry(3, 100L)); - assertThat(deleteFile.nullValueCounts()) - .containsOnly(Map.entry(1, 5L), Map.entry(2, 10L), Map.entry(3, 20L)); + assertThat(deleteFile.valueCounts()).containsOnly(Map.entry(1, 100L), Map.entry(2, 100L)); + assertThat(deleteFile.nullValueCounts()).containsOnly(Map.entry(1, 5L), Map.entry(2, 10L)); assertThat(deleteFile.nanValueCounts()).containsOnly(Map.entry(2, 3L)); - assertThat(deleteFile.avgValueSizes()).containsOnly(Map.entry(3, 12)); assertThat(deleteFile.lowerBounds()) .containsOnly( Map.entry(1, Conversions.toByteBuffer(Types.IntegerType.get(), 1)), @@ -294,7 +351,6 @@ void dvDeleteFileAdapterDelegation() { .offset(128L) .sizeInBytes(256L) .cardinality(10L) - .keyMetadata(KEY_METADATA) .build(); TrackingStruct tracking = @@ -338,7 +394,6 @@ void dvDeleteFileAdapterDelegation() { assertThat(dvFile.recordCount()).isEqualTo(dv.cardinality()); assertThat(dvFile.contentOffset()).isEqualTo(dv.offset()); assertThat(dvFile.contentSizeInBytes()).isEqualTo(dv.sizeInBytes()); - assertThat(dvFile.keyMetadata()).isEqualTo(KEY_METADATA); // fileSizeInBytes reports the DV blob size, not the full Puffin file size. assertThat(dvFile.fileSizeInBytes()).isEqualTo(dv.sizeInBytes()); // referencedDataFile is delegated to the tracked data file's location. @@ -355,6 +410,7 @@ void dvDeleteFileAdapterDelegation() { // fields that are null for DVs assertThat(dvFile.sortOrderId()).isNull(); assertThat(dvFile.firstRowId()).isNull(); + assertThat(dvFile.keyMetadata()).isNull(); assertThat(dvFile.splitOffsets()).isNull(); assertThat(dvFile.equalityFieldIds()).isNull(); assertThat(dvFile.columnSizes()).isNull(); @@ -440,7 +496,7 @@ private static TrackedFile serializableDataEntry() { .offset(128L) .sizeInBytes(256L) .cardinality(10L) - .keyMetadata(KEY_METADATA) + .keyMetadata(MANIFEST_KEY_METADATA) .build(); return new TrackedFileStruct( @@ -457,7 +513,7 @@ private static TrackedFile serializableDataEntry() { 3, // sortOrderId dv, // deletionVector null, // manifestInfo - KEY_METADATA, + MANIFEST_KEY_METADATA, ImmutableList.of(50L, 100L), // splitOffsets null); // equalityIds } @@ -671,6 +727,556 @@ void unknownSpecIdThrows() { .hasMessageContaining("Cannot find partition spec for spec ID"); } + @Test + void specIdMismatchThrows() { + TrackedFileStruct file = + new TrackedFileStruct( + null, + FileContent.DATA, + 0, + null, + null, + 0L, + 0L, + PARTITIONED_SPEC_ID, + null, + null, + null, + null, + null, + null, + null, + null); + int mismatchedSpecId = PARTITIONED_SPEC_ID + 1; + PartitionSpec mismatched = + PartitionSpec.builderFor(PARTITION_SCHEMA) + .identity("category") + .withSpecId(mismatchedSpecId) + .build(); + + assertThatThrownBy( + () -> + TrackedFileAdapters.asDataFile( + file, ImmutableMap.of(PARTITIONED_SPEC_ID, mismatched))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage( + "File spec ID %s does not match partition spec %s", + PARTITIONED_SPEC_ID, mismatchedSpecId); + } + + @ParameterizedTest + @EnumSource( + value = FileContent.class, + names = {"DATA_MANIFEST", "DELETE_MANIFEST"}) + void manifestFileAdapterDelegation(FileContent contentType) { + TrackedFile file = + new TrackedFileStruct( + MANIFEST_TRACKING, + contentType, + FORMAT_VERSION_V4, + MANIFEST_LOCATION, + FileFormat.PARQUET, + 10L, // recordCount + MANIFEST_FILE_SIZE, + null, // specId + null, // partition + null, // contentStats + null, // sortOrderId + null, // deletionVector + MANIFEST_INFO, + MANIFEST_KEY_METADATA, + null, // splitOffsets + null); // equalityIds + + ManifestFile manifest = TrackedFileAdapters.asManifestFile(file); + + ManifestContent expectedContent = + contentType == FileContent.DATA_MANIFEST ? ManifestContent.DATA : ManifestContent.DELETES; + assertThat(manifest.path()).isEqualTo(MANIFEST_LOCATION); + assertThat(manifest.length()).isEqualTo(MANIFEST_FILE_SIZE); + assertThat(manifest.content()).isEqualTo(expectedContent); + assertThat(manifest.sequenceNumber()).isEqualTo(DATA_SEQUENCE_NUMBER); + assertThat(manifest.minSequenceNumber()).isEqualTo(MANIFEST_INFO.minSequenceNumber()); + assertThat(manifest.snapshotId()).isEqualTo(SNAPSHOT_ID); + assertThat(manifest.addedFilesCount()).isEqualTo(MANIFEST_INFO.addedFilesCount()); + assertThat(manifest.addedRowsCount()).isEqualTo(MANIFEST_INFO.addedRowsCount()); + assertThat(manifest.existingFilesCount()).isEqualTo(MANIFEST_INFO.existingFilesCount()); + assertThat(manifest.existingRowsCount()).isEqualTo(MANIFEST_INFO.existingRowsCount()); + assertThat(manifest.deletedFilesCount()).isEqualTo(MANIFEST_INFO.deletedFilesCount()); + assertThat(manifest.deletedRowsCount()).isEqualTo(MANIFEST_INFO.deletedRowsCount()); + assertThat(manifest.firstRowId()).isEqualTo(FIRST_ROW_ID); + assertThat(manifest.keyMetadata()).isEqualTo(MANIFEST_KEY_METADATA); + assertThat(manifest.manifestDeletionVector().buffer()).isEqualTo(ByteBuffer.wrap(MANIFEST_DV)); + assertThat(manifest.partitions()).isNull(); + assertThatThrownBy(manifest::partitionSpecId) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessage("v4 manifests are not bound to a single partition spec"); + } + + @Test + void manifestFileAdapterCopy() { + TrackedFile file = Mockito.mock(TrackedFile.class); + TrackedFile fileCopy = Mockito.mock(TrackedFile.class); + Mockito.when(file.contentType()).thenReturn(FileContent.DATA_MANIFEST); + Mockito.when(file.copy()).thenReturn(fileCopy); + Mockito.when(fileCopy.location()).thenReturn(MANIFEST_LOCATION); + + ManifestFile copy = TrackedFileAdapters.asManifestFile(file).copy(); + + // copy() delegates to the tracked file's copy(), which deep-copies the nested structs. + Mockito.verify(file).copy(); + assertThat(copy.path()).isEqualTo(MANIFEST_LOCATION); + } + + @ParameterizedTest + @EnumSource( + value = FileContent.class, + mode = EnumSource.Mode.EXCLUDE, + names = {"DATA_MANIFEST", "DELETE_MANIFEST"}) + void manifestFileAdapterRejectsNonManifestContent(FileContent contentType) { + TrackedFileStruct file = dummyTrackedFile(contentType); + + assertThatThrownBy(() -> TrackedFileAdapters.asManifestFile(file)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid content type for ManifestFile: %s", contentType); + } + + @Test + void dataFileWithoutDeletionVectorReturnsNull() { + TrackedFile fileWithoutDv = mock(TrackedFile.class); + when(fileWithoutDv.contentType()).thenReturn(FileContent.DATA); + when(fileWithoutDv.deletionVector()).thenReturn(null); + + assertThat(TrackedFileAdapters.asDataFile(fileWithoutDv, UNPARTITIONED).deletionVector()) + .isNull(); + } + + @Test + void testDataFileWrapperAdded() { + DataFile file = DATA_FILE; + TrackedFileAdapters.DataTrackedFile wrapper = + TrackedFileAdapters.forDataFile( + FORMAT_VERSION_V4, TABLE_SCHEMA, METRICS_CONFIG, PARTITION_TYPE); + TrackedFile result = wrapper.wrap(file, ADDED_TRACKING); + + assertThat(result.tracking().status()).isEqualTo(EntryStatus.ADDED); + assertThat(result.tracking().snapshotId()).isEqualTo(SNAPSHOT_ID); + assertThat(result.tracking().dataSequenceNumber()).isNull(); + assertThat(result.tracking().fileSequenceNumber()).isNull(); + assertThat(result.tracking().firstRowId()).isNull(); + assertWriteDataFields(result, file, FORMAT_VERSION_V4); + } + + @Test + void testDataFileWrapperAddedWithNullSnapshotId() { + // Staged-write ADDED entry (FastAppend writes the manifest before the commit snapshot is + // assigned). Snapshot ID is inherited at read time from the enclosing root manifest's + // added_snapshot_id. + DataFile file = DATA_FILE; + Tracking tracking = + new TrackingStruct(EntryStatus.ADDED, null, null, null, null, null, null, null); + TrackedFileAdapters.DataTrackedFile wrapper = + TrackedFileAdapters.forDataFile( + FORMAT_VERSION_V4, TABLE_SCHEMA, METRICS_CONFIG, PARTITION_TYPE); + TrackedFile result = wrapper.wrap(file, tracking); + + assertThat(result.tracking().status()).isEqualTo(EntryStatus.ADDED); + assertThat(result.tracking().snapshotId()).isNull(); + assertWriteDataFields(result, file, FORMAT_VERSION_V4); + } + + @Test + void testDataFileWrapperExisting() { + DataFile file = DATA_FILE; + TrackedFileAdapters.DataTrackedFile wrapper = + TrackedFileAdapters.forDataFile( + FORMAT_VERSION_V4, TABLE_SCHEMA, METRICS_CONFIG, PARTITION_TYPE); + TrackedFile result = wrapper.wrap(file, EXISTING_TRACKING); + + assertThat(result.tracking().status()).isEqualTo(EntryStatus.EXISTING); + assertThat(result.tracking().snapshotId()).isEqualTo(SNAPSHOT_ID); + assertThat(result.tracking().dataSequenceNumber()).isEqualTo(DATA_SEQ); + assertThat(result.tracking().fileSequenceNumber()).isEqualTo(FILE_SEQ); + assertWriteDataFields(result, file, FORMAT_VERSION_V4); + } + + @Test + void testDataFileWrapperDeleted() { + DataFile file = DATA_FILE; + TrackedFileAdapters.DataTrackedFile wrapper = + TrackedFileAdapters.forDataFile( + FORMAT_VERSION_V4, TABLE_SCHEMA, METRICS_CONFIG, PARTITION_TYPE); + TrackedFile result = wrapper.wrap(file, DELETED_TRACKING); + + assertThat(result.tracking().status()).isEqualTo(EntryStatus.DELETED); + assertThat(result.tracking().snapshotId()).isEqualTo(SNAPSHOT_ID); + assertThat(result.tracking().dataSequenceNumber()).isEqualTo(DATA_SEQ); + assertThat(result.tracking().fileSequenceNumber()).isEqualTo(FILE_SEQ); + } + + @Test + void testDataFileWrapperReuse() { + // Same wrapper instance, two different files. The second wrap() call must not leak state from + // the first: this matches how V4Writer will hold one wrapper and rebind per row. + DataFile file1 = DATA_FILE; + DataFile file2 = + new GenericDataFile( + UNPARTITIONED_SPEC.specId(), + "s3://bucket/data/file2.parquet", + FileFormat.PARQUET, + PartitionData.EMPTY, + 2048L, + new Metrics(200L, null, null, null, null), + null, + ImmutableList.of(0L), + null, + null); + + TrackedFileAdapters.DataTrackedFile wrapper = + TrackedFileAdapters.forDataFile( + FORMAT_VERSION_V4, TABLE_SCHEMA, METRICS_CONFIG, PARTITION_TYPE); + + wrapper.wrap(file1, ADDED_TRACKING); + assertThat(wrapper.location()).isEqualTo(DATA_PATH); + assertThat(wrapper.recordCount()).isEqualTo(100L); + + wrapper.wrap(file2, EXISTING_TRACKING); + assertThat(wrapper.location()).isEqualTo("s3://bucket/data/file2.parquet"); + assertThat(wrapper.recordCount()).isEqualTo(200L); + assertThat(wrapper.tracking().status()).isEqualTo(EntryStatus.EXISTING); + assertThat(wrapper.fileSizeInBytes()).isEqualTo(2048L); + + // No state leaks from file1 after rebinding to file2. + assertThat(wrapper.location()).isNotEqualTo(file1.location()); + assertThat(wrapper.recordCount()).isNotEqualTo(file1.recordCount()); + assertThat(wrapper.fileSizeInBytes()).isNotEqualTo(file1.fileSizeInBytes()); + assertThat(wrapper.tracking().status()).isNotEqualTo(EntryStatus.ADDED); + } + + @Test + void testDataFileWrapperRejectsNullFile() { + TrackedFileAdapters.DataTrackedFile wrapper = + TrackedFileAdapters.forDataFile( + FORMAT_VERSION_V4, TABLE_SCHEMA, METRICS_CONFIG, PARTITION_TYPE); + assertThatThrownBy(() -> wrapper.wrap(null, ADDED_TRACKING)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Invalid file: null"); + } + + @Test + void testDataFileWrapperRejectsNullTracking() { + DataFile file = DATA_FILE; + TrackedFileAdapters.DataTrackedFile wrapper = + TrackedFileAdapters.forDataFile( + FORMAT_VERSION_V4, TABLE_SCHEMA, METRICS_CONFIG, PARTITION_TYPE); + assertThatThrownBy(() -> wrapper.wrap(file, null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Invalid tracking: null"); + } + + @Test + void testEqualityDeleteFileWrapper() { + DeleteFile file = EQUALITY_DELETE_FILE; + TrackedFileAdapters.EqualityDeleteTrackedFile wrapper = + TrackedFileAdapters.forEqualityDeleteFile( + FORMAT_VERSION_V4, TABLE_SCHEMA, METRICS_CONFIG, PARTITION_TYPE); + TrackedFile result = wrapper.wrap(file, ADDED_TRACKING); + + assertThat(result.contentType()).isEqualTo(FileContent.EQUALITY_DELETES); + assertThat(result.tracking().status()).isEqualTo(EntryStatus.ADDED); + assertThat(result.tracking().firstRowId()).isNull(); + assertThat(result.equalityIds()).containsExactly(1); + assertThat(result.sortOrderId()).isNull(); + assertThat(result.location()).isEqualTo(DELETE_PATH); + } + + @Test + void testEqualityDeleteFileWrapperRejectsV3Dv() { + // A deletion vector (in Puffin) is not an equality delete: in v4 it is carried + // inline on its data file's TrackedFile row rather than as a delete manifest entry. + DeleteFile dv = + new GenericDeleteFile( + UNPARTITIONED_SPEC.specId(), + FileContent.POSITION_DELETES, + DELETE_PATH, + FileFormat.PUFFIN, + PartitionData.EMPTY, + 512L, + new Metrics(10L, null, null, null, null), + null, + null, + null, + null, + DATA_PATH, + 0L, + 512L); + TrackedFileAdapters.EqualityDeleteTrackedFile wrapper = + TrackedFileAdapters.forEqualityDeleteFile( + FORMAT_VERSION_V4, TABLE_SCHEMA, METRICS_CONFIG, PARTITION_TYPE); + assertThatThrownBy(() -> wrapper.wrap(dv, ADDED_TRACKING)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Invalid content for delete file: POSITION_DELETES"); + } + + @Test + void testEqualityDeleteFileWrapperRejectsV2PositionDeletes() { + // A v2 position delete file (in Parquet) is not an equality delete and has no v4 leaf + // representation, so the equality-delete wrapper rejects it. + DeleteFile positionDeletes = + new GenericDeleteFile( + UNPARTITIONED_SPEC.specId(), + FileContent.POSITION_DELETES, + DELETE_PATH, + FileFormat.PARQUET, + PartitionData.EMPTY, + 512L, + new Metrics(10L, null, null, null, null), + null, + null, + null, + null, + null, + null, + null); + TrackedFileAdapters.EqualityDeleteTrackedFile wrapper = + TrackedFileAdapters.forEqualityDeleteFile( + FORMAT_VERSION_V4, TABLE_SCHEMA, METRICS_CONFIG, PARTITION_TYPE); + assertThatThrownBy(() -> wrapper.wrap(positionDeletes, ADDED_TRACKING)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Invalid content for delete file: POSITION_DELETES"); + } + + @Test + void testDataFileWrapperContentStats() { + DataFile file = DATA_FILE_WITH_METRICS; + TrackedFileAdapters.DataTrackedFile wrapper = + TrackedFileAdapters.forDataFile( + FORMAT_VERSION_V4, TABLE_SCHEMA, METRICS_CONFIG, PARTITION_TYPE); + TrackedFile result = wrapper.wrap(file, ADDED_TRACKING); + + ContentStats stats = result.contentStats(); + assertThat(stats).isNotNull(); + assertThat(stats.fieldStats()).extracting(FieldStats::fieldId).containsExactlyInAnyOrder(1, 2); + + FieldStats idStats = stats.statsFor(1); + assertThat(idStats.valueCount()).isEqualTo(100L); + assertThat(idStats.lowerBound()).isEqualTo(1); + assertThat(idStats.upperBound()).isEqualTo(1000); + } + + @Test + void testDataFileWrapperPartitionProjection() { + // Multi-spec carry-over: the writer spec has one field but the union partition type carries an + // additional field from another live spec. The wrapper must place the writer's value at the + // field-ID position and leave the extra union field null. + PartitionSpec writerSpec = PartitionSpec.builderFor(TABLE_SCHEMA).bucket("id", 16).build(); + Types.NestedField writerField = writerSpec.partitionType().fields().get(0); + Types.NestedField extraField = + Types.NestedField.optional(2000, "extra", Types.IntegerType.get()); + Types.StructType unionType = Types.StructType.of(writerField, extraField); + + PartitionData partition = new PartitionData(writerSpec.partitionType()); + partition.set(0, 7); + DataFile file = + new GenericDataFile( + writerSpec.specId(), + DATA_PATH, + FileFormat.PARQUET, + partition, + 1024L, + new Metrics(100L, null, null, null, null), + null, + ImmutableList.of(0L), + null, + null); + + TrackedFileAdapters.DataTrackedFile wrapper = + TrackedFileAdapters.forDataFile(FORMAT_VERSION_V4, TABLE_SCHEMA, METRICS_CONFIG, unionType); + TrackedFile result = wrapper.wrap(file, ADDED_TRACKING); + + StructLike projected = result.partition(); + int writerPos = unionType.fields().indexOf(writerField); + int extraPos = unionType.fields().indexOf(extraField); + assertThat(projected.get(writerPos, Integer.class)).isEqualTo(7); + assertThat(projected.get(extraPos, Integer.class)).isNull(); + } + + @Test + void testDataFileDoubleWrapRoundTrip() { + // DataFile -> TrackedFile (write) -> DataFile (read) round-trip preserves fields observable on + // the DataFile API. + DataFile source = DATA_FILE_WITH_METRICS; + Map specs = + ImmutableMap.of(UNPARTITIONED_SPEC.specId(), UNPARTITIONED_SPEC); + + TrackedFile tracked = + TrackedFileAdapters.forDataFile( + FORMAT_VERSION_V4, TABLE_SCHEMA, METRICS_CONFIG, PARTITION_TYPE) + .wrap(source, EXISTING_TRACKING); + + DataFile roundTripped = TrackedFileAdapters.asDataFile(tracked, specs); + + assertThat(roundTripped.content()).isEqualTo(FileContent.DATA); + assertThat(roundTripped.location()).isEqualTo(source.location()); + assertThat(roundTripped.format()).isEqualTo(source.format()); + assertThat(roundTripped.recordCount()).isEqualTo(source.recordCount()); + assertThat(roundTripped.fileSizeInBytes()).isEqualTo(source.fileSizeInBytes()); + assertThat(roundTripped.specId()).isEqualTo(source.specId()); + // partition survives as a StructProjection view; compare positionally by field type + assertThat( + Comparators.forType(PARTITION_TYPE) + .compare(roundTripped.partition(), source.partition())) + .isEqualTo(0); + assertThat(roundTripped.sortOrderId()).isEqualTo(source.sortOrderId()); + assertThat(roundTripped.splitOffsets()).isEqualTo(source.splitOffsets()); + assertThat(roundTripped.keyMetadata()).isEqualTo(source.keyMetadata()); + assertThat(roundTripped.dataSequenceNumber()).isEqualTo(DATA_SEQ); + assertThat(roundTripped.fileSequenceNumber()).isEqualTo(FILE_SEQ); + assertThat(roundTripped.firstRowId()).isNull(); + assertThat(roundTripped.valueCounts()).containsAllEntriesOf(source.valueCounts()); + assertThat(roundTripped.nullValueCounts()).containsAllEntriesOf(source.nullValueCounts()); + // METRICS_WITH_BOUNDS has an empty nan_value_counts map; the read side surfaces null when no + // column tracks nan + assertThat(roundTripped.nanValueCounts()).isNull(); + assertThat(roundTripped.lowerBounds()).containsAllEntriesOf(source.lowerBounds()); + assertThat(roundTripped.upperBounds()).containsAllEntriesOf(source.upperBounds()); + // the write wrapper deliberately does not carry column_sizes; a round-trip loses that field + assertThat(roundTripped.columnSizes()).isNull(); + } + + @Test + void testManifestReferenceWrapperForV4Manifest() { + GenericManifestFile manifest = v4WriteManifestFile(ManifestContent.DATA, 6L, 4); + TrackedFileAdapters.ManifestTrackedFile wrapper = TrackedFileAdapters.forManifestReference(); + TrackedFile result = wrapper.wrap(manifest, EntryStatus.ADDED, 1000L); + + assertThat(result.contentType()).isEqualTo(FileContent.DATA_MANIFEST); + assertThat(result.formatVersion()).isEqualTo(4); + assertThat(result.location()).isEqualTo(MANIFEST_PATH); + assertThat(result.tracking().status()).isEqualTo(EntryStatus.ADDED); + assertThat(result.tracking().firstRowId()).isEqualTo(1000L); + assertThat(result.recordCount()).isEqualTo(6L); + assertThat(result.manifestInfo()).isNotNull(); + assertThat(result.manifestInfo().addedFilesCount()).isEqualTo(2); + assertThat(result.manifestInfo().existingFilesCount()).isEqualTo(3); + assertThat(result.manifestInfo().deletedFilesCount()).isEqualTo(1); + assertThat(result.manifestInfo().addedRowsCount()).isEqualTo(200L); + assertThat(result.manifestInfo().replacedFilesCount()).isEqualTo(0); + assertThat(result.manifestInfo().replacedRowsCount()).isEqualTo(0L); + } + + @Test + void testManifestReferenceWrapperForPreV4() { + // Pre-v4 manifest has formatVersion=LEGACY_FORMAT_VERSION by default; the wrapper sums + // per-status counts. + ManifestFile manifest = writeManifestFile(ManifestContent.DELETES); + TrackedFileAdapters.ManifestTrackedFile wrapper = TrackedFileAdapters.forManifestReference(); + TrackedFile result = wrapper.wrap(manifest, EntryStatus.EXISTING, null); + + assertThat(result.contentType()).isEqualTo(FileContent.DELETE_MANIFEST); + assertThat(result.formatVersion()).isEqualTo(ManifestFile.LEGACY_FORMAT_VERSION); + assertThat(result.recordCount()).isEqualTo(6L); + assertThat(result.tracking().status()).isEqualTo(EntryStatus.EXISTING); + assertThat(result.tracking().firstRowId()).isNull(); + } + + @Test + void testManifestReferenceWrapperRejectsFirstRowIdOnDeleteManifest() { + ManifestFile manifest = writeManifestFile(ManifestContent.DELETES); + TrackedFileAdapters.ManifestTrackedFile wrapper = TrackedFileAdapters.forManifestReference(); + assertThatThrownBy(() -> wrapper.wrap(manifest, EntryStatus.ADDED, 100L)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("firstRowId is only valid for DATA manifests"); + } + + @Test + void testManifestReferenceWrapperRejectsUnassignedSequenceNumber() { + ManifestFile manifest = + writeManifestFile(ManifestContent.DATA, ManifestWriter.UNASSIGNED_SEQ, 4L); + TrackedFileAdapters.ManifestTrackedFile wrapper = TrackedFileAdapters.forManifestReference(); + assertThatThrownBy(() -> wrapper.wrap(manifest, EntryStatus.ADDED, null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("sequence_number is unassigned"); + } + + @Test + void testManifestReferenceWrapperRejectsV4WithoutRecordCount() { + GenericManifestFile manifest = + v4WriteManifestFile(ManifestContent.DATA, null /* recordCount */, 4); + TrackedFileAdapters.ManifestTrackedFile wrapper = TrackedFileAdapters.forManifestReference(); + assertThatThrownBy(() -> wrapper.wrap(manifest, EntryStatus.ADDED, null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("record_count must be set"); + } + + private static void assertWriteDataFields(TrackedFile result, DataFile file, int formatVersion) { + assertThat(result.contentType()).isEqualTo(FileContent.DATA); + assertThat(result.formatVersion()).isEqualTo(formatVersion); + assertThat(result.location()).isEqualTo(file.location()); + assertThat(result.fileFormat()).isEqualTo(file.format()); + assertThat(result.recordCount()).isEqualTo(file.recordCount()); + assertThat(result.fileSizeInBytes()).isEqualTo(file.fileSizeInBytes()); + assertThat(result.specId()).isEqualTo(file.specId()); + assertThat(result.sortOrderId()).isEqualTo(file.sortOrderId()); + assertThat(result.keyMetadata()).isEqualTo(file.keyMetadata()); + assertThat(result.splitOffsets()).isEqualTo(file.splitOffsets()); + // data files carry no manifest-info or deletion-vector metadata + assertThat(result.manifestInfo()).isNull(); + assertThat(result.deletionVector()).isNull(); + // equalityIds is null for data files (only equality-delete files populate it) + assertThat(result.equalityIds()).isNull(); + } + + private static ManifestFile writeManifestFile(ManifestContent content) { + return writeManifestFile(content, 5L, 4L); + } + + private static ManifestFile writeManifestFile( + ManifestContent content, long sequenceNumber, long minSequenceNumber) { + List partitions = ImmutableList.of(); + return new GenericManifestFile( + MANIFEST_PATH, + 2048L, + UNPARTITIONED_SPEC.specId(), + content, + sequenceNumber, + minSequenceNumber, + SNAPSHOT_ID, + partitions, + null, + 2, + 200L, + 3, + 300L, + 1, + 100L, + null); + } + + private static GenericManifestFile v4WriteManifestFile( + ManifestContent content, Long recordCount, int formatVersion) { + List partitions = ImmutableList.of(); + return new GenericManifestFile( + MANIFEST_PATH, + 2048L, + UNPARTITIONED_SPEC.specId(), + content, + 5L, + 4L, + SNAPSHOT_ID, + partitions, + null, + 2, + 200L, + 3, + 300L, + 1, + 100L, + null, + recordCount, + formatVersion); + } + private static void assertNullTrackingFields(ContentFile file) { assertThat(file.pos()).isNull(); assertThat(file.manifestLocation()).isNull(); From 20438e4ac774a9e027577551d8987382a7561424 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Mon, 7 Sep 2026 15:09:05 -0700 Subject: [PATCH 02/11] Core: keep v4 write wrappers off GenericManifestFile. Leave replaced counts as boxed zeros for consistency with other ManifestFile counters, unwrap already-adapted content files and manifest references, drop the formatVersion factory arg, and construct write wrappers from the TrackedFile writer schema instead of schema fragments. Report an absent partition or content stats as null rather than an empty struct, matching TrackedFileStruct. Newly written manifest-reference rows report format version 4; a rewrite of an already-adapted ManifestFile returns the original TrackedFile so a persisted 0 survives. Generated-by: Cursor Grok 4.6 Generated-by: Cursor Claude Opus 5 Co-authored-by: Cursor --- .../java/org/apache/iceberg/ManifestFile.java | 22 +-- .../apache/iceberg/GenericManifestFile.java | 64 +------ .../apache/iceberg/MapBackedContentStats.java | 29 ++- .../org/apache/iceberg/TableMetadata.java | 1 - .../java/org/apache/iceberg/TrackedFile.java | 2 +- .../apache/iceberg/TrackedFileAdapters.java | 162 ++++++++-------- .../iceberg/TestMapBackedContentStats.java | 42 +++-- .../iceberg/TestTrackedFileAdapters.java | 175 ++++++++++-------- 8 files changed, 231 insertions(+), 266 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/ManifestFile.java b/api/src/main/java/org/apache/iceberg/ManifestFile.java index 4d26c6e4d90d..7442eebead1c 100644 --- a/api/src/main/java/org/apache/iceberg/ManifestFile.java +++ b/api/src/main/java/org/apache/iceberg/ManifestFile.java @@ -29,9 +29,6 @@ public interface ManifestFile { int PARTITION_SUMMARIES_ELEMENT_ID = 508; - /** Format version for pre-v4 manifest files. */ - int LEGACY_FORMAT_VERSION = 0; - Types.NestedField PATH = required(500, "manifest_path", Types.StringType.get(), "Location URI with FS scheme"); Types.NestedField LENGTH = @@ -189,20 +186,14 @@ default boolean hasDeletedFiles() { /** Returns the total number of rows in all files with status DELETED in the manifest file. */ Long deletedRowsCount(); - /** - * Returns the number of files with status REPLACED in the manifest file, or null for pre-v4 - * manifests. - */ + /** Returns the number of files with status REPLACED in the manifest file. */ default Integer replacedFilesCount() { - return null; + return 0; } - /** - * Returns the total number of rows in all files with status REPLACED in the manifest file, or - * null for pre-v4 manifests. - */ + /** Returns the total number of rows in all files with status REPLACED in the manifest file. */ default Long replacedRowsCount() { - return null; + return 0L; } /** @@ -234,11 +225,6 @@ default ManifestBitmap manifestDeletionVector() { return null; } - /** Returns the number of entries in the manifest file, or null for pre-v4 manifests. */ - default Long recordCount() { - return null; - } - /** Returns the writer format version of the manifest, or 0 if it predates format tracking. */ default int formatVersion() { return 0; diff --git a/core/src/main/java/org/apache/iceberg/GenericManifestFile.java b/core/src/main/java/org/apache/iceberg/GenericManifestFile.java index f03578eb9851..9624484ffe0c 100644 --- a/core/src/main/java/org/apache/iceberg/GenericManifestFile.java +++ b/core/src/main/java/org/apache/iceberg/GenericManifestFile.java @@ -60,8 +60,6 @@ public class GenericManifestFile extends SupportsIndexProjection private PartitionFieldSummary[] partitions = null; private byte[] keyMetadata = null; private Long firstRowId = null; - private Long recordCount = null; - private int formatVersion = LEGACY_FORMAT_VERSION; /** Used by Avro reflection to instantiate this class when reading manifest files. */ public GenericManifestFile(Schema avroSchema) { @@ -96,7 +94,7 @@ public GenericManifestFile(Schema avroSchema) { this.firstRowId = null; } - /** Pre-v4 constructor. */ + /** Adjust the arg order to avoid conflict with the public constructor below */ GenericManifestFile( String path, long length, @@ -132,50 +130,6 @@ public GenericManifestFile(Schema avroSchema) { this.partitions = partitions == null ? null : partitions.toArray(new PartitionFieldSummary[0]); this.keyMetadata = ByteBuffers.toByteArray(keyMetadata); this.firstRowId = firstRowId; - this.recordCount = null; - this.formatVersion = LEGACY_FORMAT_VERSION; - } - - /** v4+ constructor variant that accepts recordCount and formatVersion. */ - GenericManifestFile( - String path, - long length, - int specId, - ManifestContent content, - long sequenceNumber, - long minSequenceNumber, - Long snapshotId, - List partitions, - ByteBuffer keyMetadata, - Integer addedFilesCount, - Long addedRowsCount, - Integer existingFilesCount, - Long existingRowsCount, - Integer deletedFilesCount, - Long deletedRowsCount, - Long firstRowId, - Long recordCount, - int formatVersion) { - super(ManifestFile.schema().columns().size()); - this.avroSchema = AVRO_SCHEMA; - this.manifestPath = path; - this.length = length; - this.specId = specId; - this.content = content; - this.sequenceNumber = sequenceNumber; - this.minSequenceNumber = minSequenceNumber; - this.snapshotId = snapshotId; - this.addedFilesCount = addedFilesCount; - this.addedRowsCount = addedRowsCount; - this.existingFilesCount = existingFilesCount; - this.existingRowsCount = existingRowsCount; - this.deletedFilesCount = deletedFilesCount; - this.deletedRowsCount = deletedRowsCount; - this.partitions = partitions == null ? null : partitions.toArray(new PartitionFieldSummary[0]); - this.keyMetadata = ByteBuffers.toByteArray(keyMetadata); - this.firstRowId = firstRowId; - this.recordCount = recordCount; - this.formatVersion = formatVersion; } /** @@ -218,8 +172,6 @@ private GenericManifestFile(GenericManifestFile toCopy) { ? null : Arrays.copyOf(toCopy.keyMetadata, toCopy.keyMetadata.length); this.firstRowId = toCopy.firstRowId; - this.recordCount = toCopy.recordCount; - this.formatVersion = toCopy.formatVersion; } /** Constructor for Java serialization. */ @@ -320,16 +272,6 @@ public Long firstRowId() { return firstRowId; } - @Override - public Long recordCount() { - return recordCount; - } - - @Override - public int formatVersion() { - return formatVersion; - } - @Override public int size() { return ManifestFile.schema().columns().size(); @@ -526,9 +468,7 @@ private CopyBuilder(ManifestFile toCopy) { toCopy.existingRowsCount(), toCopy.deletedFilesCount(), toCopy.deletedRowsCount(), - toCopy.firstRowId(), - toCopy.recordCount(), - toCopy.formatVersion()); + toCopy.firstRowId()); } } diff --git a/core/src/main/java/org/apache/iceberg/MapBackedContentStats.java b/core/src/main/java/org/apache/iceberg/MapBackedContentStats.java index 5bb55bb5cc75..99bd7be667a7 100644 --- a/core/src/main/java/org/apache/iceberg/MapBackedContentStats.java +++ b/core/src/main/java/org/apache/iceberg/MapBackedContentStats.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.types.Conversions; @@ -40,19 +41,19 @@ class MapBackedContentStats implements ContentStats, StructLike { private Map lowerBounds; private Map upperBounds; - MapBackedContentStats(Schema tableSchema, MetricsConfig metricsConfig) { - this.struct = StatsUtil.statsWriteSchema(tableSchema, metricsConfig); + MapBackedContentStats(Types.StructType contentStatsType) { + Preconditions.checkArgument(contentStatsType != null, "Invalid content stats type: null"); + this.struct = contentStatsType; List fields = struct.fields(); this.posToId = new int[fields.size()]; this.statsById = Maps.newHashMapWithExpectedSize(fields.size()); for (int i = 0; i < fields.size(); i += 1) { Types.NestedField field = fields.get(i); int fieldId = StatsUtil.toFieldId(field.fieldId()); + Types.StructType fieldStatsType = field.type().asStructType(); posToId[i] = fieldId; statsById.put( - fieldId, - new MapBackedFieldStats<>( - field.type().asStructType(), fieldId, tableSchema.findType(fieldId))); + fieldId, new MapBackedFieldStats<>(fieldStatsType, fieldId, boundType(fieldStatsType))); } } @@ -95,6 +96,16 @@ public Types.StructType type() { return struct; } + private static Type boundType(Types.StructType fieldStats) { + for (Types.NestedField field : fieldStats.fields()) { + if (StatsUtil.statOffset(field.fieldId()) == StatsUtil.LOWER_BOUND_OFFSET) { + return field.type(); + } + } + + return null; + } + @Override public int size() { return struct.fields().size(); @@ -151,6 +162,10 @@ public Types.StructType type() { @Override @SuppressWarnings("unchecked") public T lowerBound() { + if (boundType == null) { + return null; + } + ByteBuffer buf = lowerBounds == null ? null : lowerBounds.get(fieldId); return buf == null ? null : (T) Conversions.fromByteBuffer(boundType, buf); } @@ -158,6 +173,10 @@ public T lowerBound() { @Override @SuppressWarnings("unchecked") public T upperBound() { + if (boundType == null) { + return null; + } + ByteBuffer buf = upperBounds == null ? null : upperBounds.get(fieldId); return buf == null ? null : (T) Conversions.fromByteBuffer(boundType, buf); } diff --git a/core/src/main/java/org/apache/iceberg/TableMetadata.java b/core/src/main/java/org/apache/iceberg/TableMetadata.java index cd48ac2e6fa7..a43d4145907a 100644 --- a/core/src/main/java/org/apache/iceberg/TableMetadata.java +++ b/core/src/main/java/org/apache/iceberg/TableMetadata.java @@ -59,7 +59,6 @@ public class TableMetadata implements Serializable { static final int MIN_FORMAT_VERSION_ROW_LINEAGE = 3; static final int MIN_FORMAT_VERSION_PARQUET_MANIFESTS = 4; static final int MIN_FORMAT_VERSION_OPTIONAL_LOCATION = 4; - static final int MIN_FORMAT_VERSION_ADAPTIVE_MANIFEST_TREE = 4; static final int INITIAL_SPEC_ID = 0; static final int INITIAL_SORT_ORDER_ID = 1; static final int INITIAL_SCHEMA_ID = 0; diff --git a/core/src/main/java/org/apache/iceberg/TrackedFile.java b/core/src/main/java/org/apache/iceberg/TrackedFile.java index a02d50cf356b..c8bd3e5d817a 100644 --- a/core/src/main/java/org/apache/iceberg/TrackedFile.java +++ b/core/src/main/java/org/apache/iceberg/TrackedFile.java @@ -172,7 +172,7 @@ static Schema readSchema(Types.StructType partitionType, Types.StructType conten /** Returns partition for this file as a {@link StructLike}, or null. */ StructLike partition(); - /** Returns the content stats for this entry. */ + /** Returns the content stats for this entry, or null. */ ContentStats contentStats(); /** Returns the ID of the sort order for this file, or null. */ diff --git a/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java b/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java index 6f3eca37581a..731660cfae50 100644 --- a/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java +++ b/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java @@ -24,6 +24,7 @@ import java.util.Map; import java.util.Set; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.StructProjection; @@ -76,36 +77,20 @@ static ManifestFile asManifestFile(TrackedFile file) { /** * Returns a reusable wrapper that presents a {@link DataFile} as a {@link TrackedFile} row. * - * @param formatVersion the target table's format version (must be 4+) - * @param tableSchema table schema used to build {@link ContentStats} from the file's stats - * @param metricsConfig metrics config used to prune the content stats schema - * @param partitionType target partition struct type; use one spec's partition type for a - * single-spec manifest, or the union across live specs for a multi-spec manifest + * @param writeSchema the TrackedFile writer schema, including partition and content-stats types */ - static DataTrackedFile forDataFile( - int formatVersion, - Schema tableSchema, - MetricsConfig metricsConfig, - Types.StructType partitionType) { - return new DataTrackedFile(formatVersion, tableSchema, metricsConfig, partitionType); + static DataTrackedFile forDataFile(Schema writeSchema) { + return new DataTrackedFile(writeSchema); } /** * Returns a reusable wrapper that presents an equality {@link DeleteFile} as a {@link * TrackedFile} row. * - * @param formatVersion the target table's format version (must be 4+) - * @param tableSchema table schema used to build {@link ContentStats} from the file's stats - * @param metricsConfig metrics config used to prune the content stats schema - * @param partitionType target partition struct type; use one spec's partition type for a - * single-spec manifest, or the union across live specs for a multi-spec manifest + * @param writeSchema the TrackedFile writer schema, including partition and content-stats types */ - static EqualityDeleteTrackedFile forEqualityDeleteFile( - int formatVersion, - Schema tableSchema, - MetricsConfig metricsConfig, - Types.StructType partitionType) { - return new EqualityDeleteTrackedFile(formatVersion, tableSchema, metricsConfig, partitionType); + static EqualityDeleteTrackedFile forEqualityDeleteFile(Schema writeSchema) { + return new EqualityDeleteTrackedFile(writeSchema); } /** @@ -477,6 +462,10 @@ private TrackedManifestFile(TrackedFile file) { this.file = file; } + private TrackedFile file() { + return file; + } + @Override public String path() { return file.location(); @@ -585,7 +574,6 @@ public ManifestFile copy() { /** Shared base for content-file (DATA / EQUALITY_DELETES) write-direction wrappers. */ abstract static class ContentTrackedFile> implements TrackedFile, StructLike { - private final int formatVersion; private final Types.StructType partitionType; private final MapBackedContentStats statsWrapper; @@ -594,33 +582,27 @@ abstract static class ContentTrackedFile> private StructProjection partition; private ContentStats stats; - ContentTrackedFile( - int formatVersion, - Schema tableSchema, - MetricsConfig metricsConfig, - Types.StructType partitionType) { - Preconditions.checkArgument( - formatVersion >= TableMetadata.MIN_FORMAT_VERSION_ADAPTIVE_MANIFEST_TREE, - "Invalid format version for adaptive manifest tree: %s (must be >= %s)", - formatVersion, - TableMetadata.MIN_FORMAT_VERSION_ADAPTIVE_MANIFEST_TREE); - Preconditions.checkArgument(tableSchema != null, "Invalid table schema: null"); - Preconditions.checkArgument(metricsConfig != null, "Invalid metrics config: null"); - Preconditions.checkArgument(partitionType != null, "Invalid partition type: null"); - this.formatVersion = formatVersion; - this.partitionType = partitionType; - this.statsWrapper = new MapBackedContentStats(tableSchema, metricsConfig); - } - - void wrapWithTracking(F newFile, Tracking newTracking) { + ContentTrackedFile(Schema writeSchema) { + Preconditions.checkArgument(writeSchema != null, "Invalid write schema: null"); + Types.StructType statsType = nestedStruct(writeSchema, TrackedFile.CONTENT_STATS_ID); + this.partitionType = nestedStruct(writeSchema, TrackedFile.PARTITION_ID); + this.statsWrapper = statsType != null ? new MapBackedContentStats(statsType) : null; + } + + TrackedFile wrapWithTracking(F newFile, Tracking newTracking) { Preconditions.checkArgument(newFile != null, "Invalid file: null"); Preconditions.checkArgument(newTracking != null, "Invalid tracking: null"); validateContent(newFile); + if (newFile instanceof TrackedContentFile) { + return ((TrackedContentFile) newFile).file(); + } + this.file = newFile; - this.partition = projectPartition(newFile, partitionType); - this.stats = statsWrapper.wrap(newFile); + this.partition = partitionType != null ? projectPartition(newFile, partitionType) : null; + this.stats = statsWrapper != null ? statsWrapper.wrap(newFile) : null; this.tracking = newTracking; + return this; } /** Content-type-specific validation of the wrapped file. */ @@ -637,7 +619,7 @@ public Tracking tracking() { @Override public int formatVersion() { - return formatVersion; + return TableMetadata.MIN_FORMAT_VERSION_PARQUET_MANIFESTS; } @Override @@ -736,18 +718,13 @@ public void set(int pos, T value) { /** Wraps a {@link DataFile} as a {@link TrackedFile} row. */ static class DataTrackedFile extends ContentTrackedFile { - DataTrackedFile( - int formatVersion, - Schema tableSchema, - MetricsConfig metricsConfig, - Types.StructType partitionType) { - super(formatVersion, tableSchema, metricsConfig, partitionType); + DataTrackedFile(Schema writeSchema) { + super(writeSchema); } /** Re-points this wrapper at {@code newFile} in place. */ - public DataTrackedFile wrap(DataFile newFile, Tracking tracking) { - wrapWithTracking(newFile, tracking); - return this; + public TrackedFile wrap(DataFile newFile, Tracking tracking) { + return wrapWithTracking(newFile, tracking); } @Override @@ -771,18 +748,13 @@ public Integer sortOrderId() { /** Wraps an equality {@link DeleteFile} as a {@link TrackedFile} row. */ static class EqualityDeleteTrackedFile extends ContentTrackedFile { - EqualityDeleteTrackedFile( - int formatVersion, - Schema tableSchema, - MetricsConfig metricsConfig, - Types.StructType partitionType) { - super(formatVersion, tableSchema, metricsConfig, partitionType); + EqualityDeleteTrackedFile(Schema writeSchema) { + super(writeSchema); } /** Re-points this wrapper at {@code newFile} in place. */ - public EqualityDeleteTrackedFile wrap(DeleteFile newFile, Tracking tracking) { - wrapWithTracking(newFile, tracking); - return this; + public TrackedFile wrap(DeleteFile newFile, Tracking tracking) { + return wrapWithTracking(newFile, tracking); } @Override @@ -822,18 +794,17 @@ static class ManifestTrackedFile implements TrackedFile, StructLike { * @param status entry status for the reference * @param firstRowId first-row-id resolved by the caller for a DATA manifest reference, or null * for a DELETE manifest reference + * @return this wrapper, or the original {@link TrackedFile} if {@code newManifest} is already + * adapted from a tracked file */ - public ManifestTrackedFile wrap(ManifestFile newManifest, EntryStatus status, Long firstRowId) { + public TrackedFile wrap(ManifestFile newManifest, EntryStatus status, Long firstRowId) { Preconditions.checkArgument(newManifest != null, "Invalid manifest file: null"); Preconditions.checkArgument(status != null, "Invalid status: null"); - int formatVersion = newManifest.formatVersion(); - Preconditions.checkArgument( - formatVersion == ManifestFile.LEGACY_FORMAT_VERSION - || formatVersion >= TableMetadata.MIN_FORMAT_VERSION_ADAPTIVE_MANIFEST_TREE, - "Invalid manifest format_version: %s (must be %s for pre-v4 or >= %s for v4+)", - formatVersion, - ManifestFile.LEGACY_FORMAT_VERSION, - TableMetadata.MIN_FORMAT_VERSION_ADAPTIVE_MANIFEST_TREE); + + if (newManifest instanceof TrackedManifestFile) { + return ((TrackedManifestFile) newManifest).file(); + } + Long manifestSnapshotId = newManifest.snapshotId(); Preconditions.checkArgument(manifestSnapshotId != null, "Invalid manifest snapshot id: null"); long manifestSeq = newManifest.sequenceNumber(); @@ -875,7 +846,9 @@ public FileContent contentType() { @Override public int formatVersion() { - return manifest.formatVersion(); + // newly written ManifestFile instances have no persisted version; carry-over rows keep + // theirs via unwrap of TrackedManifestFile + return TableMetadata.MIN_FORMAT_VERSION_PARQUET_MANIFESTS; } @Override @@ -995,9 +968,6 @@ public int deletedFilesCount() { return zeroIfNull(manifest.deletedFilesCount()); } - // TODO: GenericManifestFile doesn't yet plumb REPLACED aggregates, so replacedFilesCount() and - // replacedRowsCount() resolve to 0 today. Wire the aggregation from leaf-manifest writers up to - // the manifest_list entry in a follow-up. @Override public int replacedFilesCount() { return zeroIfNull(manifest.replacedFilesCount()); @@ -1091,6 +1061,28 @@ private static long zeroIfNull(Long value) { } } + /** + * Returns the struct type of a nested field in a write schema, or null if the schema projects no + * fields for it. A field with no fields to write is unknown in the schema and is reported as + * missing rather than empty, matching {@link TrackedFileStruct}. + */ + private static Types.StructType nestedStruct(Schema writeSchema, int fieldId) { + Type type = writeSchema.findType(fieldId); + Preconditions.checkArgument(type != null, "Invalid write schema: missing field ID %s", fieldId); + if (type.typeId() == Type.TypeID.UNKNOWN) { + return null; + } + + Preconditions.checkArgument( + type.isStructType(), + "Invalid write schema field %s: expected struct or unknown, got %s", + fieldId, + type); + + Types.StructType struct = type.asStructType(); + return struct.fields().isEmpty() ? null : struct; + } + private static int resolveSpecId(TrackedFile file, Map specsById) { Integer specId = file.specId(); if (specId != null) { @@ -1132,20 +1124,8 @@ private static StructProjection projectPartition( return StructProjection.createAllowMissing(sourceType, partitionType).wrap(partition); } - /** - * Resolves record_count for a manifest-reference row. v4+ manifests carry a persisted - * record_count; pre-v4 manifests sum the per-status file counts. - */ + /** Resolves record_count for a manifest-reference row from its per-status file counts. */ private static long resolveRecordCount(ManifestFile manifest) { - if (manifest.formatVersion() >= TableMetadata.MIN_FORMAT_VERSION_ADAPTIVE_MANIFEST_TREE) { - Long persisted = manifest.recordCount(); - Preconditions.checkArgument( - persisted != null, - "Invalid v4 manifest reference for %s: record_count must be set by the writer", - manifest.path()); - return persisted; - } - long total = 0L; if (manifest.addedFilesCount() != null) { total += manifest.addedFilesCount(); @@ -1159,6 +1139,10 @@ private static long resolveRecordCount(ManifestFile manifest) { total += manifest.deletedFilesCount(); } + if (manifest.replacedFilesCount() != null) { + total += manifest.replacedFilesCount(); + } + return total; } } diff --git a/core/src/test/java/org/apache/iceberg/TestMapBackedContentStats.java b/core/src/test/java/org/apache/iceberg/TestMapBackedContentStats.java index 5324eb8dd527..3da2398aad3f 100644 --- a/core/src/test/java/org/apache/iceberg/TestMapBackedContentStats.java +++ b/core/src/test/java/org/apache/iceberg/TestMapBackedContentStats.java @@ -76,7 +76,8 @@ class TestMapBackedContentStats { @Test void boundDecodingPerType() { MapBackedContentStats stats = - new MapBackedContentStats(SCHEMA, METRICS_CONFIG).wrap(FILE_WITH_STATS); + new MapBackedContentStats(StatsUtil.statsWriteSchema(SCHEMA, METRICS_CONFIG)) + .wrap(FILE_WITH_STATS); FieldStats id = stats.statsFor(1); assertThat(id.lowerBound()).isInstanceOf(Integer.class).isEqualTo(1); @@ -107,7 +108,8 @@ void missingBoundsDecodeToNull() { ImmutableMap.of(), ImmutableMap.of(), ImmutableMap.of()); - MapBackedContentStats stats = new MapBackedContentStats(SCHEMA, METRICS_CONFIG).wrap(file); + MapBackedContentStats stats = + new MapBackedContentStats(StatsUtil.statsWriteSchema(SCHEMA, METRICS_CONFIG)).wrap(file); FieldStats id = stats.statsFor(1); assertThat(id.lowerBound()).isNull(); @@ -118,7 +120,8 @@ void missingBoundsDecodeToNull() { @Test void countsAndDefaults() { MapBackedContentStats stats = - new MapBackedContentStats(SCHEMA, METRICS_CONFIG).wrap(FILE_WITH_STATS); + new MapBackedContentStats(StatsUtil.statsWriteSchema(SCHEMA, METRICS_CONFIG)) + .wrap(FILE_WITH_STATS); FieldStats id = stats.statsFor(1); assertThat(id.hasValueCount()).isTrue(); @@ -163,7 +166,8 @@ void countsOnlyColumnOmitsBounds() { SCHEMA, SortOrder.unsorted()); MapBackedContentStats stats = - new MapBackedContentStats(SCHEMA, countsConfig).wrap(FILE_WITH_STATS); + new MapBackedContentStats(StatsUtil.statsWriteSchema(SCHEMA, countsConfig)) + .wrap(FILE_WITH_STATS); // counts mode drops lower_bound/upper_bound/tight_bounds; only the counts remain FieldStats score = stats.statsFor(2); @@ -186,7 +190,8 @@ void countsOnlyStructLike() { SCHEMA, SortOrder.unsorted()); MapBackedContentStats stats = - new MapBackedContentStats(SCHEMA, countsConfig).wrap(FILE_WITH_STATS); + new MapBackedContentStats(StatsUtil.statsWriteSchema(SCHEMA, countsConfig)) + .wrap(FILE_WITH_STATS); FieldStats score = stats.statsFor(2); // struct order: value_count, null_value_count, nan_value_count (bounds pruned in counts mode) @@ -198,7 +203,8 @@ void countsOnlyStructLike() { @Test void absentFieldHasNoStats() { MapBackedContentStats stats = - new MapBackedContentStats(SCHEMA, METRICS_CONFIG).wrap(FILE_WITH_STATS); + new MapBackedContentStats(StatsUtil.statsWriteSchema(SCHEMA, METRICS_CONFIG)) + .wrap(FILE_WITH_STATS); // field 5 (flag) has no entry in any stat map assertThat(stats.statsFor(5)).isNull(); @@ -209,7 +215,8 @@ void absentFieldHasNoStats() { @Test void typeMatchesStatsReadSchema() { - MapBackedContentStats stats = new MapBackedContentStats(SCHEMA, METRICS_CONFIG); + MapBackedContentStats stats = + new MapBackedContentStats(StatsUtil.statsWriteSchema(SCHEMA, METRICS_CONFIG)); assertThat(stats.type()) .isEqualTo(StatsUtil.statsReadSchema(SCHEMA, ImmutableList.of(1, 2, 3, 4, 5))); } @@ -219,7 +226,8 @@ void defaultStructLike() { // the default (bound-bearing) struct for an optional float exposes all six fields via // StructLike MapBackedContentStats stats = - new MapBackedContentStats(SCHEMA, METRICS_CONFIG).wrap(FILE_WITH_STATS); + new MapBackedContentStats(StatsUtil.statsWriteSchema(SCHEMA, METRICS_CONFIG)) + .wrap(FILE_WITH_STATS); FieldStats score = stats.statsFor(2); // struct order: lower_bound, upper_bound, tight_bounds, value_count, null_value_count, @@ -232,7 +240,8 @@ void defaultStructLike() { @Test void contentStructLikeGetReturnsChildrenOrNull() { MapBackedContentStats stats = - new MapBackedContentStats(SCHEMA, METRICS_CONFIG).wrap(FILE_WITH_STATS); + new MapBackedContentStats(StatsUtil.statsWriteSchema(SCHEMA, METRICS_CONFIG)) + .wrap(FILE_WITH_STATS); Set nullFieldIds = Sets.newHashSet(); for (int pos = 0; pos < stats.size(); pos += 1) { @@ -253,7 +262,8 @@ void contentStructLikeGetReturnsChildrenOrNull() { @Test void copyNotSupported() { MapBackedContentStats stats = - new MapBackedContentStats(SCHEMA, METRICS_CONFIG).wrap(FILE_WITH_STATS); + new MapBackedContentStats(StatsUtil.statsWriteSchema(SCHEMA, METRICS_CONFIG)) + .wrap(FILE_WITH_STATS); // the reusable wrapper is serialized directly; snapshots must be materialized via a writer assertThatThrownBy(stats::copy) @@ -268,7 +278,8 @@ void copyNotSupported() { @Test void setNotSupported() { MapBackedContentStats stats = - new MapBackedContentStats(SCHEMA, METRICS_CONFIG).wrap(FILE_WITH_STATS); + new MapBackedContentStats(StatsUtil.statsWriteSchema(SCHEMA, METRICS_CONFIG)) + .wrap(FILE_WITH_STATS); assertThatThrownBy(() -> stats.set(0, null)) .isInstanceOf(UnsupportedOperationException.class) @@ -280,7 +291,8 @@ void fieldStatsCopyNotSupported() { // the reusable field wrapper is serialized directly; snapshots must be materialized via a // writer MapBackedContentStats stats = - new MapBackedContentStats(SCHEMA, METRICS_CONFIG).wrap(FILE_WITH_STATS); + new MapBackedContentStats(StatsUtil.statsWriteSchema(SCHEMA, METRICS_CONFIG)) + .wrap(FILE_WITH_STATS); assertThatThrownBy(stats.statsFor(1)::copy) .isInstanceOf(UnsupportedOperationException.class) @@ -290,7 +302,8 @@ void fieldStatsCopyNotSupported() { @Test void fieldStatsSetNotSupported() { MapBackedContentStats stats = - new MapBackedContentStats(SCHEMA, METRICS_CONFIG).wrap(FILE_WITH_STATS); + new MapBackedContentStats(StatsUtil.statsWriteSchema(SCHEMA, METRICS_CONFIG)) + .wrap(FILE_WITH_STATS); assertThatThrownBy(() -> ((StructLike) stats.statsFor(1)).set(0, 5)) .isInstanceOf(UnsupportedOperationException.class) @@ -299,7 +312,8 @@ void fieldStatsSetNotSupported() { @Test void reuseRebindsBounds() { - MapBackedContentStats stats = new MapBackedContentStats(SCHEMA, METRICS_CONFIG); + MapBackedContentStats stats = + new MapBackedContentStats(StatsUtil.statsWriteSchema(SCHEMA, METRICS_CONFIG)); stats.wrap(FILE_WITH_STATS); assertThat(stats.statsFor(1).lowerBound()).isEqualTo(1); diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java index e641097de65d..d8c44203d505 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java @@ -93,6 +93,8 @@ class TestTrackedFileAdapters { MetricsConfig.from(ImmutableMap.of(), TABLE_SCHEMA, SortOrder.unsorted()); private static final PartitionSpec UNPARTITIONED_SPEC = PartitionSpec.unpartitioned(); private static final Types.StructType PARTITION_TYPE = UNPARTITIONED_SPEC.partitionType(); + private static final Schema WRITE_SCHEMA = + TrackedFile.schema(PARTITION_TYPE, StatsUtil.statsWriteSchema(TABLE_SCHEMA, METRICS_CONFIG)); private static final long SNAPSHOT_ID = 42L; private static final Tracking MANIFEST_TRACKING = @@ -854,9 +856,7 @@ void dataFileWithoutDeletionVectorReturnsNull() { @Test void testDataFileWrapperAdded() { DataFile file = DATA_FILE; - TrackedFileAdapters.DataTrackedFile wrapper = - TrackedFileAdapters.forDataFile( - FORMAT_VERSION_V4, TABLE_SCHEMA, METRICS_CONFIG, PARTITION_TYPE); + TrackedFileAdapters.DataTrackedFile wrapper = TrackedFileAdapters.forDataFile(WRITE_SCHEMA); TrackedFile result = wrapper.wrap(file, ADDED_TRACKING); assertThat(result.tracking().status()).isEqualTo(EntryStatus.ADDED); @@ -875,9 +875,7 @@ void testDataFileWrapperAddedWithNullSnapshotId() { DataFile file = DATA_FILE; Tracking tracking = new TrackingStruct(EntryStatus.ADDED, null, null, null, null, null, null, null); - TrackedFileAdapters.DataTrackedFile wrapper = - TrackedFileAdapters.forDataFile( - FORMAT_VERSION_V4, TABLE_SCHEMA, METRICS_CONFIG, PARTITION_TYPE); + TrackedFileAdapters.DataTrackedFile wrapper = TrackedFileAdapters.forDataFile(WRITE_SCHEMA); TrackedFile result = wrapper.wrap(file, tracking); assertThat(result.tracking().status()).isEqualTo(EntryStatus.ADDED); @@ -888,9 +886,7 @@ void testDataFileWrapperAddedWithNullSnapshotId() { @Test void testDataFileWrapperExisting() { DataFile file = DATA_FILE; - TrackedFileAdapters.DataTrackedFile wrapper = - TrackedFileAdapters.forDataFile( - FORMAT_VERSION_V4, TABLE_SCHEMA, METRICS_CONFIG, PARTITION_TYPE); + TrackedFileAdapters.DataTrackedFile wrapper = TrackedFileAdapters.forDataFile(WRITE_SCHEMA); TrackedFile result = wrapper.wrap(file, EXISTING_TRACKING); assertThat(result.tracking().status()).isEqualTo(EntryStatus.EXISTING); @@ -903,9 +899,7 @@ void testDataFileWrapperExisting() { @Test void testDataFileWrapperDeleted() { DataFile file = DATA_FILE; - TrackedFileAdapters.DataTrackedFile wrapper = - TrackedFileAdapters.forDataFile( - FORMAT_VERSION_V4, TABLE_SCHEMA, METRICS_CONFIG, PARTITION_TYPE); + TrackedFileAdapters.DataTrackedFile wrapper = TrackedFileAdapters.forDataFile(WRITE_SCHEMA); TrackedFile result = wrapper.wrap(file, DELETED_TRACKING); assertThat(result.tracking().status()).isEqualTo(EntryStatus.DELETED); @@ -932,9 +926,7 @@ void testDataFileWrapperReuse() { null, null); - TrackedFileAdapters.DataTrackedFile wrapper = - TrackedFileAdapters.forDataFile( - FORMAT_VERSION_V4, TABLE_SCHEMA, METRICS_CONFIG, PARTITION_TYPE); + TrackedFileAdapters.DataTrackedFile wrapper = TrackedFileAdapters.forDataFile(WRITE_SCHEMA); wrapper.wrap(file1, ADDED_TRACKING); assertThat(wrapper.location()).isEqualTo(DATA_PATH); @@ -955,9 +947,7 @@ void testDataFileWrapperReuse() { @Test void testDataFileWrapperRejectsNullFile() { - TrackedFileAdapters.DataTrackedFile wrapper = - TrackedFileAdapters.forDataFile( - FORMAT_VERSION_V4, TABLE_SCHEMA, METRICS_CONFIG, PARTITION_TYPE); + TrackedFileAdapters.DataTrackedFile wrapper = TrackedFileAdapters.forDataFile(WRITE_SCHEMA); assertThatThrownBy(() -> wrapper.wrap(null, ADDED_TRACKING)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("Invalid file: null"); @@ -966,9 +956,7 @@ void testDataFileWrapperRejectsNullFile() { @Test void testDataFileWrapperRejectsNullTracking() { DataFile file = DATA_FILE; - TrackedFileAdapters.DataTrackedFile wrapper = - TrackedFileAdapters.forDataFile( - FORMAT_VERSION_V4, TABLE_SCHEMA, METRICS_CONFIG, PARTITION_TYPE); + TrackedFileAdapters.DataTrackedFile wrapper = TrackedFileAdapters.forDataFile(WRITE_SCHEMA); assertThatThrownBy(() -> wrapper.wrap(file, null)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("Invalid tracking: null"); @@ -978,8 +966,7 @@ void testDataFileWrapperRejectsNullTracking() { void testEqualityDeleteFileWrapper() { DeleteFile file = EQUALITY_DELETE_FILE; TrackedFileAdapters.EqualityDeleteTrackedFile wrapper = - TrackedFileAdapters.forEqualityDeleteFile( - FORMAT_VERSION_V4, TABLE_SCHEMA, METRICS_CONFIG, PARTITION_TYPE); + TrackedFileAdapters.forEqualityDeleteFile(WRITE_SCHEMA); TrackedFile result = wrapper.wrap(file, ADDED_TRACKING); assertThat(result.contentType()).isEqualTo(FileContent.EQUALITY_DELETES); @@ -1011,8 +998,7 @@ void testEqualityDeleteFileWrapperRejectsV3Dv() { 0L, 512L); TrackedFileAdapters.EqualityDeleteTrackedFile wrapper = - TrackedFileAdapters.forEqualityDeleteFile( - FORMAT_VERSION_V4, TABLE_SCHEMA, METRICS_CONFIG, PARTITION_TYPE); + TrackedFileAdapters.forEqualityDeleteFile(WRITE_SCHEMA); assertThatThrownBy(() -> wrapper.wrap(dv, ADDED_TRACKING)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("Invalid content for delete file: POSITION_DELETES"); @@ -1039,8 +1025,7 @@ void testEqualityDeleteFileWrapperRejectsV2PositionDeletes() { null, null); TrackedFileAdapters.EqualityDeleteTrackedFile wrapper = - TrackedFileAdapters.forEqualityDeleteFile( - FORMAT_VERSION_V4, TABLE_SCHEMA, METRICS_CONFIG, PARTITION_TYPE); + TrackedFileAdapters.forEqualityDeleteFile(WRITE_SCHEMA); assertThatThrownBy(() -> wrapper.wrap(positionDeletes, ADDED_TRACKING)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("Invalid content for delete file: POSITION_DELETES"); @@ -1049,9 +1034,7 @@ void testEqualityDeleteFileWrapperRejectsV2PositionDeletes() { @Test void testDataFileWrapperContentStats() { DataFile file = DATA_FILE_WITH_METRICS; - TrackedFileAdapters.DataTrackedFile wrapper = - TrackedFileAdapters.forDataFile( - FORMAT_VERSION_V4, TABLE_SCHEMA, METRICS_CONFIG, PARTITION_TYPE); + TrackedFileAdapters.DataTrackedFile wrapper = TrackedFileAdapters.forDataFile(WRITE_SCHEMA); TrackedFile result = wrapper.wrap(file, ADDED_TRACKING); ContentStats stats = result.contentStats(); @@ -1064,6 +1047,44 @@ void testDataFileWrapperContentStats() { assertThat(idStats.upperBound()).isEqualTo(1000); } + @Test + void dataFileWrapperWithoutMetricsHasNoContentStats() { + // Disabling metrics for every column produces an empty stats struct, which TrackedFile.schema + // projects as unknown. The wrapper reports missing stats rather than an empty stats struct. + MetricsConfig noMetrics = + MetricsConfig.from( + ImmutableMap.of( + TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "id", "none", + TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "score", "none"), + TABLE_SCHEMA, + SortOrder.unsorted()); + Schema writeSchema = + TrackedFile.schema(PARTITION_TYPE, StatsUtil.statsWriteSchema(TABLE_SCHEMA, noMetrics)); + assertThat(writeSchema.findType(TrackedFile.CONTENT_STATS_ID)) + .isInstanceOf(Types.UnknownType.class); + + TrackedFileAdapters.DataTrackedFile wrapper = TrackedFileAdapters.forDataFile(writeSchema); + TrackedFile result = wrapper.wrap(DATA_FILE_WITH_METRICS, ADDED_TRACKING); + + assertThat(result.contentStats()).isNull(); + } + + @Test + void dataFileWrapperUnpartitionedHasNoPartition() { + // An unpartitioned spec produces an empty partition struct, which TrackedFile.schema projects + // as unknown. An absent partition is null, as it is in TrackedFileStruct. + assertThat(WRITE_SCHEMA.findType(TrackedFile.PARTITION_ID)) + .isInstanceOf(Types.UnknownType.class); + + TrackedFileAdapters.DataTrackedFile wrapper = TrackedFileAdapters.forDataFile(WRITE_SCHEMA); + TrackedFile result = wrapper.wrap(DATA_FILE, ADDED_TRACKING); + + assertThat(result.partition()).isNull(); + // the read direction normalizes the missing partition back to an empty tuple + assertThat(TrackedFileAdapters.asDataFile(result, UNPARTITIONED).partition()) + .isEqualTo(PartitionData.EMPTY); + } + @Test void testDataFileWrapperPartitionProjection() { // Multi-spec carry-over: the writer spec has one field but the union partition type carries an @@ -1091,7 +1112,9 @@ void testDataFileWrapperPartitionProjection() { null); TrackedFileAdapters.DataTrackedFile wrapper = - TrackedFileAdapters.forDataFile(FORMAT_VERSION_V4, TABLE_SCHEMA, METRICS_CONFIG, unionType); + TrackedFileAdapters.forDataFile( + TrackedFile.schema( + unionType, StatsUtil.statsWriteSchema(TABLE_SCHEMA, METRICS_CONFIG))); TrackedFile result = wrapper.wrap(file, ADDED_TRACKING); StructLike projected = result.partition(); @@ -1110,9 +1133,7 @@ void testDataFileDoubleWrapRoundTrip() { ImmutableMap.of(UNPARTITIONED_SPEC.specId(), UNPARTITIONED_SPEC); TrackedFile tracked = - TrackedFileAdapters.forDataFile( - FORMAT_VERSION_V4, TABLE_SCHEMA, METRICS_CONFIG, PARTITION_TYPE) - .wrap(source, EXISTING_TRACKING); + TrackedFileAdapters.forDataFile(WRITE_SCHEMA).wrap(source, EXISTING_TRACKING); DataFile roundTripped = TrackedFileAdapters.asDataFile(tracked, specs); @@ -1145,13 +1166,47 @@ void testDataFileDoubleWrapRoundTrip() { } @Test - void testManifestReferenceWrapperForV4Manifest() { - GenericManifestFile manifest = v4WriteManifestFile(ManifestContent.DATA, 6L, 4); + void dataFileAdapterUnwrapsToOriginalTrackedFile() { + TrackedFile original = dummyTrackedFile(FileContent.DATA); + DataFile adapted = TrackedFileAdapters.asDataFile(original, UNPARTITIONED); + + TrackedFile result = + TrackedFileAdapters.forDataFile(WRITE_SCHEMA).wrap(adapted, ADDED_TRACKING); + + assertThat(result).isSameAs(original); + } + + @Test + void equalityDeleteFileAdapterUnwrapsToOriginalTrackedFile() { + TrackedFile original = dummyTrackedFile(FileContent.EQUALITY_DELETES); + DeleteFile adapted = TrackedFileAdapters.asEqualityDeleteFile(original, UNPARTITIONED); + + TrackedFile result = + TrackedFileAdapters.forEqualityDeleteFile(WRITE_SCHEMA).wrap(adapted, ADDED_TRACKING); + + assertThat(result).isSameAs(original); + } + + @Test + void manifestReferenceAdapterUnwrapsToOriginalTrackedFile() { + TrackedFile original = dummyTrackedFile(FileContent.DATA_MANIFEST, 0); + ManifestFile adapted = TrackedFileAdapters.asManifestFile(original); + + TrackedFile result = + TrackedFileAdapters.forManifestReference().wrap(adapted, EntryStatus.ADDED, 1000L); + + assertThat(result).isSameAs(original); + assertThat(result.formatVersion()).isZero(); + } + + @Test + void testDataManifestReferenceWrapper() { + ManifestFile manifest = writeManifestFile(ManifestContent.DATA); TrackedFileAdapters.ManifestTrackedFile wrapper = TrackedFileAdapters.forManifestReference(); TrackedFile result = wrapper.wrap(manifest, EntryStatus.ADDED, 1000L); assertThat(result.contentType()).isEqualTo(FileContent.DATA_MANIFEST); - assertThat(result.formatVersion()).isEqualTo(4); + assertThat(result.formatVersion()).isEqualTo(FORMAT_VERSION_V4); assertThat(result.location()).isEqualTo(MANIFEST_PATH); assertThat(result.tracking().status()).isEqualTo(EntryStatus.ADDED); assertThat(result.tracking().firstRowId()).isEqualTo(1000L); @@ -1166,15 +1221,13 @@ void testManifestReferenceWrapperForV4Manifest() { } @Test - void testManifestReferenceWrapperForPreV4() { - // Pre-v4 manifest has formatVersion=LEGACY_FORMAT_VERSION by default; the wrapper sums - // per-status counts. + void testDeleteManifestReferenceWrapper() { ManifestFile manifest = writeManifestFile(ManifestContent.DELETES); TrackedFileAdapters.ManifestTrackedFile wrapper = TrackedFileAdapters.forManifestReference(); TrackedFile result = wrapper.wrap(manifest, EntryStatus.EXISTING, null); assertThat(result.contentType()).isEqualTo(FileContent.DELETE_MANIFEST); - assertThat(result.formatVersion()).isEqualTo(ManifestFile.LEGACY_FORMAT_VERSION); + assertThat(result.formatVersion()).isEqualTo(FORMAT_VERSION_V4); assertThat(result.recordCount()).isEqualTo(6L); assertThat(result.tracking().status()).isEqualTo(EntryStatus.EXISTING); assertThat(result.tracking().firstRowId()).isNull(); @@ -1199,16 +1252,6 @@ void testManifestReferenceWrapperRejectsUnassignedSequenceNumber() { .hasMessageContaining("sequence_number is unassigned"); } - @Test - void testManifestReferenceWrapperRejectsV4WithoutRecordCount() { - GenericManifestFile manifest = - v4WriteManifestFile(ManifestContent.DATA, null /* recordCount */, 4); - TrackedFileAdapters.ManifestTrackedFile wrapper = TrackedFileAdapters.forManifestReference(); - assertThatThrownBy(() -> wrapper.wrap(manifest, EntryStatus.ADDED, null)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("record_count must be set"); - } - private static void assertWriteDataFields(TrackedFile result, DataFile file, int formatVersion) { assertThat(result.contentType()).isEqualTo(FileContent.DATA); assertThat(result.formatVersion()).isEqualTo(formatVersion); @@ -1253,30 +1296,6 @@ private static ManifestFile writeManifestFile( null); } - private static GenericManifestFile v4WriteManifestFile( - ManifestContent content, Long recordCount, int formatVersion) { - List partitions = ImmutableList.of(); - return new GenericManifestFile( - MANIFEST_PATH, - 2048L, - UNPARTITIONED_SPEC.specId(), - content, - 5L, - 4L, - SNAPSHOT_ID, - partitions, - null, - 2, - 200L, - 3, - 300L, - 1, - 100L, - null, - recordCount, - formatVersion); - } - private static void assertNullTrackingFields(ContentFile file) { assertThat(file.pos()).isNull(); assertThat(file.manifestLocation()).isNull(); @@ -1298,10 +1317,14 @@ private static PartitionData partition(String category) { /** Minimal file with no tracking, used by the rejection and null-tracking tests. */ private static TrackedFileStruct dummyTrackedFile(FileContent contentType) { + return dummyTrackedFile(contentType, FORMAT_VERSION_V4); + } + + private static TrackedFileStruct dummyTrackedFile(FileContent contentType, int formatVersion) { return new TrackedFileStruct( null, contentType, - FORMAT_VERSION_V4, + formatVersion, DATA_FILE_LOCATION, FileFormat.PARQUET, 1L, From db1f921db5837fe48a22eb154a60ed4129549b97 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Mon, 7 Sep 2026 22:35:14 -0700 Subject: [PATCH 03/11] Core: Delegate ManifestFile.formatVersion and replaced counts from the wrapped TrackedFile. The asManifestFile adapter inherited ManifestFile defaults, so a persisted format version and non-zero replaced counts were dropped on the ManifestFile view. Generated-by: Cursor Grok 4.6 Co-authored-by: Cursor --- .../apache/iceberg/TrackedFileAdapters.java | 10 ++ .../iceberg/TestTrackedFileAdapters.java | 129 +----------------- 2 files changed, 17 insertions(+), 122 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java b/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java index 731660cfae50..9db3b1982dce 100644 --- a/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java +++ b/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java @@ -540,6 +540,16 @@ public Long deletedRowsCount() { return file.manifestInfo().deletedRowsCount(); } + @Override + public Integer replacedFilesCount() { + return file.manifestInfo().replacedFilesCount(); + } + + @Override + public Long replacedRowsCount() { + return file.manifestInfo().replacedRowsCount(); + } + @Override public List partitions() { return null; diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java index d8c44203d505..c64794d5eab7 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java @@ -116,12 +116,12 @@ class TestTrackedFileAdapters { .addedFilesCount(3) .existingFilesCount(5) .deletedFilesCount(2) - .replacedFilesCount(0) + .replacedFilesCount(4) .modifiedFilesCount(0) .addedRowsCount(300L) .existingRowsCount(500L) .deletedRowsCount(200L) - .replacedRowsCount(0L) + .replacedRowsCount(40L) .modifiedRowsCount(0L) .minSequenceNumber(7L) .dv(ByteBuffer.wrap(MumblingTestUtil.onlyFirstBitSetBytes())) @@ -579,6 +579,8 @@ void manifestFileAdapterDelegation(FileContent contentType) { assertThat(manifest.existingRowsCount()).isEqualTo(MANIFEST_INFO.existingRowsCount()); assertThat(manifest.deletedFilesCount()).isEqualTo(MANIFEST_INFO.deletedFilesCount()); assertThat(manifest.deletedRowsCount()).isEqualTo(MANIFEST_INFO.deletedRowsCount()); + assertThat(manifest.replacedFilesCount()).isEqualTo(MANIFEST_INFO.replacedFilesCount()); + assertThat(manifest.replacedRowsCount()).isEqualTo(MANIFEST_INFO.replacedRowsCount()); assertThat(manifest.firstRowId()).isEqualTo(FIRST_ROW_ID); assertThat(manifest.keyMetadata()).isEqualTo(MANIFEST_KEY_METADATA); assertThat(manifest.manifestDeletionVector().buffer()) @@ -730,127 +732,10 @@ void unknownSpecIdThrows() { } @Test - void specIdMismatchThrows() { - TrackedFileStruct file = - new TrackedFileStruct( - null, - FileContent.DATA, - 0, - null, - null, - 0L, - 0L, - PARTITIONED_SPEC_ID, - null, - null, - null, - null, - null, - null, - null, - null); - int mismatchedSpecId = PARTITIONED_SPEC_ID + 1; - PartitionSpec mismatched = - PartitionSpec.builderFor(PARTITION_SCHEMA) - .identity("category") - .withSpecId(mismatchedSpecId) - .build(); - - assertThatThrownBy( - () -> - TrackedFileAdapters.asDataFile( - file, ImmutableMap.of(PARTITIONED_SPEC_ID, mismatched))) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage( - "File spec ID %s does not match partition spec %s", - PARTITIONED_SPEC_ID, mismatchedSpecId); - } - - @ParameterizedTest - @EnumSource( - value = FileContent.class, - names = {"DATA_MANIFEST", "DELETE_MANIFEST"}) - void manifestFileAdapterDelegation(FileContent contentType) { - TrackedFile file = - new TrackedFileStruct( - MANIFEST_TRACKING, - contentType, - FORMAT_VERSION_V4, - MANIFEST_LOCATION, - FileFormat.PARQUET, - 10L, // recordCount - MANIFEST_FILE_SIZE, - null, // specId - null, // partition - null, // contentStats - null, // sortOrderId - null, // deletionVector - MANIFEST_INFO, - MANIFEST_KEY_METADATA, - null, // splitOffsets - null); // equalityIds - - ManifestFile manifest = TrackedFileAdapters.asManifestFile(file); - - ManifestContent expectedContent = - contentType == FileContent.DATA_MANIFEST ? ManifestContent.DATA : ManifestContent.DELETES; - assertThat(manifest.path()).isEqualTo(MANIFEST_LOCATION); - assertThat(manifest.length()).isEqualTo(MANIFEST_FILE_SIZE); - assertThat(manifest.content()).isEqualTo(expectedContent); - assertThat(manifest.sequenceNumber()).isEqualTo(DATA_SEQUENCE_NUMBER); - assertThat(manifest.minSequenceNumber()).isEqualTo(MANIFEST_INFO.minSequenceNumber()); - assertThat(manifest.snapshotId()).isEqualTo(SNAPSHOT_ID); - assertThat(manifest.addedFilesCount()).isEqualTo(MANIFEST_INFO.addedFilesCount()); - assertThat(manifest.addedRowsCount()).isEqualTo(MANIFEST_INFO.addedRowsCount()); - assertThat(manifest.existingFilesCount()).isEqualTo(MANIFEST_INFO.existingFilesCount()); - assertThat(manifest.existingRowsCount()).isEqualTo(MANIFEST_INFO.existingRowsCount()); - assertThat(manifest.deletedFilesCount()).isEqualTo(MANIFEST_INFO.deletedFilesCount()); - assertThat(manifest.deletedRowsCount()).isEqualTo(MANIFEST_INFO.deletedRowsCount()); - assertThat(manifest.firstRowId()).isEqualTo(FIRST_ROW_ID); - assertThat(manifest.keyMetadata()).isEqualTo(MANIFEST_KEY_METADATA); - assertThat(manifest.manifestDeletionVector().buffer()).isEqualTo(ByteBuffer.wrap(MANIFEST_DV)); - assertThat(manifest.partitions()).isNull(); - assertThatThrownBy(manifest::partitionSpecId) - .isInstanceOf(UnsupportedOperationException.class) - .hasMessage("v4 manifests are not bound to a single partition spec"); - } - - @Test - void manifestFileAdapterCopy() { - TrackedFile file = Mockito.mock(TrackedFile.class); - TrackedFile fileCopy = Mockito.mock(TrackedFile.class); - Mockito.when(file.contentType()).thenReturn(FileContent.DATA_MANIFEST); - Mockito.when(file.copy()).thenReturn(fileCopy); - Mockito.when(fileCopy.location()).thenReturn(MANIFEST_LOCATION); - - ManifestFile copy = TrackedFileAdapters.asManifestFile(file).copy(); - - // copy() delegates to the tracked file's copy(), which deep-copies the nested structs. - Mockito.verify(file).copy(); - assertThat(copy.path()).isEqualTo(MANIFEST_LOCATION); - } - - @ParameterizedTest - @EnumSource( - value = FileContent.class, - mode = EnumSource.Mode.EXCLUDE, - names = {"DATA_MANIFEST", "DELETE_MANIFEST"}) - void manifestFileAdapterRejectsNonManifestContent(FileContent contentType) { - TrackedFileStruct file = dummyTrackedFile(contentType); - - assertThatThrownBy(() -> TrackedFileAdapters.asManifestFile(file)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Invalid content type for ManifestFile: %s", contentType); - } - - @Test - void dataFileWithoutDeletionVectorReturnsNull() { - TrackedFile fileWithoutDv = mock(TrackedFile.class); - when(fileWithoutDv.contentType()).thenReturn(FileContent.DATA); - when(fileWithoutDv.deletionVector()).thenReturn(null); + void manifestFileAdapterDelegatesFormatVersion() { + TrackedFile original = dummyTrackedFile(FileContent.DATA_MANIFEST, 0); - assertThat(TrackedFileAdapters.asDataFile(fileWithoutDv, UNPARTITIONED).deletionVector()) - .isNull(); + assertThat(TrackedFileAdapters.asManifestFile(original).formatVersion()).isZero(); } @Test From 4f2d3584cf850db9e4d3c9d08ce38b2ac6ea7d73 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Tue, 29 Sep 2026 21:57:28 -0700 Subject: [PATCH 04/11] Core: Reshape v4 write adapters as conversion wrappers. Drop StructLike and write-schema from TrackedFileAdapters, add ManifestFile modified counts, and align MapBackedContentStats with review feedback. Generated-by: Cursor Grok 4.6 Co-authored-by: Cursor --- .../java/org/apache/iceberg/ManifestFile.java | 10 + .../apache/iceberg/MapBackedContentStats.java | 185 +++---- .../java/org/apache/iceberg/TrackedFile.java | 2 +- .../apache/iceberg/TrackedFileAdapters.java | 474 ++++++----------- .../org/apache/iceberg/TrackedFileStruct.java | 40 +- .../iceberg/TestMapBackedContentStats.java | 187 ++----- .../iceberg/TestTrackedFileAdapters.java | 486 +++++++----------- 7 files changed, 465 insertions(+), 919 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/ManifestFile.java b/api/src/main/java/org/apache/iceberg/ManifestFile.java index 7442eebead1c..32c7adae679f 100644 --- a/api/src/main/java/org/apache/iceberg/ManifestFile.java +++ b/api/src/main/java/org/apache/iceberg/ManifestFile.java @@ -196,6 +196,16 @@ default Long replacedRowsCount() { return 0L; } + /** Returns the number of files with status MODIFIED in the manifest file. */ + default Integer modifiedFilesCount() { + return 0; + } + + /** Returns the total number of rows in all files with status MODIFIED in the manifest file. */ + default Long modifiedRowsCount() { + return 0L; + } + /** * Returns a list of {@link PartitionFieldSummary partition field summaries}. * diff --git a/core/src/main/java/org/apache/iceberg/MapBackedContentStats.java b/core/src/main/java/org/apache/iceberg/MapBackedContentStats.java index 99bd7be667a7..fc68cfad48c2 100644 --- a/core/src/main/java/org/apache/iceberg/MapBackedContentStats.java +++ b/core/src/main/java/org/apache/iceberg/MapBackedContentStats.java @@ -19,8 +19,8 @@ package org.apache.iceberg; import java.nio.ByteBuffer; -import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.Iterables; @@ -29,124 +29,100 @@ import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; -/** Reusable {@link ContentStats} view over a legacy {@link ContentFile}'s stat maps. */ -class MapBackedContentStats implements ContentStats, StructLike { - private final Types.StructType struct; - private final int[] posToId; - private final Map> statsById; +/** Reusable {@link ContentStats} view over a {@link ContentFile}'s stat maps. */ +class MapBackedContentStats implements ContentStats { + private final Types.StructType type; + private final Map> statsById = Maps.newHashMap(); private Map valueCounts; private Map nullValueCounts; private Map nanValueCounts; + private Map avgValueSizes; private Map lowerBounds; private Map upperBounds; - MapBackedContentStats(Types.StructType contentStatsType) { - Preconditions.checkArgument(contentStatsType != null, "Invalid content stats type: null"); - this.struct = contentStatsType; - List fields = struct.fields(); - this.posToId = new int[fields.size()]; - this.statsById = Maps.newHashMapWithExpectedSize(fields.size()); - for (int i = 0; i < fields.size(); i += 1) { - Types.NestedField field = fields.get(i); - int fieldId = StatsUtil.toFieldId(field.fieldId()); - Types.StructType fieldStatsType = field.type().asStructType(); - posToId[i] = fieldId; - statsById.put( - fieldId, new MapBackedFieldStats<>(fieldStatsType, fieldId, boundType(fieldStatsType))); - } + MapBackedContentStats(Schema tableSchema, MetricsConfig metricsConfig) { + Preconditions.checkArgument(tableSchema != null, "Invalid table schema: null"); + Preconditions.checkArgument(metricsConfig != null, "Invalid metrics config: null"); + this.type = StatsUtil.statsWriteSchema(tableSchema, metricsConfig); } MapBackedContentStats wrap(ContentFile file) { this.valueCounts = file.valueCounts(); this.nullValueCounts = file.nullValueCounts(); this.nanValueCounts = file.nanValueCounts(); + this.avgValueSizes = file.avgValueSizes(); this.lowerBounds = file.lowerBounds(); this.upperBounds = file.upperBounds(); return this; } - private boolean hasStats(int id) { - return containsId(valueCounts, id) - || containsId(nullValueCounts, id) - || containsId(nanValueCounts, id) - || containsId(lowerBounds, id) - || containsId(upperBounds, id); - } - - private static boolean containsId(Map map, int id) { - return map != null && map.containsKey(id); - } - @Override public Iterable> fieldStats() { - return Iterables.transform( - Iterables.filter(statsById.entrySet(), entry -> hasStats(entry.getKey())), - Map.Entry::getValue); + return Iterables.filter( + Iterables.transform(type.fields(), field -> statsFor(StatsUtil.toFieldId(field.fieldId()))), + Objects::nonNull); } @Override @SuppressWarnings("unchecked") public FieldStats statsFor(int fieldId) { - return hasStats(fieldId) ? (FieldStats) statsById.get(fieldId) : null; - } - - @Override - public Types.StructType type() { - return struct; - } + if (!hasStats(fieldId)) { + return null; + } - private static Type boundType(Types.StructType fieldStats) { - for (Types.NestedField field : fieldStats.fields()) { - if (StatsUtil.statOffset(field.fieldId()) == StatsUtil.LOWER_BOUND_OFFSET) { - return field.type(); - } + FieldStats fieldStats = statsById.get(fieldId); + if (fieldStats == null) { + fieldStats = new MapBackedFieldStats<>(fieldId); + statsById.put(fieldId, fieldStats); } - return null; + return (FieldStats) fieldStats; } @Override - public int size() { - return struct.fields().size(); + public Types.StructType type() { + return type; } @Override - public T get(int pos, Class javaClass) { - int id = posToId[pos]; - return javaClass.cast(hasStats(id) ? statsById.get(id) : null); + public ContentStats copy() { + throw new UnsupportedOperationException("copy is not implemented"); } @Override - public void set(int pos, T value) { - throw new UnsupportedOperationException( - "Reusable content stats wrapper does not support set()"); + public ContentStats copy(Set fieldIds) { + throw new UnsupportedOperationException("copy is not implemented"); } - @Override - public ContentStats copy() { - throw new UnsupportedOperationException( - "Reusable content stats wrapper does not support copy(); materialize via a writer instead"); + private boolean hasStats(int id) { + return containsId(valueCounts, id) + || containsId(nullValueCounts, id) + || containsId(nanValueCounts, id) + || containsId(avgValueSizes, id) + || containsId(lowerBounds, id) + || containsId(upperBounds, id); } - @Override - public ContentStats copy(Set fieldIds) { - throw new UnsupportedOperationException( - "Reusable content stats wrapper does not support copy(); materialize via a writer instead"); + private static boolean containsId(Map map, int id) { + return map != null && map.containsKey(id); } /** Reusable {@link FieldStats} view over one field's entries in a {@link ContentFile}'s maps. */ - private class MapBackedFieldStats implements FieldStats, StructLike { - private final Types.StructType struct; + private class MapBackedFieldStats implements FieldStats { private final int fieldId; + private final Types.StructType struct; private final Type boundType; - private final int[] posToOffset; - MapBackedFieldStats(Types.StructType struct, int fieldId, Type boundType) { - this.struct = struct; + MapBackedFieldStats(int fieldId) { + Types.NestedField field = type.field(StatsUtil.toBaseId(fieldId)); + Preconditions.checkArgument( + field != null, + "Cannot convert stats for field ID %s: unknown, not a scalar, or not in metrics config", + fieldId); this.fieldId = fieldId; - this.boundType = boundType; - this.posToOffset = posToOffset(struct); + this.struct = field.type().asStructType(); + this.boundType = struct.fieldType(StatsUtil.LOWER_BOUND_NAME); } @Override @@ -160,25 +136,22 @@ public Types.StructType type() { } @Override - @SuppressWarnings("unchecked") public T lowerBound() { - if (boundType == null) { - return null; - } - - ByteBuffer buf = lowerBounds == null ? null : lowerBounds.get(fieldId); - return buf == null ? null : (T) Conversions.fromByteBuffer(boundType, buf); + return decodeBound(lowerBounds); } @Override - @SuppressWarnings("unchecked") public T upperBound() { - if (boundType == null) { + return decodeBound(upperBounds); + } + + @SuppressWarnings("unchecked") + private T decodeBound(Map bounds) { + if (boundType == null || bounds == null) { return null; } - ByteBuffer buf = upperBounds == null ? null : upperBounds.get(fieldId); - return buf == null ? null : (T) Conversions.fromByteBuffer(boundType, buf); + return (T) Conversions.fromByteBuffer(boundType, bounds.get(fieldId)); } @Override @@ -218,56 +191,16 @@ public long nanValueCount() { @Override public Integer avgValueSizeInBytes() { - return null; - } - - private Long count(Map counts) { - return counts == null ? null : counts.get(fieldId); - } - - @Override - public int size() { - return struct.fields().size(); - } - - @Override - public C get(int pos, Class javaClass) { - return javaClass.cast(getOffset(posToOffset[pos])); - } - - private Object getOffset(int offset) { - return switch (offset) { - case StatsUtil.LOWER_BOUND_OFFSET -> lowerBound(); - case StatsUtil.UPPER_BOUND_OFFSET -> upperBound(); - case StatsUtil.TIGHT_BOUNDS_OFFSET -> tightBounds(); - case StatsUtil.VALUE_COUNT_OFFSET -> count(valueCounts); - case StatsUtil.NULL_VALUE_COUNT_OFFSET -> count(nullValueCounts); - case StatsUtil.NAN_VALUE_COUNT_OFFSET -> count(nanValueCounts); - case StatsUtil.AVG_VALUE_SIZE_OFFSET -> null; - default -> throw new UnsupportedOperationException("Unsupported stats offset: " + offset); - }; - } - - @Override - public void set(int pos, C value) { - throw new UnsupportedOperationException( - "Reusable field stats wrapper does not support set()"); + return avgValueSizes == null ? null : avgValueSizes.get(fieldId); } @Override public FieldStats copy() { - throw new UnsupportedOperationException( - "Reusable field stats wrapper does not support copy(); materialize via a writer instead"); + throw new UnsupportedOperationException("copy is not implemented"); } - private static int[] posToOffset(Types.StructType struct) { - List fields = struct.fields(); - int[] offsets = new int[fields.size()]; - for (int i = 0; i < offsets.length; i += 1) { - offsets[i] = StatsUtil.statOffset(fields.get(i).fieldId()); - } - - return offsets; + private Long count(Map counts) { + return counts == null ? null : counts.get(fieldId); } } } diff --git a/core/src/main/java/org/apache/iceberg/TrackedFile.java b/core/src/main/java/org/apache/iceberg/TrackedFile.java index c8bd3e5d817a..a02d50cf356b 100644 --- a/core/src/main/java/org/apache/iceberg/TrackedFile.java +++ b/core/src/main/java/org/apache/iceberg/TrackedFile.java @@ -172,7 +172,7 @@ static Schema readSchema(Types.StructType partitionType, Types.StructType conten /** Returns partition for this file as a {@link StructLike}, or null. */ StructLike partition(); - /** Returns the content stats for this entry, or null. */ + /** Returns the content stats for this entry. */ ContentStats contentStats(); /** Returns the ID of the sort order for this file, or null. */ diff --git a/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java b/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java index 9db3b1982dce..3c451442fe1e 100644 --- a/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java +++ b/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java @@ -24,21 +24,13 @@ import java.util.Map; import java.util.Set; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; -import org.apache.iceberg.types.Type; -import org.apache.iceberg.types.Types; -import org.apache.iceberg.util.StructProjection; /** - * Adapts between the {@link TrackedFile} row and the {@link DataFile} / {@link DeleteFile} / {@link - * ManifestFile} APIs in both directions. + * Adapts between the {@link TrackedFile} model and the {@link DataFile} / {@link DeleteFile} / + * {@link ManifestFile} APIs in both directions. */ class TrackedFileAdapters { - private static final int TRACKED_FILE_FIELD_COUNT = - TrackedFile.schema(Types.StructType.of(), Types.StructType.of()).asStruct().fields().size(); - - private static final int MANIFEST_INFO_FIELD_COUNT = ManifestInfo.schema().fields().size(); - private TrackedFileAdapters() {} static DataFile asDataFile(TrackedFile file, Map specsById) { @@ -74,30 +66,13 @@ static ManifestFile asManifestFile(TrackedFile file) { return new TrackedManifestFile(file); } - /** - * Returns a reusable wrapper that presents a {@link DataFile} as a {@link TrackedFile} row. - * - * @param writeSchema the TrackedFile writer schema, including partition and content-stats types - */ - static DataTrackedFile forDataFile(Schema writeSchema) { - return new DataTrackedFile(writeSchema); - } - - /** - * Returns a reusable wrapper that presents an equality {@link DeleteFile} as a {@link - * TrackedFile} row. - * - * @param writeSchema the TrackedFile writer schema, including partition and content-stats types - */ - static EqualityDeleteTrackedFile forEqualityDeleteFile(Schema writeSchema) { - return new EqualityDeleteTrackedFile(writeSchema); + /** Returns a reusable adapter from {@link DataFile} to {@link TrackedFile}. */ + static DataTrackedFile forDataFile(Schema tableSchema, MetricsConfig metricsConfig) { + return new DataTrackedFile(tableSchema, metricsConfig); } - /** - * Returns a reusable wrapper that presents a {@link ManifestFile} as a {@link TrackedFile} leaf - * manifest row. - */ - static ManifestTrackedFile forManifestReference() { + /** Returns a reusable adapter from {@link ManifestFile} to {@link TrackedFile}. */ + static ManifestTrackedFile forManifestFile() { return new ManifestTrackedFile(); } @@ -550,6 +525,16 @@ public Long replacedRowsCount() { return file.manifestInfo().replacedRowsCount(); } + @Override + public Integer modifiedFilesCount() { + return file.manifestInfo().modifiedFilesCount(); + } + + @Override + public Long modifiedRowsCount() { + return file.manifestInfo().modifiedRowsCount(); + } + @Override public List partitions() { return null; @@ -581,55 +566,61 @@ public ManifestFile copy() { } } - /** Shared base for content-file (DATA / EQUALITY_DELETES) write-direction wrappers. */ - abstract static class ContentTrackedFile> - implements TrackedFile, StructLike { - private final Types.StructType partitionType; + /** Adapts a {@link DataFile} to {@link TrackedFile}. */ + static class DataTrackedFile implements TrackedFile { private final MapBackedContentStats statsWrapper; - private Tracking tracking; - private F file; - private StructProjection partition; + private DataFile file; private ContentStats stats; - ContentTrackedFile(Schema writeSchema) { - Preconditions.checkArgument(writeSchema != null, "Invalid write schema: null"); - Types.StructType statsType = nestedStruct(writeSchema, TrackedFile.CONTENT_STATS_ID); - this.partitionType = nestedStruct(writeSchema, TrackedFile.PARTITION_ID); - this.statsWrapper = statsType != null ? new MapBackedContentStats(statsType) : null; + DataTrackedFile(Schema tableSchema, MetricsConfig metricsConfig) { + this.statsWrapper = new MapBackedContentStats(tableSchema, metricsConfig); } - TrackedFile wrapWithTracking(F newFile, Tracking newTracking) { - Preconditions.checkArgument(newFile != null, "Invalid file: null"); - Preconditions.checkArgument(newTracking != null, "Invalid tracking: null"); - validateContent(newFile); + /** Re-points this adapter at a {@link DataFile} from the public API. Tracking is unset. */ + public TrackedFile wrap(DataFile newFile) { + return wrapFile(newFile, null); + } - if (newFile instanceof TrackedContentFile) { - return ((TrackedContentFile) newFile).file(); + /** + * Re-points this adapter at a {@link ManifestEntry}. Converts the contained data file and the + * entry's tracking fields. + */ + public TrackedFile wrap(ManifestEntry entry) { + Preconditions.checkArgument(entry != null, "Invalid entry: null"); + return wrapFile(entry.file(), trackingFrom(entry, entry.file())); + } + + private TrackedFile wrapFile(DataFile newFile, Tracking newTracking) { + if (newFile instanceof TrackedDataFile tracked) { + return tracked.file(); } + Preconditions.checkArgument(newFile != null, "Invalid file: null"); + Preconditions.checkArgument( + newFile.content() == FileContent.DATA, + "Invalid content for data file: %s", + newFile.content()); + this.file = newFile; - this.partition = partitionType != null ? projectPartition(newFile, partitionType) : null; - this.stats = statsWrapper != null ? statsWrapper.wrap(newFile) : null; + this.stats = hasContentStats(newFile) ? statsWrapper.wrap(newFile) : null; this.tracking = newTracking; return this; } - /** Content-type-specific validation of the wrapped file. */ - abstract void validateContent(F newFile); - - protected F file() { - return file; - } - @Override public Tracking tracking() { return tracking; } + @Override + public FileContent contentType() { + return FileContent.DATA; + } + @Override public int formatVersion() { - return TableMetadata.MIN_FORMAT_VERSION_PARQUET_MANIFESTS; + throw new IllegalStateException("Format version is assigned at write time"); } @Override @@ -654,12 +645,13 @@ public long fileSizeInBytes() { @Override public Integer specId() { + // Files in one manifest may use different specs; this is the spec for this data file only. return file.specId(); } @Override public StructLike partition() { - return partition; + return file.partition(); } @Override @@ -669,12 +661,12 @@ public ContentStats contentStats() { @Override public Integer sortOrderId() { - return null; + return file.sortOrderId(); } @Override public DeletionVector deletionVector() { - return null; + return file.deletionVector(); } @Override @@ -699,95 +691,17 @@ public List equalityIds() { @Override public TrackedFile copy() { - throw new UnsupportedOperationException( - "Reusable content-file wrapper does not support copy(); materialize via a writer instead"); + throw new UnsupportedOperationException("copy is not implemented"); } @Override public TrackedFile copyWithStats(Set requestedColumnIds) { - throw new UnsupportedOperationException( - "Reusable content-file wrapper does not support copyWithStats()"); - } - - @Override - public int size() { - return TRACKED_FILE_FIELD_COUNT; - } - - @Override - public T get(int pos, Class javaClass) { - return javaClass.cast(TrackedFileStruct.getByPos(this, pos)); - } - - @Override - public void set(int pos, T value) { - throw new UnsupportedOperationException( - "Reusable content-file wrapper does not support set()"); + throw new UnsupportedOperationException("copy is not implemented"); } } - /** Wraps a {@link DataFile} as a {@link TrackedFile} row. */ - static class DataTrackedFile extends ContentTrackedFile { - DataTrackedFile(Schema writeSchema) { - super(writeSchema); - } - - /** Re-points this wrapper at {@code newFile} in place. */ - public TrackedFile wrap(DataFile newFile, Tracking tracking) { - return wrapWithTracking(newFile, tracking); - } - - @Override - void validateContent(DataFile newFile) { - Preconditions.checkArgument( - newFile.content() == FileContent.DATA, - "Invalid content for data file: %s", - newFile.content()); - } - - @Override - public FileContent contentType() { - return FileContent.DATA; - } - - @Override - public Integer sortOrderId() { - return file().sortOrderId(); - } - } - - /** Wraps an equality {@link DeleteFile} as a {@link TrackedFile} row. */ - static class EqualityDeleteTrackedFile extends ContentTrackedFile { - EqualityDeleteTrackedFile(Schema writeSchema) { - super(writeSchema); - } - - /** Re-points this wrapper at {@code newFile} in place. */ - public TrackedFile wrap(DeleteFile newFile, Tracking tracking) { - return wrapWithTracking(newFile, tracking); - } - - @Override - void validateContent(DeleteFile newFile) { - Preconditions.checkArgument( - newFile.content() == FileContent.EQUALITY_DELETES, - "Invalid content for delete file: %s", - newFile.content()); - } - - @Override - public FileContent contentType() { - return FileContent.EQUALITY_DELETES; - } - - @Override - public List equalityIds() { - return file().equalityFieldIds(); - } - } - - /** Wraps a {@link ManifestFile} as a v4+ leaf manifest row. */ - static class ManifestTrackedFile implements TrackedFile, StructLike { + /** Adapts a {@link ManifestFile} to {@link TrackedFile}. */ + static class ManifestTrackedFile implements TrackedFile { private final WrappedManifestInfo manifestInfo = new WrappedManifestInfo(); private Tracking tracking; private ManifestFile manifest; @@ -797,49 +711,23 @@ static class ManifestTrackedFile implements TrackedFile, StructLike { ManifestTrackedFile() {} /** - * Re-points this wrapper at {@code newManifest} in place. - * - * @param newManifest manifest file being referenced; must carry an assigned {@code - * sequence_number} and {@code min_sequence_number} - * @param status entry status for the reference - * @param firstRowId first-row-id resolved by the caller for a DATA manifest reference, or null - * for a DELETE manifest reference - * @return this wrapper, or the original {@link TrackedFile} if {@code newManifest} is already - * adapted from a tracked file + * Re-points this adapter at {@code newManifest}. Converts the manifest's own fields only; + * write-time tracking updates are applied by the versioned writer. */ - public TrackedFile wrap(ManifestFile newManifest, EntryStatus status, Long firstRowId) { - Preconditions.checkArgument(newManifest != null, "Invalid manifest file: null"); - Preconditions.checkArgument(status != null, "Invalid status: null"); - - if (newManifest instanceof TrackedManifestFile) { - return ((TrackedManifestFile) newManifest).file(); + public TrackedFile wrap(ManifestFile newManifest) { + if (newManifest instanceof TrackedManifestFile tracked) { + return tracked.file(); } - Long manifestSnapshotId = newManifest.snapshotId(); - Preconditions.checkArgument(manifestSnapshotId != null, "Invalid manifest snapshot id: null"); - long manifestSeq = newManifest.sequenceNumber(); - Preconditions.checkArgument( - manifestSeq != ManifestWriter.UNASSIGNED_SEQ, - "Invalid manifest reference %s: sequence_number is unassigned", - newManifest.path()); - Preconditions.checkArgument( - newManifest.minSequenceNumber() != ManifestWriter.UNASSIGNED_SEQ, - "Invalid manifest reference %s: min_sequence_number is unassigned", - newManifest.path()); - Preconditions.checkArgument( - firstRowId == null || newManifest.content() == ManifestContent.DATA, - "firstRowId is only valid for DATA manifests, but content is %s", - newManifest.content()); + Preconditions.checkArgument(newManifest != null, "Invalid manifest file: null"); this.manifest = newManifest; this.contentType = newManifest.content() == ManifestContent.DATA ? FileContent.DATA_MANIFEST : FileContent.DELETE_MANIFEST; - this.recordCount = resolveRecordCount(newManifest); - this.tracking = - new TrackingStruct( - status, manifestSnapshotId, manifestSeq, manifestSeq, null, firstRowId, null, null); + this.recordCount = manifestRecordCount(newManifest); + this.tracking = trackingFrom(newManifest); this.manifestInfo.wrap(newManifest); return this; } @@ -856,9 +744,7 @@ public FileContent contentType() { @Override public int formatVersion() { - // newly written ManifestFile instances have no persisted version; carry-over rows keep - // theirs via unwrap of TrackedManifestFile - return TableMetadata.MIN_FORMAT_VERSION_PARQUET_MANIFESTS; + return manifest.formatVersion(); } @Override @@ -873,6 +759,7 @@ public FileFormat fileFormat() { @Override public long recordCount() { + // Number of TrackedFile rows stored in the manifest. return recordCount; } @@ -883,6 +770,8 @@ public long fileSizeInBytes() { @Override public Integer specId() { + // Spec the wrapped manifest was written with. Data file entries in a v4 manifest file may + // use different specs. return manifest.partitionSpecId(); } @@ -928,35 +817,17 @@ public List equalityIds() { @Override public TrackedFile copy() { - throw new UnsupportedOperationException( - "Reusable manifest-reference wrapper does not support copy()"); + throw new UnsupportedOperationException("copy is not implemented"); } @Override public TrackedFile copyWithStats(Set requestedColumnIds) { - throw new UnsupportedOperationException( - "Reusable manifest-reference wrapper does not support copyWithStats()"); - } - - @Override - public int size() { - return TRACKED_FILE_FIELD_COUNT; - } - - @Override - public T get(int pos, Class javaClass) { - return javaClass.cast(TrackedFileStruct.getByPos(this, pos)); - } - - @Override - public void set(int pos, T value) { - throw new UnsupportedOperationException( - "Reusable manifest-reference wrapper does not support set()"); + throw new UnsupportedOperationException("copy is not implemented"); } } /** Reusable {@link ManifestInfo} view over a {@link ManifestFile}'s counts. */ - private static class WrappedManifestInfo implements ManifestInfo, StructLike { + private static class WrappedManifestInfo implements ManifestInfo { private ManifestFile manifest; void wrap(ManifestFile newManifest) { @@ -965,52 +836,52 @@ void wrap(ManifestFile newManifest) { @Override public int addedFilesCount() { - return zeroIfNull(manifest.addedFilesCount()); + return manifest.addedFilesCount(); } @Override public int existingFilesCount() { - return zeroIfNull(manifest.existingFilesCount()); + return manifest.existingFilesCount(); } @Override public int deletedFilesCount() { - return zeroIfNull(manifest.deletedFilesCount()); + return manifest.deletedFilesCount(); } @Override public int replacedFilesCount() { - return zeroIfNull(manifest.replacedFilesCount()); + return manifest.replacedFilesCount(); } @Override public int modifiedFilesCount() { - return 0; + return manifest.modifiedFilesCount(); } @Override public long addedRowsCount() { - return zeroIfNull(manifest.addedRowsCount()); + return manifest.addedRowsCount(); } @Override public long existingRowsCount() { - return zeroIfNull(manifest.existingRowsCount()); + return manifest.existingRowsCount(); } @Override public long deletedRowsCount() { - return zeroIfNull(manifest.deletedRowsCount()); + return manifest.deletedRowsCount(); } @Override public long replacedRowsCount() { - return zeroIfNull(manifest.replacedRowsCount()); + return manifest.replacedRowsCount(); } @Override public long modifiedRowsCount() { - return 0L; + return manifest.modifiedRowsCount(); } @Override @@ -1025,72 +896,95 @@ public ManifestBitmap manifestDeletionVector() { @Override public ManifestInfo copy() { - throw new UnsupportedOperationException( - "Reusable manifest-info wrapper does not support copy()"); + throw new UnsupportedOperationException("copy is not implemented"); } + } - @Override - public int size() { - return MANIFEST_INFO_FIELD_COUNT; - } + private static Tracking trackingFrom(ManifestEntry entry, ContentFile file) { + return new TrackingStruct( + entryStatus(entry.status()), + entry.snapshotId(), + entry.dataSequenceNumber(), + entry.fileSequenceNumber(), + null, + file.firstRowId(), + null, + null); + } - @Override - public T get(int pos, Class javaClass) { - Object value = - switch (pos) { - case 0 -> addedFilesCount(); - case 1 -> existingFilesCount(); - case 2 -> deletedFilesCount(); - case 3 -> replacedFilesCount(); - case 4 -> modifiedFilesCount(); - case 5 -> addedRowsCount(); - case 6 -> existingRowsCount(); - case 7 -> deletedRowsCount(); - case 8 -> replacedRowsCount(); - case 9 -> modifiedRowsCount(); - case 10 -> minSequenceNumber(); - case 11 -> - manifestDeletionVector() != null ? manifestDeletionVector().buffer() : null; - default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos); - }; - return javaClass.cast(value); - } + private static Tracking trackingFrom(ManifestFile manifest) { + return new TrackingStruct( + null, + manifest.snapshotId(), + manifest.sequenceNumber(), + manifest.sequenceNumber(), + null, + manifest.firstRowId(), + null, + null); + } - @Override - public void set(int pos, T value) { - throw new UnsupportedOperationException( - "Reusable manifest-info wrapper does not support set()"); - } + private static EntryStatus entryStatus(ManifestEntry.Status status) { + return switch (status) { + case EXISTING -> EntryStatus.EXISTING; + case ADDED -> EntryStatus.ADDED; + case DELETED -> EntryStatus.DELETED; + }; + } - private static int zeroIfNull(Integer value) { - return value != null ? value : 0; - } + private static boolean hasContentStats(ContentFile file) { + return isPresent(file.valueCounts()) + || isPresent(file.nullValueCounts()) + || isPresent(file.nanValueCounts()) + || isPresent(file.avgValueSizes()) + || isPresent(file.lowerBounds()) + || isPresent(file.upperBounds()); + } - private static long zeroIfNull(Long value) { - return value != null ? value : 0L; - } + private static boolean isPresent(Map map) { + return map != null && !map.isEmpty(); } /** - * Returns the struct type of a nested field in a write schema, or null if the schema projects no - * fields for it. A field with no fields to write is unknown in the schema and is reported as - * missing rather than empty, matching {@link TrackedFileStruct}. + * Record count of a manifest is the number of TrackedFile rows it stores: the sum of per-status + * file counts. Missing counts fail rather than producing an incorrect total. */ - private static Types.StructType nestedStruct(Schema writeSchema, int fieldId) { - Type type = writeSchema.findType(fieldId); - Preconditions.checkArgument(type != null, "Invalid write schema: missing field ID %s", fieldId); - if (type.typeId() == Type.TypeID.UNKNOWN) { - return null; - } - + private static long manifestRecordCount(ManifestFile manifest) { + Preconditions.checkNotNull( + manifest.addedFilesCount(), + "Cannot convert manifest %s: missing added files count", + manifest.path()); + Preconditions.checkNotNull( + manifest.existingFilesCount(), + "Cannot convert manifest %s: missing existing files count", + manifest.path()); + Preconditions.checkNotNull( + manifest.deletedFilesCount(), + "Cannot convert manifest %s: missing deleted files count", + manifest.path()); + Preconditions.checkNotNull( + manifest.replacedFilesCount(), + "Cannot convert manifest %s: missing replaced files count", + manifest.path()); + Preconditions.checkNotNull( + manifest.modifiedFilesCount(), + "Cannot convert manifest %s: missing modified files count", + manifest.path()); Preconditions.checkArgument( - type.isStructType(), - "Invalid write schema field %s: expected struct or unknown, got %s", - fieldId, - type); - - Types.StructType struct = type.asStructType(); - return struct.fields().isEmpty() ? null : struct; + manifest.replacedFilesCount() == 0, + "Cannot convert manifest %s: replaced files count must be 0 for v3 or earlier manifests, was %s", + manifest.path(), + manifest.replacedFilesCount()); + Preconditions.checkArgument( + manifest.modifiedFilesCount() == 0, + "Cannot convert manifest %s: modified files count must be 0 for v3 or earlier manifests, was %s", + manifest.path(), + manifest.modifiedFilesCount()); + return (long) manifest.addedFilesCount() + + manifest.existingFilesCount() + + manifest.deletedFilesCount() + + manifest.replacedFilesCount() + + manifest.modifiedFilesCount(); } private static int resolveSpecId(TrackedFile file, Map specsById) { @@ -1111,48 +1005,4 @@ private static int resolveSpecId(TrackedFile file, Map s throw new IllegalArgumentException( "Cannot find unpartitioned spec in specs: " + specsById.keySet()); } - - /** - * Projects the file's per-spec partition tuple into the target partition schema by field ID. - * Fields present in the target but not in the file's spec land as null. - */ - private static StructProjection projectPartition( - ContentFile file, Types.StructType partitionType) { - StructLike partition = file.partition(); - Types.StructType sourceType; - if (partition instanceof PartitionData) { - sourceType = ((PartitionData) partition).getPartitionType(); - } else if (partition == null || partition.size() == 0) { - sourceType = Types.StructType.of(); - } else { - throw new IllegalArgumentException( - String.format( - "Cannot project partition for %s: partition type is unavailable for %s", - file.location(), partition)); - } - - return StructProjection.createAllowMissing(sourceType, partitionType).wrap(partition); - } - - /** Resolves record_count for a manifest-reference row from its per-status file counts. */ - private static long resolveRecordCount(ManifestFile manifest) { - long total = 0L; - if (manifest.addedFilesCount() != null) { - total += manifest.addedFilesCount(); - } - - if (manifest.existingFilesCount() != null) { - total += manifest.existingFilesCount(); - } - - if (manifest.deletedFilesCount() != null) { - total += manifest.deletedFilesCount(); - } - - if (manifest.replacedFilesCount() != null) { - total += manifest.replacedFilesCount(); - } - - return total; - } } diff --git a/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java b/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java index 42f394a81a91..7a333968247d 100644 --- a/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java +++ b/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java @@ -267,31 +267,27 @@ public TrackedFile copyWithStats(Set requestedColumnIds) { @Override protected T internalGet(int pos, Class javaClass) { - return javaClass.cast(getByPos(this, pos)); + return javaClass.cast(getByPos(pos)); } - /** - * Returns the value at the given position in TrackedFile schema field order, shared with the - * write-direction wrappers so that the position-to-field mapping is defined in one place. - */ - static Object getByPos(TrackedFile file, int pos) { + private Object getByPos(int pos) { return switch (pos) { - case 0 -> file.tracking(); - case 1 -> file.contentType() != null ? file.contentType().id() : null; - case 2 -> file.formatVersion(); - case 3 -> file.location(); - case 4 -> file.fileFormat() != null ? file.fileFormat().toString() : null; - case 5 -> file.recordCount(); - case 6 -> file.fileSizeInBytes(); - case 7 -> file.specId(); - case 8 -> file.partition(); - case 9 -> file.contentStats(); - case 10 -> file.sortOrderId(); - case 11 -> file.deletionVector(); - case 12 -> file.manifestInfo(); - case 13 -> file.keyMetadata(); - case 14 -> file.splitOffsets(); - case 15 -> file.equalityIds(); + case 0 -> tracking; + case 1 -> contentType != null ? contentType.id() : null; + case 2 -> formatVersion; + case 3 -> location; + case 4 -> fileFormat != null ? fileFormat.toString() : null; + case 5 -> recordCount; + case 6 -> fileSizeInBytes; + case 7 -> specId; + case 8 -> partitionData; + case 9 -> contentStats; + case 10 -> sortOrderId; + case 11 -> deletionVector; + case 12 -> manifestInfo; + case 13 -> keyMetadata(); + case 14 -> splitOffsets(); + case 15 -> equalityIds(); default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos); }; } diff --git a/core/src/test/java/org/apache/iceberg/TestMapBackedContentStats.java b/core/src/test/java/org/apache/iceberg/TestMapBackedContentStats.java index 3da2398aad3f..8286f719ea22 100644 --- a/core/src/test/java/org/apache/iceberg/TestMapBackedContentStats.java +++ b/core/src/test/java/org/apache/iceberg/TestMapBackedContentStats.java @@ -22,15 +22,10 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.nio.ByteBuffer; -import java.util.Comparator; import java.util.Map; -import java.util.Set; -import org.apache.iceberg.TestHelpers.Row; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; -import org.apache.iceberg.relocated.com.google.common.collect.Sets; -import org.apache.iceberg.types.Comparators; import org.apache.iceberg.types.Conversions; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; @@ -38,7 +33,6 @@ class TestMapBackedContentStats { - // field 5 (flag) is intentionally left out of every stat map to exercise the absent-field path. private static final Schema SCHEMA = new Schema( Types.NestedField.required(1, "id", Types.IntegerType.get()), @@ -47,15 +41,14 @@ class TestMapBackedContentStats { Types.NestedField.optional(4, "name", Types.StringType.get()), Types.NestedField.optional(5, "flag", Types.BooleanType.get())); - private static final MetricsConfig METRICS_CONFIG = - MetricsConfig.from(ImmutableMap.of(), SCHEMA, SortOrder.unsorted()); + private static final MetricsConfig METRICS_CONFIG = MetricsTestUtil.from(Map.of(), SCHEMA); private static final PartitionData EMPTY_PARTITION = new PartitionData(PartitionSpec.unpartitioned().partitionType()); /** - * A file with stats on fields 1-4 (int, float, long, string) but none on field 5 (boolean). Field - * 3 (ts) intentionally has no value_count entry to exercise the absent-count throw path. + * Stats on fields 1-4; field 5 is absent. Field 3 has no value_count entry so the absent-count + * getter throws. */ private static final DataFile FILE_WITH_STATS = dataFile( @@ -73,11 +66,19 @@ class TestMapBackedContentStats { 3, buf(Types.LongType.get(), 999L), 4, buf(Types.StringType.get(), "zzz"))); + @Test + void contentStatsType() { + Types.StructType expected = StatsUtil.statsWriteSchema(SCHEMA, METRICS_CONFIG); + MapBackedContentStats stats = new MapBackedContentStats(SCHEMA, METRICS_CONFIG); + + assertThat(stats.type()).isEqualTo(expected); + assertThat(stats.wrap(FILE_WITH_STATS).type()).isEqualTo(expected); + } + @Test void boundDecodingPerType() { MapBackedContentStats stats = - new MapBackedContentStats(StatsUtil.statsWriteSchema(SCHEMA, METRICS_CONFIG)) - .wrap(FILE_WITH_STATS); + new MapBackedContentStats(SCHEMA, METRICS_CONFIG).wrap(FILE_WITH_STATS); FieldStats id = stats.statsFor(1); assertThat(id.lowerBound()).isInstanceOf(Integer.class).isEqualTo(1); @@ -100,7 +101,6 @@ void boundDecodingPerType() { @Test void missingBoundsDecodeToNull() { - // value counts present but no lower/upper bound entries for the field DataFile file = dataFile( ImmutableMap.of(1, 100L), @@ -108,8 +108,7 @@ void missingBoundsDecodeToNull() { ImmutableMap.of(), ImmutableMap.of(), ImmutableMap.of()); - MapBackedContentStats stats = - new MapBackedContentStats(StatsUtil.statsWriteSchema(SCHEMA, METRICS_CONFIG)).wrap(file); + MapBackedContentStats stats = new MapBackedContentStats(SCHEMA, METRICS_CONFIG).wrap(file); FieldStats id = stats.statsFor(1); assertThat(id.lowerBound()).isNull(); @@ -118,27 +117,24 @@ void missingBoundsDecodeToNull() { } @Test - void countsAndDefaults() { + void countsAndTightBounds() { MapBackedContentStats stats = - new MapBackedContentStats(StatsUtil.statsWriteSchema(SCHEMA, METRICS_CONFIG)) - .wrap(FILE_WITH_STATS); + new MapBackedContentStats(SCHEMA, METRICS_CONFIG).wrap(FILE_WITH_STATS); FieldStats id = stats.statsFor(1); assertThat(id.hasValueCount()).isTrue(); assertThat(id.valueCount()).isEqualTo(100L); - // id tracks no null or nan count; callers must check presence first, since the getters throw - // when the count is absent (as with FieldStatsStruct) assertThat(id.hasNullValueCount()).isFalse(); assertThat(id.hasNanValueCount()).isFalse(); - // primitive unbox of an absent Long count throws the JVM's helpful NPE assertThatThrownBy(id::nullValueCount) .isInstanceOf(NullPointerException.class) .hasMessageContaining("Long.longValue()"); assertThatThrownBy(id::nanValueCount) .isInstanceOf(NullPointerException.class) .hasMessageContaining("Long.longValue()"); - // the wrapper never tracks tight bounds or avg value size on the write path + // ContentFile maps have no tight-bounds flag so the view always reports false. assertThat(id.tightBounds()).isFalse(); + // FILE_WITH_STATS has no avg-size map. assertThat(id.avgValueSizeInBytes()).isNull(); FieldStats score = stats.statsFor(2); @@ -147,7 +143,6 @@ void countsAndDefaults() { assertThat(score.hasNanValueCount()).isTrue(); assertThat(score.nanValueCount()).isEqualTo(3L); - // ts has other stats but no value_count entry — statsFor is non-null, valueCount() must throw FieldStats ts = stats.statsFor(3); assertThat(ts.hasValueCount()).isFalse(); assertThatThrownBy(ts::valueCount) @@ -158,55 +153,11 @@ void countsAndDefaults() { } @Test - void countsOnlyColumnOmitsBounds() { - // configure the float column (score) as counts-only; its stats sub-struct must drop bounds - MetricsConfig countsConfig = - MetricsConfig.from( - ImmutableMap.of("write.metadata.metrics.column.score", "counts"), - SCHEMA, - SortOrder.unsorted()); - MapBackedContentStats stats = - new MapBackedContentStats(StatsUtil.statsWriteSchema(SCHEMA, countsConfig)) - .wrap(FILE_WITH_STATS); - - // counts mode drops lower_bound/upper_bound/tight_bounds; only the counts remain - FieldStats score = stats.statsFor(2); - assertThat(score.type().fields()) - .extracting(Types.NestedField::name) - .containsExactly("value_count", "null_value_count", "nan_value_count"); - - // a column left at the default mode still carries bounds - assertThat(stats.statsFor(1).type().fields()) - .extracting(Types.NestedField::name) - .contains("lower_bound", "upper_bound"); - } - - @Test - void countsOnlyStructLike() { - // the pruned counts-only struct must expose its counts in the right positions via StructLike - MetricsConfig countsConfig = - MetricsConfig.from( - ImmutableMap.of("write.metadata.metrics.column.score", "counts"), - SCHEMA, - SortOrder.unsorted()); + void fieldWithoutStatsIsExcluded() { MapBackedContentStats stats = - new MapBackedContentStats(StatsUtil.statsWriteSchema(SCHEMA, countsConfig)) - .wrap(FILE_WITH_STATS); + new MapBackedContentStats(SCHEMA, METRICS_CONFIG).wrap(FILE_WITH_STATS); - FieldStats score = stats.statsFor(2); - // struct order: value_count, null_value_count, nan_value_count (bounds pruned in counts mode) - Row expected = Row.of(100L, 5L, 3L); - Comparator comparator = Comparators.forType(score.type()); - assertThat(comparator.compare((StructLike) score, expected)).isEqualTo(0); - } - - @Test - void absentFieldHasNoStats() { - MapBackedContentStats stats = - new MapBackedContentStats(StatsUtil.statsWriteSchema(SCHEMA, METRICS_CONFIG)) - .wrap(FILE_WITH_STATS); - - // field 5 (flag) has no entry in any stat map + assertThat(stats.type().field(StatsUtil.toBaseId(5))).isNotNull(); assertThat(stats.statsFor(5)).isNull(); assertThat(stats.fieldStats()) .extracting(FieldStats::fieldId) @@ -214,106 +165,49 @@ void absentFieldHasNoStats() { } @Test - void typeMatchesStatsReadSchema() { - MapBackedContentStats stats = - new MapBackedContentStats(StatsUtil.statsWriteSchema(SCHEMA, METRICS_CONFIG)); - assertThat(stats.type()) - .isEqualTo(StatsUtil.statsReadSchema(SCHEMA, ImmutableList.of(1, 2, 3, 4, 5))); - } - - @Test - void defaultStructLike() { - // the default (bound-bearing) struct for an optional float exposes all six fields via - // StructLike - MapBackedContentStats stats = - new MapBackedContentStats(StatsUtil.statsWriteSchema(SCHEMA, METRICS_CONFIG)) - .wrap(FILE_WITH_STATS); - - FieldStats score = stats.statsFor(2); - // struct order: lower_bound, upper_bound, tight_bounds, value_count, null_value_count, - // nan_value_count - Row expected = Row.of(1.5f, 9.5f, false, 100L, 5L, 3L); - Comparator comparator = Comparators.forType(score.type()); - assertThat(comparator.compare((StructLike) score, expected)).isEqualTo(0); - } + void statsForUnknownFieldIdIsRejected() { + DataFile file = + dataFile( + ImmutableMap.of(99, 1L), + ImmutableMap.of(), + ImmutableMap.of(), + ImmutableMap.of(), + ImmutableMap.of()); - @Test - void contentStructLikeGetReturnsChildrenOrNull() { - MapBackedContentStats stats = - new MapBackedContentStats(StatsUtil.statsWriteSchema(SCHEMA, METRICS_CONFIG)) - .wrap(FILE_WITH_STATS); - - Set nullFieldIds = Sets.newHashSet(); - for (int pos = 0; pos < stats.size(); pos += 1) { - FieldStats child = stats.get(pos, FieldStats.class); - int fieldId = StatsUtil.toFieldId(stats.type().fields().get(pos).fieldId()); - if (child == null) { - nullFieldIds.add(fieldId); - } else { - assertThat(child.fieldId()).isEqualTo(fieldId); - assertThat(stats.statsFor(fieldId)).isSameAs(child); - } - } - - // only field 5 (flag) lacks stats - assertThat(nullFieldIds).containsExactly(5); + assertThatThrownBy( + () -> new MapBackedContentStats(SCHEMA, METRICS_CONFIG).wrap(file).statsFor(99)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage( + "Cannot convert stats for field ID 99: unknown, not a scalar, or not in metrics config"); } @Test void copyNotSupported() { MapBackedContentStats stats = - new MapBackedContentStats(StatsUtil.statsWriteSchema(SCHEMA, METRICS_CONFIG)) - .wrap(FILE_WITH_STATS); + new MapBackedContentStats(SCHEMA, METRICS_CONFIG).wrap(FILE_WITH_STATS); - // the reusable wrapper is serialized directly; snapshots must be materialized via a writer assertThatThrownBy(stats::copy) .isInstanceOf(UnsupportedOperationException.class) - .hasMessageContaining("does not support copy()"); + .hasMessage("copy is not implemented"); assertThatThrownBy(() -> stats.copy(ImmutableSet.of(1))) .isInstanceOf(UnsupportedOperationException.class) - .hasMessageContaining("does not support copy()"); - } - - @Test - void setNotSupported() { - MapBackedContentStats stats = - new MapBackedContentStats(StatsUtil.statsWriteSchema(SCHEMA, METRICS_CONFIG)) - .wrap(FILE_WITH_STATS); - - assertThatThrownBy(() -> stats.set(0, null)) - .isInstanceOf(UnsupportedOperationException.class) - .hasMessageContaining("does not support set()"); + .hasMessage("copy is not implemented"); } @Test void fieldStatsCopyNotSupported() { - // the reusable field wrapper is serialized directly; snapshots must be materialized via a - // writer MapBackedContentStats stats = - new MapBackedContentStats(StatsUtil.statsWriteSchema(SCHEMA, METRICS_CONFIG)) - .wrap(FILE_WITH_STATS); + new MapBackedContentStats(SCHEMA, METRICS_CONFIG).wrap(FILE_WITH_STATS); assertThatThrownBy(stats.statsFor(1)::copy) .isInstanceOf(UnsupportedOperationException.class) - .hasMessageContaining("does not support copy()"); - } - - @Test - void fieldStatsSetNotSupported() { - MapBackedContentStats stats = - new MapBackedContentStats(StatsUtil.statsWriteSchema(SCHEMA, METRICS_CONFIG)) - .wrap(FILE_WITH_STATS); - - assertThatThrownBy(() -> ((StructLike) stats.statsFor(1)).set(0, 5)) - .isInstanceOf(UnsupportedOperationException.class) - .hasMessageContaining("does not support set()"); + .hasMessage("copy is not implemented"); } @Test void reuseRebindsBounds() { - MapBackedContentStats stats = - new MapBackedContentStats(StatsUtil.statsWriteSchema(SCHEMA, METRICS_CONFIG)); + MapBackedContentStats stats = new MapBackedContentStats(SCHEMA, METRICS_CONFIG); stats.wrap(FILE_WITH_STATS); assertThat(stats.statsFor(1).lowerBound()).isEqualTo(1); @@ -330,7 +224,6 @@ void reuseRebindsBounds() { assertThat(stats.statsFor(1).lowerBound()).isEqualTo(500); assertThat(stats.statsFor(1).upperBound()).isEqualTo(5000); assertThat(stats.statsFor(1).valueCount()).isEqualTo(50L); - // fields that had stats in file1 but not file2 are now absent assertThat(stats.statsFor(2)).isNull(); } diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java index c64794d5eab7..28340a77e89b 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java @@ -30,7 +30,6 @@ import org.apache.iceberg.mumbling.MumblingTestUtil; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; -import org.apache.iceberg.types.Comparators; import org.apache.iceberg.types.Conversions; import org.apache.iceberg.types.Types; import org.junit.jupiter.api.Test; @@ -48,6 +47,7 @@ class TestTrackedFileAdapters { // Tracking values that the delegation tests validate. private static final long MANIFEST_POS = 3L; + private static final long SNAPSHOT_ID = 42L; private static final long DATA_SEQUENCE_NUMBER = 10L; private static final long FILE_SEQUENCE_NUMBER = 11L; private static final long FIRST_ROW_ID = 1000L; @@ -56,6 +56,8 @@ class TestTrackedFileAdapters { private static final Map UNPARTITIONED = ImmutableMap.of(UNPARTITIONED_SPEC_ID, PartitionSpec.unpartitioned()); + private static final ByteBuffer KEY_METADATA = ByteBuffer.wrap(new byte[] {1, 2, 3}); + private static final Schema PARTITION_SCHEMA = new Schema(Types.NestedField.required(1, "category", Types.StringType.get())); private static final int PARTITIONED_SPEC_ID = 1; @@ -72,30 +74,32 @@ class TestTrackedFileAdapters { private static final Schema TABLE_SCHEMA = new Schema( - optional(1, "id", Types.IntegerType.get()), optional(2, "score", Types.FloatType.get())); + optional(1, "id", Types.IntegerType.get()), + optional(2, "score", Types.FloatType.get()), + optional(3, "geom", Types.GeometryType.crs84())); + private static final MetricsConfig METRICS_CONFIG = MetricsTestUtil.from(Map.of(), TABLE_SCHEMA); private static final Types.StructType CONTENT_STATS_TYPE = - StatsUtil.statsReadSchema(TABLE_SCHEMA, ImmutableList.of(1, 2)); + StatsUtil.statsReadSchema(TABLE_SCHEMA, ImmutableList.of(1, 2, 3)); private static final FieldStats ID_STATS = StatsTestUtil.mockFieldStats( CONTENT_STATS_TYPE.fieldType("id").asStructType(), 1, 1, 1000, 100L, 5L, null); private static final FieldStats SCORE_STATS = StatsTestUtil.mockFieldStats( CONTENT_STATS_TYPE.fieldType("score").asStructType(), 2, 1.0f, 100.0f, 100L, 10L, 3L); + private static final FieldStats GEOM_STATS = + StatsTestUtil.mockFieldStats( + CONTENT_STATS_TYPE.fieldType("geom").asStructType(), 3, null, null, 100L, 20L, null, 12); private static final ContentStatsStruct CONTENT_STATS = new ContentStatsStruct(CONTENT_STATS_TYPE); static { CONTENT_STATS.setStats(1, ID_STATS); CONTENT_STATS.setStats(2, SCORE_STATS); + CONTENT_STATS.setStats(3, GEOM_STATS); } - private static final MetricsConfig METRICS_CONFIG = - MetricsConfig.from(ImmutableMap.of(), TABLE_SCHEMA, SortOrder.unsorted()); private static final PartitionSpec UNPARTITIONED_SPEC = PartitionSpec.unpartitioned(); private static final Types.StructType PARTITION_TYPE = UNPARTITIONED_SPEC.partitionType(); - private static final Schema WRITE_SCHEMA = - TrackedFile.schema(PARTITION_TYPE, StatsUtil.statsWriteSchema(TABLE_SCHEMA, METRICS_CONFIG)); - private static final long SNAPSHOT_ID = 42L; private static final Tracking MANIFEST_TRACKING = new TrackingStruct( @@ -109,7 +113,6 @@ class TestTrackedFileAdapters { null); // replacedPositions private static final ByteBuffer MANIFEST_KEY_METADATA = ByteBuffer.wrap(new byte[] {7, 8, 9}); - private static final byte[] MANIFEST_DV = MumblingTestUtil.onlyFirstBitSetBytes(); private static final ManifestInfo MANIFEST_INFO = ManifestInfoStruct.builder() @@ -117,22 +120,16 @@ class TestTrackedFileAdapters { .existingFilesCount(5) .deletedFilesCount(2) .replacedFilesCount(4) - .modifiedFilesCount(0) + .modifiedFilesCount(1) .addedRowsCount(300L) .existingRowsCount(500L) .deletedRowsCount(200L) .replacedRowsCount(40L) - .modifiedRowsCount(0L) + .modifiedRowsCount(10L) .minSequenceNumber(7L) .dv(ByteBuffer.wrap(MumblingTestUtil.onlyFirstBitSetBytes())) .build(); - private static final long DATA_SEQ = 7L; - private static final long FILE_SEQ = 11L; - private static final String DATA_PATH = "s3://bucket/data/file.parquet"; - private static final String DELETE_PATH = "s3://bucket/data/eq-delete.parquet"; - private static final String MANIFEST_PATH = "s3://bucket/metadata/manifest.parquet"; - private static final Metrics METRICS_WITH_BOUNDS = new Metrics( 100L, @@ -143,18 +140,10 @@ class TestTrackedFileAdapters { ImmutableMap.of(1, Conversions.toByteBuffer(Types.IntegerType.get(), 1)), ImmutableMap.of(1, Conversions.toByteBuffer(Types.IntegerType.get(), 1000))); - private static final Tracking ADDED_TRACKING = TrackingBuilder.added(SNAPSHOT_ID).build(); - private static final Tracking EXISTING_TRACKING = - new TrackingStruct( - EntryStatus.EXISTING, SNAPSHOT_ID, DATA_SEQ, FILE_SEQ, null, null, null, null); - private static final Tracking DELETED_TRACKING = - new TrackingStruct( - EntryStatus.DELETED, SNAPSHOT_ID, DATA_SEQ, FILE_SEQ, null, null, null, null); - private static final DataFile DATA_FILE = new GenericDataFile( UNPARTITIONED_SPEC.specId(), - DATA_PATH, + DATA_FILE_LOCATION, FileFormat.PARQUET, PartitionData.EMPTY, 1024L, @@ -166,7 +155,7 @@ class TestTrackedFileAdapters { private static final DataFile DATA_FILE_WITH_METRICS = new GenericDataFile( UNPARTITIONED_SPEC.specId(), - DATA_PATH, + DATA_FILE_LOCATION, FileFormat.PARQUET, PartitionData.EMPTY, 1024L, @@ -175,22 +164,6 @@ class TestTrackedFileAdapters { ImmutableList.of(0L), null, null); - private static final DeleteFile EQUALITY_DELETE_FILE = - new GenericDeleteFile( - UNPARTITIONED_SPEC.specId(), - FileContent.EQUALITY_DELETES, - DELETE_PATH, - FileFormat.PARQUET, - PartitionData.EMPTY, - 512L, - new Metrics(50L, null, null, null, null), - new int[] {1}, - null, - null, - null, - null, - null, - null); @Test void dataFileAdapterDelegation() { @@ -207,6 +180,7 @@ void dataFileAdapterDelegation() { tracking.set(MANIFEST_LOCATION_ORDINAL, MANIFEST_LOCATION); tracking.set(MANIFEST_POSITION_ORDINAL, MANIFEST_POS); + DeletionVector dv = mock(DeletionVector.class); TrackedFile file = new TrackedFileStruct( tracking, @@ -220,7 +194,7 @@ void dataFileAdapterDelegation() { PARTITION, CONTENT_STATS, 3, - null, + dv, null, ByteBuffer.wrap(new byte[] {1, 2, 3}), ImmutableList.of(50L, 100L), @@ -243,11 +217,15 @@ void dataFileAdapterDelegation() { assertThat(dataFile.keyMetadata()).isEqualTo(ByteBuffer.wrap(new byte[] {1, 2, 3})); assertThat(dataFile.splitOffsets()).containsExactly(50L, 100L); assertThat(dataFile.manifestLocation()).isEqualTo(MANIFEST_LOCATION); + assertThat(dataFile.deletionVector()).isSameAs(dv); assertThat(dataFile.equalityFieldIds()).isNull(); assertThat(dataFile.columnSizes()).isNull(); - assertThat(dataFile.valueCounts()).containsOnly(Map.entry(1, 100L), Map.entry(2, 100L)); - assertThat(dataFile.nullValueCounts()).containsOnly(Map.entry(1, 5L), Map.entry(2, 10L)); + assertThat(dataFile.valueCounts()) + .containsOnly(Map.entry(1, 100L), Map.entry(2, 100L), Map.entry(3, 100L)); + assertThat(dataFile.nullValueCounts()) + .containsOnly(Map.entry(1, 5L), Map.entry(2, 10L), Map.entry(3, 20L)); assertThat(dataFile.nanValueCounts()).containsOnly(Map.entry(2, 3L)); + assertThat(dataFile.avgValueSizes()).containsOnly(Map.entry(3, 12)); assertThat(dataFile.lowerBounds()) .containsOnly( Map.entry(1, Conversions.toByteBuffer(Types.IntegerType.get(), 1)), @@ -322,9 +300,12 @@ void equalityDeleteFileAdapterDelegation() { assertThat(deleteFile.manifestLocation()).isEqualTo(MANIFEST_LOCATION); assertThat(deleteFile.equalityFieldIds()).containsExactly(1, 2, 3); assertThat(deleteFile.columnSizes()).isNull(); - assertThat(deleteFile.valueCounts()).containsOnly(Map.entry(1, 100L), Map.entry(2, 100L)); - assertThat(deleteFile.nullValueCounts()).containsOnly(Map.entry(1, 5L), Map.entry(2, 10L)); + assertThat(deleteFile.valueCounts()) + .containsOnly(Map.entry(1, 100L), Map.entry(2, 100L), Map.entry(3, 100L)); + assertThat(deleteFile.nullValueCounts()) + .containsOnly(Map.entry(1, 5L), Map.entry(2, 10L), Map.entry(3, 20L)); assertThat(deleteFile.nanValueCounts()).containsOnly(Map.entry(2, 3L)); + assertThat(deleteFile.avgValueSizes()).containsOnly(Map.entry(3, 12)); assertThat(deleteFile.lowerBounds()) .containsOnly( Map.entry(1, Conversions.toByteBuffer(Types.IntegerType.get(), 1)), @@ -353,6 +334,7 @@ void dvDeleteFileAdapterDelegation() { .offset(128L) .sizeInBytes(256L) .cardinality(10L) + .keyMetadata(KEY_METADATA) .build(); TrackingStruct tracking = @@ -396,6 +378,7 @@ void dvDeleteFileAdapterDelegation() { assertThat(dvFile.recordCount()).isEqualTo(dv.cardinality()); assertThat(dvFile.contentOffset()).isEqualTo(dv.offset()); assertThat(dvFile.contentSizeInBytes()).isEqualTo(dv.sizeInBytes()); + assertThat(dvFile.keyMetadata()).isEqualTo(KEY_METADATA); // fileSizeInBytes reports the DV blob size, not the full Puffin file size. assertThat(dvFile.fileSizeInBytes()).isEqualTo(dv.sizeInBytes()); // referencedDataFile is delegated to the tracked data file's location. @@ -412,7 +395,6 @@ void dvDeleteFileAdapterDelegation() { // fields that are null for DVs assertThat(dvFile.sortOrderId()).isNull(); assertThat(dvFile.firstRowId()).isNull(); - assertThat(dvFile.keyMetadata()).isNull(); assertThat(dvFile.splitOffsets()).isNull(); assertThat(dvFile.equalityFieldIds()).isNull(); assertThat(dvFile.columnSizes()).isNull(); @@ -498,7 +480,7 @@ private static TrackedFile serializableDataEntry() { .offset(128L) .sizeInBytes(256L) .cardinality(10L) - .keyMetadata(MANIFEST_KEY_METADATA) + .keyMetadata(KEY_METADATA) .build(); return new TrackedFileStruct( @@ -515,7 +497,7 @@ private static TrackedFile serializableDataEntry() { 3, // sortOrderId dv, // deletionVector null, // manifestInfo - MANIFEST_KEY_METADATA, + KEY_METADATA, ImmutableList.of(50L, 100L), // splitOffsets null); // equalityIds } @@ -581,6 +563,8 @@ void manifestFileAdapterDelegation(FileContent contentType) { assertThat(manifest.deletedRowsCount()).isEqualTo(MANIFEST_INFO.deletedRowsCount()); assertThat(manifest.replacedFilesCount()).isEqualTo(MANIFEST_INFO.replacedFilesCount()); assertThat(manifest.replacedRowsCount()).isEqualTo(MANIFEST_INFO.replacedRowsCount()); + assertThat(manifest.modifiedFilesCount()).isEqualTo(MANIFEST_INFO.modifiedFilesCount()); + assertThat(manifest.modifiedRowsCount()).isEqualTo(MANIFEST_INFO.modifiedRowsCount()); assertThat(manifest.firstRowId()).isEqualTo(FIRST_ROW_ID); assertThat(manifest.keyMetadata()).isEqualTo(MANIFEST_KEY_METADATA); assertThat(manifest.manifestDeletionVector().buffer()) @@ -592,6 +576,13 @@ void manifestFileAdapterDelegation(FileContent contentType) { .hasMessage("v4 manifests are not bound to a single partition spec"); } + @Test + void manifestFileAdapterKeepsZeroFormatVersion() { + TrackedFile original = dummyTrackedFile(FileContent.DATA_MANIFEST, 0); + + assertThat(TrackedFileAdapters.asManifestFile(original).formatVersion()).isZero(); + } + @Test void manifestFileAdapterCopy() { TrackedFile file = Mockito.mock(TrackedFile.class); @@ -732,72 +723,75 @@ void unknownSpecIdThrows() { } @Test - void manifestFileAdapterDelegatesFormatVersion() { - TrackedFile original = dummyTrackedFile(FileContent.DATA_MANIFEST, 0); + void dataTrackedFileAdapterHasNoTracking() { + TrackedFile result = + TrackedFileAdapters.forDataFile(TABLE_SCHEMA, METRICS_CONFIG).wrap(DATA_FILE); - assertThat(TrackedFileAdapters.asManifestFile(original).formatVersion()).isZero(); + assertThat(result.tracking()).isNull(); + assertWrappedDataFileMatchesFileFields(result, DATA_FILE); + assertThatThrownBy(result::formatVersion) + .isInstanceOf(IllegalStateException.class) + .hasMessage("Format version is assigned at write time"); } @Test - void testDataFileWrapperAdded() { - DataFile file = DATA_FILE; - TrackedFileAdapters.DataTrackedFile wrapper = TrackedFileAdapters.forDataFile(WRITE_SCHEMA); - TrackedFile result = wrapper.wrap(file, ADDED_TRACKING); + void dataTrackedFileAdapterFromAddedManifestEntry() { + TrackedFile result = + TrackedFileAdapters.forDataFile(TABLE_SCHEMA, METRICS_CONFIG).wrap(addedEntry(DATA_FILE)); assertThat(result.tracking().status()).isEqualTo(EntryStatus.ADDED); assertThat(result.tracking().snapshotId()).isEqualTo(SNAPSHOT_ID); assertThat(result.tracking().dataSequenceNumber()).isNull(); assertThat(result.tracking().fileSequenceNumber()).isNull(); assertThat(result.tracking().firstRowId()).isNull(); - assertWriteDataFields(result, file, FORMAT_VERSION_V4); + assertWrappedDataFileMatchesFileFields(result, DATA_FILE); } @Test - void testDataFileWrapperAddedWithNullSnapshotId() { - // Staged-write ADDED entry (FastAppend writes the manifest before the commit snapshot is - // assigned). Snapshot ID is inherited at read time from the enclosing root manifest's - // added_snapshot_id. - DataFile file = DATA_FILE; - Tracking tracking = - new TrackingStruct(EntryStatus.ADDED, null, null, null, null, null, null, null); - TrackedFileAdapters.DataTrackedFile wrapper = TrackedFileAdapters.forDataFile(WRITE_SCHEMA); - TrackedFile result = wrapper.wrap(file, tracking); + void dataTrackedFileAdapterFromAddedManifestEntryWithNullSnapshotId() { + TrackedFile result = + TrackedFileAdapters.forDataFile(TABLE_SCHEMA, METRICS_CONFIG) + .wrap(newEntry().wrapAppend(null, DATA_FILE)); assertThat(result.tracking().status()).isEqualTo(EntryStatus.ADDED); assertThat(result.tracking().snapshotId()).isNull(); - assertWriteDataFields(result, file, FORMAT_VERSION_V4); + assertWrappedDataFileMatchesFileFields(result, DATA_FILE); } @Test - void testDataFileWrapperExisting() { - DataFile file = DATA_FILE; - TrackedFileAdapters.DataTrackedFile wrapper = TrackedFileAdapters.forDataFile(WRITE_SCHEMA); - TrackedFile result = wrapper.wrap(file, EXISTING_TRACKING); + void dataTrackedFileAdapterFromExistingManifestEntry() { + TrackedFile result = + TrackedFileAdapters.forDataFile(TABLE_SCHEMA, METRICS_CONFIG) + .wrap(existingEntry(DATA_FILE)); assertThat(result.tracking().status()).isEqualTo(EntryStatus.EXISTING); assertThat(result.tracking().snapshotId()).isEqualTo(SNAPSHOT_ID); - assertThat(result.tracking().dataSequenceNumber()).isEqualTo(DATA_SEQ); - assertThat(result.tracking().fileSequenceNumber()).isEqualTo(FILE_SEQ); - assertWriteDataFields(result, file, FORMAT_VERSION_V4); + assertThat(result.tracking().dataSequenceNumber()).isEqualTo(DATA_SEQUENCE_NUMBER); + assertThat(result.tracking().fileSequenceNumber()).isEqualTo(FILE_SEQUENCE_NUMBER); + assertWrappedDataFileMatchesFileFields(result, DATA_FILE); } @Test - void testDataFileWrapperDeleted() { - DataFile file = DATA_FILE; - TrackedFileAdapters.DataTrackedFile wrapper = TrackedFileAdapters.forDataFile(WRITE_SCHEMA); - TrackedFile result = wrapper.wrap(file, DELETED_TRACKING); + void dataTrackedFileAdapterFromDeletedManifestEntry() { + TrackedFile result = + TrackedFileAdapters.forDataFile(TABLE_SCHEMA, METRICS_CONFIG).wrap(deletedEntry(DATA_FILE)); assertThat(result.tracking().status()).isEqualTo(EntryStatus.DELETED); assertThat(result.tracking().snapshotId()).isEqualTo(SNAPSHOT_ID); - assertThat(result.tracking().dataSequenceNumber()).isEqualTo(DATA_SEQ); - assertThat(result.tracking().fileSequenceNumber()).isEqualTo(FILE_SEQ); + assertThat(result.tracking().dataSequenceNumber()).isEqualTo(DATA_SEQUENCE_NUMBER); + assertThat(result.tracking().fileSequenceNumber()).isEqualTo(FILE_SEQUENCE_NUMBER); } @Test - void testDataFileWrapperReuse() { - // Same wrapper instance, two different files. The second wrap() call must not leak state from - // the first: this matches how V4Writer will hold one wrapper and rebind per row. - DataFile file1 = DATA_FILE; + void dataTrackedFileAdapterReuse() { + TrackedFileAdapters.DataTrackedFile adapter = + TrackedFileAdapters.forDataFile(TABLE_SCHEMA, METRICS_CONFIG); + + adapter.wrap(DATA_FILE); + assertThat(adapter.location()).isEqualTo(DATA_FILE_LOCATION); + assertThat(adapter.recordCount()).isEqualTo(100L); + assertThat(adapter.tracking()).isNull(); + DataFile file2 = new GenericDataFile( UNPARTITIONED_SPEC.specId(), @@ -810,117 +804,27 @@ void testDataFileWrapperReuse() { ImmutableList.of(0L), null, null); - - TrackedFileAdapters.DataTrackedFile wrapper = TrackedFileAdapters.forDataFile(WRITE_SCHEMA); - - wrapper.wrap(file1, ADDED_TRACKING); - assertThat(wrapper.location()).isEqualTo(DATA_PATH); - assertThat(wrapper.recordCount()).isEqualTo(100L); - - wrapper.wrap(file2, EXISTING_TRACKING); - assertThat(wrapper.location()).isEqualTo("s3://bucket/data/file2.parquet"); - assertThat(wrapper.recordCount()).isEqualTo(200L); - assertThat(wrapper.tracking().status()).isEqualTo(EntryStatus.EXISTING); - assertThat(wrapper.fileSizeInBytes()).isEqualTo(2048L); - - // No state leaks from file1 after rebinding to file2. - assertThat(wrapper.location()).isNotEqualTo(file1.location()); - assertThat(wrapper.recordCount()).isNotEqualTo(file1.recordCount()); - assertThat(wrapper.fileSizeInBytes()).isNotEqualTo(file1.fileSizeInBytes()); - assertThat(wrapper.tracking().status()).isNotEqualTo(EntryStatus.ADDED); + adapter.wrap(existingEntry(file2)); + assertThat(adapter.location()).isEqualTo("s3://bucket/data/file2.parquet"); + assertThat(adapter.recordCount()).isEqualTo(200L); + assertThat(adapter.tracking().status()).isEqualTo(EntryStatus.EXISTING); + assertThat(adapter.fileSizeInBytes()).isEqualTo(2048L); + assertThat(adapter.location()).isNotEqualTo(DATA_FILE.location()); } @Test - void testDataFileWrapperRejectsNullFile() { - TrackedFileAdapters.DataTrackedFile wrapper = TrackedFileAdapters.forDataFile(WRITE_SCHEMA); - assertThatThrownBy(() -> wrapper.wrap(null, ADDED_TRACKING)) + void dataTrackedFileAdapterRejectsNullFile() { + TrackedFileAdapters.DataTrackedFile adapter = + TrackedFileAdapters.forDataFile(TABLE_SCHEMA, METRICS_CONFIG); + assertThatThrownBy(() -> adapter.wrap((DataFile) null)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("Invalid file: null"); } @Test - void testDataFileWrapperRejectsNullTracking() { - DataFile file = DATA_FILE; - TrackedFileAdapters.DataTrackedFile wrapper = TrackedFileAdapters.forDataFile(WRITE_SCHEMA); - assertThatThrownBy(() -> wrapper.wrap(file, null)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("Invalid tracking: null"); - } - - @Test - void testEqualityDeleteFileWrapper() { - DeleteFile file = EQUALITY_DELETE_FILE; - TrackedFileAdapters.EqualityDeleteTrackedFile wrapper = - TrackedFileAdapters.forEqualityDeleteFile(WRITE_SCHEMA); - TrackedFile result = wrapper.wrap(file, ADDED_TRACKING); - - assertThat(result.contentType()).isEqualTo(FileContent.EQUALITY_DELETES); - assertThat(result.tracking().status()).isEqualTo(EntryStatus.ADDED); - assertThat(result.tracking().firstRowId()).isNull(); - assertThat(result.equalityIds()).containsExactly(1); - assertThat(result.sortOrderId()).isNull(); - assertThat(result.location()).isEqualTo(DELETE_PATH); - } - - @Test - void testEqualityDeleteFileWrapperRejectsV3Dv() { - // A deletion vector (in Puffin) is not an equality delete: in v4 it is carried - // inline on its data file's TrackedFile row rather than as a delete manifest entry. - DeleteFile dv = - new GenericDeleteFile( - UNPARTITIONED_SPEC.specId(), - FileContent.POSITION_DELETES, - DELETE_PATH, - FileFormat.PUFFIN, - PartitionData.EMPTY, - 512L, - new Metrics(10L, null, null, null, null), - null, - null, - null, - null, - DATA_PATH, - 0L, - 512L); - TrackedFileAdapters.EqualityDeleteTrackedFile wrapper = - TrackedFileAdapters.forEqualityDeleteFile(WRITE_SCHEMA); - assertThatThrownBy(() -> wrapper.wrap(dv, ADDED_TRACKING)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("Invalid content for delete file: POSITION_DELETES"); - } - - @Test - void testEqualityDeleteFileWrapperRejectsV2PositionDeletes() { - // A v2 position delete file (in Parquet) is not an equality delete and has no v4 leaf - // representation, so the equality-delete wrapper rejects it. - DeleteFile positionDeletes = - new GenericDeleteFile( - UNPARTITIONED_SPEC.specId(), - FileContent.POSITION_DELETES, - DELETE_PATH, - FileFormat.PARQUET, - PartitionData.EMPTY, - 512L, - new Metrics(10L, null, null, null, null), - null, - null, - null, - null, - null, - null, - null); - TrackedFileAdapters.EqualityDeleteTrackedFile wrapper = - TrackedFileAdapters.forEqualityDeleteFile(WRITE_SCHEMA); - assertThatThrownBy(() -> wrapper.wrap(positionDeletes, ADDED_TRACKING)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("Invalid content for delete file: POSITION_DELETES"); - } - - @Test - void testDataFileWrapperContentStats() { - DataFile file = DATA_FILE_WITH_METRICS; - TrackedFileAdapters.DataTrackedFile wrapper = TrackedFileAdapters.forDataFile(WRITE_SCHEMA); - TrackedFile result = wrapper.wrap(file, ADDED_TRACKING); + void dataTrackedFileAdapterContentStats() { + TrackedFile result = + TrackedFileAdapters.forDataFile(TABLE_SCHEMA, METRICS_CONFIG).wrap(DATA_FILE_WITH_METRICS); ContentStats stats = result.contentStats(); assertThat(stats).isNotNull(); @@ -933,92 +837,46 @@ void testDataFileWrapperContentStats() { } @Test - void dataFileWrapperWithoutMetricsHasNoContentStats() { - // Disabling metrics for every column produces an empty stats struct, which TrackedFile.schema - // projects as unknown. The wrapper reports missing stats rather than an empty stats struct. - MetricsConfig noMetrics = - MetricsConfig.from( - ImmutableMap.of( - TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "id", "none", - TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "score", "none"), - TABLE_SCHEMA, - SortOrder.unsorted()); - Schema writeSchema = - TrackedFile.schema(PARTITION_TYPE, StatsUtil.statsWriteSchema(TABLE_SCHEMA, noMetrics)); - assertThat(writeSchema.findType(TrackedFile.CONTENT_STATS_ID)) - .isInstanceOf(Types.UnknownType.class); - - TrackedFileAdapters.DataTrackedFile wrapper = TrackedFileAdapters.forDataFile(writeSchema); - TrackedFile result = wrapper.wrap(DATA_FILE_WITH_METRICS, ADDED_TRACKING); + void dataTrackedFileAdapterWithoutMetricsHasNoContentStats() { + TrackedFile result = + TrackedFileAdapters.forDataFile(TABLE_SCHEMA, METRICS_CONFIG).wrap(DATA_FILE); assertThat(result.contentStats()).isNull(); } @Test - void dataFileWrapperUnpartitionedHasNoPartition() { - // An unpartitioned spec produces an empty partition struct, which TrackedFile.schema projects - // as unknown. An absent partition is null, as it is in TrackedFileStruct. - assertThat(WRITE_SCHEMA.findType(TrackedFile.PARTITION_ID)) - .isInstanceOf(Types.UnknownType.class); - - TrackedFileAdapters.DataTrackedFile wrapper = TrackedFileAdapters.forDataFile(WRITE_SCHEMA); - TrackedFile result = wrapper.wrap(DATA_FILE, ADDED_TRACKING); - - assertThat(result.partition()).isNull(); - // the read direction normalizes the missing partition back to an empty tuple - assertThat(TrackedFileAdapters.asDataFile(result, UNPARTITIONED).partition()) - .isEqualTo(PartitionData.EMPTY); - } - - @Test - void testDataFileWrapperPartitionProjection() { - // Multi-spec carry-over: the writer spec has one field but the union partition type carries an - // additional field from another live spec. The wrapper must place the writer's value at the - // field-ID position and leave the extra union field null. - PartitionSpec writerSpec = PartitionSpec.builderFor(TABLE_SCHEMA).bucket("id", 16).build(); - Types.NestedField writerField = writerSpec.partitionType().fields().get(0); - Types.NestedField extraField = - Types.NestedField.optional(2000, "extra", Types.IntegerType.get()); - Types.StructType unionType = Types.StructType.of(writerField, extraField); - - PartitionData partition = new PartitionData(writerSpec.partitionType()); - partition.set(0, 7); - DataFile file = + void dataTrackedFileAdapterKeepsPartitionTuple() { + DataFile partitioned = new GenericDataFile( - writerSpec.specId(), - DATA_PATH, + PARTITIONED_SPEC.specId(), + DATA_FILE_LOCATION, FileFormat.PARQUET, - partition, + PARTITION, 1024L, new Metrics(100L, null, null, null, null), null, ImmutableList.of(0L), null, null); + TrackedFile result = + TrackedFileAdapters.forDataFile(TABLE_SCHEMA, METRICS_CONFIG).wrap(partitioned); - TrackedFileAdapters.DataTrackedFile wrapper = - TrackedFileAdapters.forDataFile( - TrackedFile.schema( - unionType, StatsUtil.statsWriteSchema(TABLE_SCHEMA, METRICS_CONFIG))); - TrackedFile result = wrapper.wrap(file, ADDED_TRACKING); - - StructLike projected = result.partition(); - int writerPos = unionType.fields().indexOf(writerField); - int extraPos = unionType.fields().indexOf(extraField); - assertThat(projected.get(writerPos, Integer.class)).isEqualTo(7); - assertThat(projected.get(extraPos, Integer.class)).isNull(); + assertThat(result.partition().get(0, String.class)).isEqualTo("books"); + assertThat( + TrackedFileAdapters.asDataFile(result, specsById(PARTITIONED_SPEC)) + .partition() + .get(0, String.class)) + .isEqualTo("books"); } @Test - void testDataFileDoubleWrapRoundTrip() { - // DataFile -> TrackedFile (write) -> DataFile (read) round-trip preserves fields observable on - // the DataFile API. + void dataFileDoubleWrapRoundTrip() { DataFile source = DATA_FILE_WITH_METRICS; Map specs = ImmutableMap.of(UNPARTITIONED_SPEC.specId(), UNPARTITIONED_SPEC); TrackedFile tracked = - TrackedFileAdapters.forDataFile(WRITE_SCHEMA).wrap(source, EXISTING_TRACKING); + TrackedFileAdapters.forDataFile(TABLE_SCHEMA, METRICS_CONFIG).wrap(existingEntry(source)); DataFile roundTripped = TrackedFileAdapters.asDataFile(tracked, specs); @@ -1028,73 +886,54 @@ void testDataFileDoubleWrapRoundTrip() { assertThat(roundTripped.recordCount()).isEqualTo(source.recordCount()); assertThat(roundTripped.fileSizeInBytes()).isEqualTo(source.fileSizeInBytes()); assertThat(roundTripped.specId()).isEqualTo(source.specId()); - // partition survives as a StructProjection view; compare positionally by field type - assertThat( - Comparators.forType(PARTITION_TYPE) - .compare(roundTripped.partition(), source.partition())) - .isEqualTo(0); + assertThat(roundTripped.partition()).isEqualTo(source.partition()); assertThat(roundTripped.sortOrderId()).isEqualTo(source.sortOrderId()); assertThat(roundTripped.splitOffsets()).isEqualTo(source.splitOffsets()); assertThat(roundTripped.keyMetadata()).isEqualTo(source.keyMetadata()); - assertThat(roundTripped.dataSequenceNumber()).isEqualTo(DATA_SEQ); - assertThat(roundTripped.fileSequenceNumber()).isEqualTo(FILE_SEQ); + assertThat(roundTripped.dataSequenceNumber()).isEqualTo(DATA_SEQUENCE_NUMBER); + assertThat(roundTripped.fileSequenceNumber()).isEqualTo(FILE_SEQUENCE_NUMBER); assertThat(roundTripped.firstRowId()).isNull(); assertThat(roundTripped.valueCounts()).containsAllEntriesOf(source.valueCounts()); assertThat(roundTripped.nullValueCounts()).containsAllEntriesOf(source.nullValueCounts()); - // METRICS_WITH_BOUNDS has an empty nan_value_counts map; the read side surfaces null when no - // column tracks nan assertThat(roundTripped.nanValueCounts()).isNull(); assertThat(roundTripped.lowerBounds()).containsAllEntriesOf(source.lowerBounds()); assertThat(roundTripped.upperBounds()).containsAllEntriesOf(source.upperBounds()); - // the write wrapper deliberately does not carry column_sizes; a round-trip loses that field assertThat(roundTripped.columnSizes()).isNull(); } @Test - void dataFileAdapterUnwrapsToOriginalTrackedFile() { + void dataTrackedFileAdapterUnwrapsToOriginalTrackedFile() { TrackedFile original = dummyTrackedFile(FileContent.DATA); DataFile adapted = TrackedFileAdapters.asDataFile(original, UNPARTITIONED); TrackedFile result = - TrackedFileAdapters.forDataFile(WRITE_SCHEMA).wrap(adapted, ADDED_TRACKING); + TrackedFileAdapters.forDataFile(TABLE_SCHEMA, METRICS_CONFIG).wrap(adapted); assertThat(result).isSameAs(original); } @Test - void equalityDeleteFileAdapterUnwrapsToOriginalTrackedFile() { - TrackedFile original = dummyTrackedFile(FileContent.EQUALITY_DELETES); - DeleteFile adapted = TrackedFileAdapters.asEqualityDeleteFile(original, UNPARTITIONED); - - TrackedFile result = - TrackedFileAdapters.forEqualityDeleteFile(WRITE_SCHEMA).wrap(adapted, ADDED_TRACKING); - - assertThat(result).isSameAs(original); - } - - @Test - void manifestReferenceAdapterUnwrapsToOriginalTrackedFile() { + void manifestTrackedFileAdapterUnwrapsToOriginalTrackedFile() { TrackedFile original = dummyTrackedFile(FileContent.DATA_MANIFEST, 0); ManifestFile adapted = TrackedFileAdapters.asManifestFile(original); - TrackedFile result = - TrackedFileAdapters.forManifestReference().wrap(adapted, EntryStatus.ADDED, 1000L); + TrackedFile result = TrackedFileAdapters.forManifestFile().wrap(adapted); assertThat(result).isSameAs(original); assertThat(result.formatVersion()).isZero(); } @Test - void testDataManifestReferenceWrapper() { + void dataManifestTrackedFileAdapter() { ManifestFile manifest = writeManifestFile(ManifestContent.DATA); - TrackedFileAdapters.ManifestTrackedFile wrapper = TrackedFileAdapters.forManifestReference(); - TrackedFile result = wrapper.wrap(manifest, EntryStatus.ADDED, 1000L); + TrackedFile result = TrackedFileAdapters.forManifestFile().wrap(manifest); assertThat(result.contentType()).isEqualTo(FileContent.DATA_MANIFEST); - assertThat(result.formatVersion()).isEqualTo(FORMAT_VERSION_V4); - assertThat(result.location()).isEqualTo(MANIFEST_PATH); - assertThat(result.tracking().status()).isEqualTo(EntryStatus.ADDED); - assertThat(result.tracking().firstRowId()).isEqualTo(1000L); + assertThat(result.formatVersion()).isZero(); + assertThat(result.location()).isEqualTo(MANIFEST_LOCATION); + assertThat(result.tracking().status()).isNull(); + assertThat(result.tracking().snapshotId()).isEqualTo(SNAPSHOT_ID); + assertThat(result.tracking().firstRowId()).isNull(); assertThat(result.recordCount()).isEqualTo(6L); assertThat(result.manifestInfo()).isNotNull(); assertThat(result.manifestInfo().addedFilesCount()).isEqualTo(2); @@ -1103,43 +942,49 @@ void testDataManifestReferenceWrapper() { assertThat(result.manifestInfo().addedRowsCount()).isEqualTo(200L); assertThat(result.manifestInfo().replacedFilesCount()).isEqualTo(0); assertThat(result.manifestInfo().replacedRowsCount()).isEqualTo(0L); + assertThat(result.manifestInfo().modifiedFilesCount()).isEqualTo(0); + assertThat(result.manifestInfo().modifiedRowsCount()).isEqualTo(0L); } @Test - void testDeleteManifestReferenceWrapper() { + void deleteManifestTrackedFileAdapter() { ManifestFile manifest = writeManifestFile(ManifestContent.DELETES); - TrackedFileAdapters.ManifestTrackedFile wrapper = TrackedFileAdapters.forManifestReference(); - TrackedFile result = wrapper.wrap(manifest, EntryStatus.EXISTING, null); + TrackedFile result = TrackedFileAdapters.forManifestFile().wrap(manifest); assertThat(result.contentType()).isEqualTo(FileContent.DELETE_MANIFEST); - assertThat(result.formatVersion()).isEqualTo(FORMAT_VERSION_V4); + assertThat(result.formatVersion()).isZero(); assertThat(result.recordCount()).isEqualTo(6L); - assertThat(result.tracking().status()).isEqualTo(EntryStatus.EXISTING); + assertThat(result.tracking().status()).isNull(); assertThat(result.tracking().firstRowId()).isNull(); } @Test - void testManifestReferenceWrapperRejectsFirstRowIdOnDeleteManifest() { - ManifestFile manifest = writeManifestFile(ManifestContent.DELETES); - TrackedFileAdapters.ManifestTrackedFile wrapper = TrackedFileAdapters.forManifestReference(); - assertThatThrownBy(() -> wrapper.wrap(manifest, EntryStatus.ADDED, 100L)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("firstRowId is only valid for DATA manifests"); + void manifestTrackedFileAdapterFailsWhenAddedFilesCountMissing() { + ManifestFile manifest = mock(ManifestFile.class); + when(manifest.path()).thenReturn(MANIFEST_LOCATION); + when(manifest.content()).thenReturn(ManifestContent.DATA); + when(manifest.addedFilesCount()).thenReturn(null); + when(manifest.existingFilesCount()).thenReturn(1); + when(manifest.deletedFilesCount()).thenReturn(0); + when(manifest.replacedFilesCount()).thenReturn(0); + + assertThatThrownBy(() -> TrackedFileAdapters.forManifestFile().wrap(manifest)) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("missing added files count"); } @Test - void testManifestReferenceWrapperRejectsUnassignedSequenceNumber() { - ManifestFile manifest = - writeManifestFile(ManifestContent.DATA, ManifestWriter.UNASSIGNED_SEQ, 4L); - TrackedFileAdapters.ManifestTrackedFile wrapper = TrackedFileAdapters.forManifestReference(); - assertThatThrownBy(() -> wrapper.wrap(manifest, EntryStatus.ADDED, null)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("sequence_number is unassigned"); + void manifestTrackedFileAdapterFailsWhenAddedRowsCountMissing() { + ManifestFile manifest = writeManifestFile(ManifestContent.DATA, 5L, 4L, null); + TrackedFile tracked = TrackedFileAdapters.forManifestFile().wrap(manifest); + + assertThatThrownBy(() -> tracked.manifestInfo().addedRowsCount()) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("null"); } - private static void assertWriteDataFields(TrackedFile result, DataFile file, int formatVersion) { + private static void assertWrappedDataFileMatchesFileFields(TrackedFile result, DataFile file) { assertThat(result.contentType()).isEqualTo(FileContent.DATA); - assertThat(result.formatVersion()).isEqualTo(formatVersion); assertThat(result.location()).isEqualTo(file.location()); assertThat(result.fileFormat()).isEqualTo(file.format()); assertThat(result.recordCount()).isEqualTo(file.recordCount()); @@ -1148,22 +993,25 @@ private static void assertWriteDataFields(TrackedFile result, DataFile file, int assertThat(result.sortOrderId()).isEqualTo(file.sortOrderId()); assertThat(result.keyMetadata()).isEqualTo(file.keyMetadata()); assertThat(result.splitOffsets()).isEqualTo(file.splitOffsets()); - // data files carry no manifest-info or deletion-vector metadata assertThat(result.manifestInfo()).isNull(); assertThat(result.deletionVector()).isNull(); - // equalityIds is null for data files (only equality-delete files populate it) assertThat(result.equalityIds()).isNull(); } private static ManifestFile writeManifestFile(ManifestContent content) { - return writeManifestFile(content, 5L, 4L); + return writeManifestFile(content, 5L, 4L, 200L); } private static ManifestFile writeManifestFile( ManifestContent content, long sequenceNumber, long minSequenceNumber) { + return writeManifestFile(content, sequenceNumber, minSequenceNumber, 200L); + } + + private static ManifestFile writeManifestFile( + ManifestContent content, long sequenceNumber, long minSequenceNumber, Long addedRowsCount) { List partitions = ImmutableList.of(); return new GenericManifestFile( - MANIFEST_PATH, + MANIFEST_LOCATION, 2048L, UNPARTITIONED_SPEC.specId(), content, @@ -1173,7 +1021,7 @@ private static ManifestFile writeManifestFile( partitions, null, 2, - 200L, + addedRowsCount, 3, 300L, 1, @@ -1181,6 +1029,22 @@ private static ManifestFile writeManifestFile( null); } + private static GenericManifestEntry newEntry() { + return new GenericManifestEntry<>(ManifestEntry.getSchema(PARTITION_TYPE).asStruct()); + } + + private static ManifestEntry addedEntry(DataFile file) { + return newEntry().wrapAppend(SNAPSHOT_ID, file); + } + + private static ManifestEntry existingEntry(DataFile file) { + return newEntry().wrapExisting(SNAPSHOT_ID, DATA_SEQUENCE_NUMBER, FILE_SEQUENCE_NUMBER, file); + } + + private static ManifestEntry deletedEntry(DataFile file) { + return newEntry().wrapDelete(SNAPSHOT_ID, DATA_SEQUENCE_NUMBER, FILE_SEQUENCE_NUMBER, file); + } + private static void assertNullTrackingFields(ContentFile file) { assertThat(file.pos()).isNull(); assertThat(file.manifestLocation()).isNull(); From 906c6bd6a1df3b0edece1aecff6d77d3be062840 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Fri, 2 Oct 2026 23:04:03 -0700 Subject: [PATCH 05/11] Core: Ignore stat fields absent from the content-stats type. Reuse tracking wrappers when wrapping data and manifest files, implement StructLike on GeospatialBound, and treat pre-v4 manifests as Avro with EXISTING status. Generated-by: Cursor Grok 4.7 Co-authored-by: Cursor --- .../iceberg/geospatial/GeospatialBound.java | 24 +- .../geospatial/TestGeospatialBound.java | 30 +++ .../apache/iceberg/MapBackedContentStats.java | 75 ++++-- .../apache/iceberg/TrackedFileAdapters.java | 215 ++++++++++++++---- .../iceberg/TestMapBackedContentStats.java | 90 +++++--- .../iceberg/TestTrackedFileAdapters.java | 129 ++++++++--- 6 files changed, 446 insertions(+), 117 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/geospatial/GeospatialBound.java b/api/src/main/java/org/apache/iceberg/geospatial/GeospatialBound.java index 3425a4663801..cdb76ae3c46e 100644 --- a/api/src/main/java/org/apache/iceberg/geospatial/GeospatialBound.java +++ b/api/src/main/java/org/apache/iceberg/geospatial/GeospatialBound.java @@ -21,6 +21,7 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.Objects; +import org.apache.iceberg.StructLike; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; /** @@ -44,7 +45,7 @@ *

This class represents a lower or upper geospatial bound and handles serialization and * deserialization of these bounds to/from byte arrays, conforming to the Iceberg specification. */ -public class GeospatialBound { +public class GeospatialBound implements StructLike { /** * Parses a geospatial bound from a byte buffer according to Iceberg spec. * @@ -282,6 +283,27 @@ public boolean hasM() { return !Double.isNaN(m); } + @Override + public int size() { + return 4; + } + + @Override + public T get(int pos, Class javaClass) { + return switch (pos) { + case 0 -> javaClass.cast(x); + case 1 -> javaClass.cast(y); + case 2 -> hasZ() ? javaClass.cast(z) : null; + case 3 -> hasM() ? javaClass.cast(m) : null; + default -> throw new IllegalArgumentException("Invalid position: " + pos); + }; + } + + @Override + public void set(int pos, T value) { + throw new UnsupportedOperationException("GeospatialBound is read only"); + } + @Override public String toString() { return "GeospatialBound(" + simpleString() + ")"; diff --git a/api/src/test/java/org/apache/iceberg/geospatial/TestGeospatialBound.java b/api/src/test/java/org/apache/iceberg/geospatial/TestGeospatialBound.java index 8a6a6325cc85..7b60ddc38c1d 100644 --- a/api/src/test/java/org/apache/iceberg/geospatial/TestGeospatialBound.java +++ b/api/src/test/java/org/apache/iceberg/geospatial/TestGeospatialBound.java @@ -19,9 +19,11 @@ package org.apache.iceberg.geospatial; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import org.apache.iceberg.StructLike; import org.apache.iceberg.util.ByteBuffers; import org.junit.jupiter.api.Test; @@ -71,6 +73,34 @@ public void testCreateXYZM() { assertThat(bound.hasM()).isTrue(); } + @Test + public void testStructLikeXY() { + StructLike bound = GeospatialBound.createXY(1.0, 2.0); + assertThat(bound.size()).isEqualTo(4); + assertThat(bound.get(0, Double.class)).isEqualTo(1.0); + assertThat(bound.get(1, Double.class)).isEqualTo(2.0); + assertThat(bound.get(2, Double.class)).isNull(); + assertThat(bound.get(3, Double.class)).isNull(); + } + + @Test + public void testStructLikeXYZM() { + StructLike bound = GeospatialBound.createXYZM(3.0, 4.0, 5.0, 6.0); + assertThat(bound.size()).isEqualTo(4); + assertThat(bound.get(0, Double.class)).isEqualTo(3.0); + assertThat(bound.get(1, Double.class)).isEqualTo(4.0); + assertThat(bound.get(2, Double.class)).isEqualTo(5.0); + assertThat(bound.get(3, Double.class)).isEqualTo(6.0); + } + + @Test + public void testSet() { + GeospatialBound bound = GeospatialBound.createXY(1.0, 2.0); + assertThatThrownBy(() -> bound.set(0, 9.0)) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessage("GeospatialBound is read only"); + } + @Test public void testEqualsAndHashCode() { GeospatialBound xy1 = GeospatialBound.createXY(1.0, 2.0); diff --git a/core/src/main/java/org/apache/iceberg/MapBackedContentStats.java b/core/src/main/java/org/apache/iceberg/MapBackedContentStats.java index fc68cfad48c2..4f8d7504323d 100644 --- a/core/src/main/java/org/apache/iceberg/MapBackedContentStats.java +++ b/core/src/main/java/org/apache/iceberg/MapBackedContentStats.java @@ -19,21 +19,25 @@ package org.apache.iceberg; import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.util.Map; import java.util.Objects; import java.util.Set; +import org.apache.iceberg.geospatial.GeospatialBound; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.types.Conversions; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; /** Reusable {@link ContentStats} view over a {@link ContentFile}'s stat maps. */ class MapBackedContentStats implements ContentStats { - private final Types.StructType type; + private final Schema tableSchema; private final Map> statsById = Maps.newHashMap(); + private Types.StructType type; private Map valueCounts; private Map nullValueCounts; private Map nanValueCounts; @@ -41,10 +45,9 @@ class MapBackedContentStats implements ContentStats { private Map lowerBounds; private Map upperBounds; - MapBackedContentStats(Schema tableSchema, MetricsConfig metricsConfig) { + MapBackedContentStats(Schema tableSchema) { Preconditions.checkArgument(tableSchema != null, "Invalid table schema: null"); - Preconditions.checkArgument(metricsConfig != null, "Invalid metrics config: null"); - this.type = StatsUtil.statsWriteSchema(tableSchema, metricsConfig); + this.tableSchema = tableSchema; } MapBackedContentStats wrap(ContentFile file) { @@ -54,20 +57,23 @@ MapBackedContentStats wrap(ContentFile file) { this.avgValueSizes = file.avgValueSizes(); this.lowerBounds = file.lowerBounds(); this.upperBounds = file.upperBounds(); + this.type = null; + statsById.clear(); return this; } @Override public Iterable> fieldStats() { return Iterables.filter( - Iterables.transform(type.fields(), field -> statsFor(StatsUtil.toFieldId(field.fieldId()))), + Iterables.transform( + type().fields(), field -> statsFor(StatsUtil.toFieldId(field.fieldId()))), Objects::nonNull); } @Override @SuppressWarnings("unchecked") public FieldStats statsFor(int fieldId) { - if (!hasStats(fieldId)) { + if (!containsFieldInMaps(fieldId) || type().field(StatsUtil.toBaseId(fieldId)) == null) { return null; } @@ -82,6 +88,10 @@ public FieldStats statsFor(int fieldId) { @Override public Types.StructType type() { + if (type == null) { + this.type = StatsUtil.statsReadSchema(tableSchema, statsFieldIds()); + } + return type; } @@ -95,13 +105,30 @@ public ContentStats copy(Set fieldIds) { throw new UnsupportedOperationException("copy is not implemented"); } - private boolean hasStats(int id) { - return containsId(valueCounts, id) - || containsId(nullValueCounts, id) - || containsId(nanValueCounts, id) - || containsId(avgValueSizes, id) - || containsId(lowerBounds, id) - || containsId(upperBounds, id); + boolean containsFieldInMaps(int fieldId) { + return containsId(valueCounts, fieldId) + || containsId(nullValueCounts, fieldId) + || containsId(nanValueCounts, fieldId) + || containsId(avgValueSizes, fieldId) + || containsId(lowerBounds, fieldId) + || containsId(upperBounds, fieldId); + } + + private Set statsFieldIds() { + Set ids = Sets.newTreeSet(); + addKeys(ids, valueCounts); + addKeys(ids, nullValueCounts); + addKeys(ids, nanValueCounts); + addKeys(ids, avgValueSizes); + addKeys(ids, lowerBounds); + addKeys(ids, upperBounds); + return ids; + } + + private static void addKeys(Set ids, Map map) { + if (map != null) { + ids.addAll(map.keySet()); + } } private static boolean containsId(Map map, int id) { @@ -112,16 +139,18 @@ private static boolean containsId(Map map, int id) { private class MapBackedFieldStats implements FieldStats { private final int fieldId; private final Types.StructType struct; + private final Type columnType; private final Type boundType; MapBackedFieldStats(int fieldId) { Types.NestedField field = type.field(StatsUtil.toBaseId(fieldId)); Preconditions.checkArgument( - field != null, - "Cannot convert stats for field ID %s: unknown, not a scalar, or not in metrics config", - fieldId); + field != null, "Cannot convert stats for field ID %s: unknown or not a scalar", fieldId); + Types.NestedField column = tableSchema.findField(fieldId); + Preconditions.checkArgument(column != null, "Missing column for field ID %s", fieldId); this.fieldId = fieldId; this.struct = field.type().asStructType(); + this.columnType = column.type(); this.boundType = struct.fieldType(StatsUtil.LOWER_BOUND_NAME); } @@ -151,7 +180,19 @@ private T decodeBound(Map bounds) { return null; } - return (T) Conversions.fromByteBuffer(boundType, bounds.get(fieldId)); + ByteBuffer buffer = bounds.get(fieldId); + if (columnType.typeId() == Type.TypeID.GEOMETRY + || columnType.typeId() == Type.TypeID.GEOGRAPHY) { + // File-map geometry and geography bounds are single-point encoded. + if (buffer == null) { + return null; + } + + ByteBuffer tmp = buffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); + return (T) GeospatialBound.fromByteBuffer(tmp); + } + + return (T) Conversions.fromByteBuffer(boundType, buffer); } @Override diff --git a/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java b/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java index 3c451442fe1e..280dc05b55ab 100644 --- a/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java +++ b/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java @@ -22,6 +22,7 @@ import java.nio.ByteBuffer; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; @@ -67,8 +68,8 @@ static ManifestFile asManifestFile(TrackedFile file) { } /** Returns a reusable adapter from {@link DataFile} to {@link TrackedFile}. */ - static DataTrackedFile forDataFile(Schema tableSchema, MetricsConfig metricsConfig) { - return new DataTrackedFile(tableSchema, metricsConfig); + static DataTrackedFile forDataFile(Schema tableSchema) { + return new DataTrackedFile(tableSchema); } /** Returns a reusable adapter from {@link ManifestFile} to {@link TrackedFile}. */ @@ -569,33 +570,49 @@ public ManifestFile copy() { /** Adapts a {@link DataFile} to {@link TrackedFile}. */ static class DataTrackedFile implements TrackedFile { private final MapBackedContentStats statsWrapper; + private final WrappedEntryTracking trackingWrapper = new WrappedEntryTracking(); + private Tracking tracking; private DataFile file; private ContentStats stats; - DataTrackedFile(Schema tableSchema, MetricsConfig metricsConfig) { - this.statsWrapper = new MapBackedContentStats(tableSchema, metricsConfig); + DataTrackedFile(Schema tableSchema) { + this.statsWrapper = new MapBackedContentStats(tableSchema); } - /** Re-points this adapter at a {@link DataFile} from the public API. Tracking is unset. */ + /** + * Re-points this adapter at a {@link DataFile} from the public API. Tracking is unset. + * + *

Returns the inner {@link TrackedFile} when {@code newFile} is a {@link TrackedDataFile}. + */ public TrackedFile wrap(DataFile newFile) { - return wrapFile(newFile, null); + if (newFile instanceof TrackedDataFile tracked) { + return tracked.file(); + } + + wrapInternal(newFile); + this.tracking = null; + return this; } /** * Re-points this adapter at a {@link ManifestEntry}. Converts the contained data file and the * entry's tracking fields. + * + *

Returns the inner {@link TrackedFile} when the entry's file is a {@link TrackedDataFile}. */ public TrackedFile wrap(ManifestEntry entry) { Preconditions.checkArgument(entry != null, "Invalid entry: null"); - return wrapFile(entry.file(), trackingFrom(entry, entry.file())); - } - - private TrackedFile wrapFile(DataFile newFile, Tracking newTracking) { - if (newFile instanceof TrackedDataFile tracked) { + if (entry.file() instanceof TrackedDataFile tracked) { return tracked.file(); } + wrapInternal(entry.file()); + this.tracking = trackingWrapper.wrap(entry); + return this; + } + + private void wrapInternal(DataFile newFile) { Preconditions.checkArgument(newFile != null, "Invalid file: null"); Preconditions.checkArgument( newFile.content() == FileContent.DATA, @@ -604,8 +621,6 @@ private TrackedFile wrapFile(DataFile newFile, Tracking newTracking) { this.file = newFile; this.stats = hasContentStats(newFile) ? statsWrapper.wrap(newFile) : null; - this.tracking = newTracking; - return this; } @Override @@ -703,6 +718,7 @@ public TrackedFile copyWithStats(Set requestedColumnIds) { /** Adapts a {@link ManifestFile} to {@link TrackedFile}. */ static class ManifestTrackedFile implements TrackedFile { private final WrappedManifestInfo manifestInfo = new WrappedManifestInfo(); + private final WrappedManifestTracking trackingWrapper = new WrappedManifestTracking(); private Tracking tracking; private ManifestFile manifest; private long recordCount; @@ -713,6 +729,9 @@ static class ManifestTrackedFile implements TrackedFile { /** * Re-points this adapter at {@code newManifest}. Converts the manifest's own fields only; * write-time tracking updates are applied by the versioned writer. + * + *

Returns the inner {@link TrackedFile} when {@code newManifest} is a {@link + * TrackedManifestFile}. */ public TrackedFile wrap(ManifestFile newManifest) { if (newManifest instanceof TrackedManifestFile tracked) { @@ -727,7 +746,7 @@ public TrackedFile wrap(ManifestFile newManifest) { ? FileContent.DATA_MANIFEST : FileContent.DELETE_MANIFEST; this.recordCount = manifestRecordCount(newManifest); - this.tracking = trackingFrom(newManifest); + this.tracking = trackingWrapper.wrap(newManifest); this.manifestInfo.wrap(newManifest); return this; } @@ -754,7 +773,8 @@ public String location() { @Override public FileFormat fileFormat() { - return FileFormat.fromFileName(manifest.path()); + // Manifest files before v4 are always Avro. + return FileFormat.AVRO; } @Override @@ -787,6 +807,7 @@ public ContentStats contentStats() { @Override public Integer sortOrderId() { + // Manifests have no table sort order. return null; } @@ -828,10 +849,11 @@ public TrackedFile copyWithStats(Set requestedColumnIds) { /** Reusable {@link ManifestInfo} view over a {@link ManifestFile}'s counts. */ private static class WrappedManifestInfo implements ManifestInfo { - private ManifestFile manifest; + private ManifestFile manifest = null; - void wrap(ManifestFile newManifest) { + WrappedManifestInfo wrap(ManifestFile newManifest) { this.manifest = newManifest; + return this; } @Override @@ -900,28 +922,135 @@ public ManifestInfo copy() { } } - private static Tracking trackingFrom(ManifestEntry entry, ContentFile file) { - return new TrackingStruct( - entryStatus(entry.status()), - entry.snapshotId(), - entry.dataSequenceNumber(), - entry.fileSequenceNumber(), - null, - file.firstRowId(), - null, - null); + private static class WrappedEntryTracking implements Tracking { + private ManifestEntry entry = null; + + WrappedEntryTracking wrap(ManifestEntry newEntry) { + this.entry = newEntry; + return this; + } + + @Override + public EntryStatus status() { + return entryStatus(entry.status()); + } + + @Override + public Long snapshotId() { + return entry.snapshotId(); + } + + @Override + public Long dataSequenceNumber() { + return entry.dataSequenceNumber(); + } + + @Override + public Long fileSequenceNumber() { + return entry.fileSequenceNumber(); + } + + @Override + public Long dvSnapshotId() { + // dvSnapshotId is null because this wrapper has no DV entry + return null; + } + + @Override + public Long firstRowId() { + return entry.file().firstRowId(); + } + + @Override + public ByteBuffer deletedPositions() { + return null; + } + + @Override + public ByteBuffer replacedPositions() { + return null; + } + + @Override + public String manifestLocation() { + return entry.file().manifestLocation(); + } + + @Override + public long manifestPos() { + Long pos = entry.file().pos(); + return pos != null ? pos : -1L; + } + + @Override + public Tracking copy() { + throw new UnsupportedOperationException("copy is not implemented"); + } } - private static Tracking trackingFrom(ManifestFile manifest) { - return new TrackingStruct( - null, - manifest.snapshotId(), - manifest.sequenceNumber(), - manifest.sequenceNumber(), - null, - manifest.firstRowId(), - null, - null); + private static class WrappedManifestTracking implements Tracking { + private ManifestFile manifest = null; + + WrappedManifestTracking wrap(ManifestFile newManifest) { + this.manifest = newManifest; + return this; + } + + @Override + public EntryStatus status() { + // Pre-v4 manifests have no status and are live; the writer sets EXISTING or MODIFIED. + return EntryStatus.EXISTING; + } + + @Override + public Long snapshotId() { + return manifest.snapshotId(); + } + + @Override + public Long dataSequenceNumber() { + return manifest.sequenceNumber(); + } + + @Override + public Long fileSequenceNumber() { + return manifest.sequenceNumber(); + } + + @Override + public Long dvSnapshotId() { + return null; + } + + @Override + public Long firstRowId() { + return manifest.firstRowId(); + } + + @Override + public ByteBuffer deletedPositions() { + return null; + } + + @Override + public ByteBuffer replacedPositions() { + return null; + } + + @Override + public String manifestLocation() { + return null; + } + + @Override + public long manifestPos() { + return -1L; + } + + @Override + public Tracking copy() { + throw new UnsupportedOperationException("copy is not implemented"); + } } private static EntryStatus entryStatus(ManifestEntry.Status status) { @@ -962,21 +1091,13 @@ private static long manifestRecordCount(ManifestFile manifest) { manifest.deletedFilesCount(), "Cannot convert manifest %s: missing deleted files count", manifest.path()); - Preconditions.checkNotNull( - manifest.replacedFilesCount(), - "Cannot convert manifest %s: missing replaced files count", - manifest.path()); - Preconditions.checkNotNull( - manifest.modifiedFilesCount(), - "Cannot convert manifest %s: missing modified files count", - manifest.path()); Preconditions.checkArgument( - manifest.replacedFilesCount() == 0, + Objects.equals(manifest.replacedFilesCount(), 0), "Cannot convert manifest %s: replaced files count must be 0 for v3 or earlier manifests, was %s", manifest.path(), manifest.replacedFilesCount()); Preconditions.checkArgument( - manifest.modifiedFilesCount() == 0, + Objects.equals(manifest.modifiedFilesCount(), 0), "Cannot convert manifest %s: modified files count must be 0 for v3 or earlier manifests, was %s", manifest.path(), manifest.modifiedFilesCount()); diff --git a/core/src/test/java/org/apache/iceberg/TestMapBackedContentStats.java b/core/src/test/java/org/apache/iceberg/TestMapBackedContentStats.java index 8286f719ea22..b43e8f4c2f6e 100644 --- a/core/src/test/java/org/apache/iceberg/TestMapBackedContentStats.java +++ b/core/src/test/java/org/apache/iceberg/TestMapBackedContentStats.java @@ -23,6 +23,7 @@ import java.nio.ByteBuffer; import java.util.Map; +import org.apache.iceberg.geospatial.GeospatialBound; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; @@ -41,8 +42,6 @@ class TestMapBackedContentStats { Types.NestedField.optional(4, "name", Types.StringType.get()), Types.NestedField.optional(5, "flag", Types.BooleanType.get())); - private static final MetricsConfig METRICS_CONFIG = MetricsTestUtil.from(Map.of(), SCHEMA); - private static final PartitionData EMPTY_PARTITION = new PartitionData(PartitionSpec.unpartitioned().partitionType()); @@ -67,18 +66,41 @@ class TestMapBackedContentStats { 4, buf(Types.StringType.get(), "zzz"))); @Test - void contentStatsType() { - Types.StructType expected = StatsUtil.statsWriteSchema(SCHEMA, METRICS_CONFIG); - MapBackedContentStats stats = new MapBackedContentStats(SCHEMA, METRICS_CONFIG); + void contentStatsTypeBuiltLazilyFromMapIds() { + MapBackedContentStats stats = new MapBackedContentStats(SCHEMA); + + assertThat(stats.type()).isEqualTo(StatsUtil.statsReadSchema(SCHEMA, ImmutableList.of())); + + stats.wrap(FILE_WITH_STATS); - assertThat(stats.type()).isEqualTo(expected); - assertThat(stats.wrap(FILE_WITH_STATS).type()).isEqualTo(expected); + assertThat(stats.type()) + .isEqualTo(StatsUtil.statsReadSchema(SCHEMA, ImmutableList.of(1, 2, 3, 4))); + } + + @Test + void wrapInvalidatesType() { + MapBackedContentStats stats = new MapBackedContentStats(SCHEMA).wrap(FILE_WITH_STATS); + Types.StructType firstType = stats.type(); + assertThat(firstType) + .isEqualTo(StatsUtil.statsReadSchema(SCHEMA, ImmutableList.of(1, 2, 3, 4))); + + DataFile file2 = + dataFile( + ImmutableMap.of(1, 50L), + ImmutableMap.of(), + ImmutableMap.of(), + ImmutableMap.of(1, buf(Types.IntegerType.get(), 500)), + ImmutableMap.of(1, buf(Types.IntegerType.get(), 5000))); + stats.wrap(file2); + + assertThat(stats.type()) + .isEqualTo(StatsUtil.statsReadSchema(SCHEMA, ImmutableList.of(1))) + .isNotEqualTo(firstType); } @Test void boundDecodingPerType() { - MapBackedContentStats stats = - new MapBackedContentStats(SCHEMA, METRICS_CONFIG).wrap(FILE_WITH_STATS); + MapBackedContentStats stats = new MapBackedContentStats(SCHEMA).wrap(FILE_WITH_STATS); FieldStats id = stats.statsFor(1); assertThat(id.lowerBound()).isInstanceOf(Integer.class).isEqualTo(1); @@ -99,6 +121,25 @@ void boundDecodingPerType() { assertThat(name.upperBound().toString()).isEqualTo("zzz"); } + @Test + void geoBoundsDecode() { + GeospatialBound lower = GeospatialBound.createXY(1.0, 2.0); + GeospatialBound upper = GeospatialBound.createXYZM(3.0, 4.0, 5.0, 6.0); + Schema schema = new Schema(Types.NestedField.optional(10, "geom", Types.GeometryType.crs84())); + DataFile file = + dataFile( + ImmutableMap.of(10, 26L), + ImmutableMap.of(), + ImmutableMap.of(), + ImmutableMap.of(10, lower.toByteBuffer()), + ImmutableMap.of(10, upper.toByteBuffer())); + MapBackedContentStats stats = new MapBackedContentStats(schema).wrap(file); + + FieldStats geom = stats.statsFor(10); + assertThat(geom.lowerBound()).isInstanceOf(GeospatialBound.class).isEqualTo(lower); + assertThat(geom.upperBound()).isInstanceOf(GeospatialBound.class).isEqualTo(upper); + } + @Test void missingBoundsDecodeToNull() { DataFile file = @@ -108,7 +149,7 @@ void missingBoundsDecodeToNull() { ImmutableMap.of(), ImmutableMap.of(), ImmutableMap.of()); - MapBackedContentStats stats = new MapBackedContentStats(SCHEMA, METRICS_CONFIG).wrap(file); + MapBackedContentStats stats = new MapBackedContentStats(SCHEMA).wrap(file); FieldStats id = stats.statsFor(1); assertThat(id.lowerBound()).isNull(); @@ -118,8 +159,7 @@ void missingBoundsDecodeToNull() { @Test void countsAndTightBounds() { - MapBackedContentStats stats = - new MapBackedContentStats(SCHEMA, METRICS_CONFIG).wrap(FILE_WITH_STATS); + MapBackedContentStats stats = new MapBackedContentStats(SCHEMA).wrap(FILE_WITH_STATS); FieldStats id = stats.statsFor(1); assertThat(id.hasValueCount()).isTrue(); @@ -154,10 +194,10 @@ void countsAndTightBounds() { @Test void fieldWithoutStatsIsExcluded() { - MapBackedContentStats stats = - new MapBackedContentStats(SCHEMA, METRICS_CONFIG).wrap(FILE_WITH_STATS); + MapBackedContentStats stats = new MapBackedContentStats(SCHEMA).wrap(FILE_WITH_STATS); - assertThat(stats.type().field(StatsUtil.toBaseId(5))).isNotNull(); + assertThat(stats.containsFieldInMaps(5)).isFalse(); + assertThat(stats.type().field(StatsUtil.toBaseId(5))).isNull(); assertThat(stats.statsFor(5)).isNull(); assertThat(stats.fieldStats()) .extracting(FieldStats::fieldId) @@ -165,7 +205,7 @@ void fieldWithoutStatsIsExcluded() { } @Test - void statsForUnknownFieldIdIsRejected() { + void unknownFieldIdInMaps() { DataFile file = dataFile( ImmutableMap.of(99, 1L), @@ -173,18 +213,17 @@ void statsForUnknownFieldIdIsRejected() { ImmutableMap.of(), ImmutableMap.of(), ImmutableMap.of()); + MapBackedContentStats stats = new MapBackedContentStats(SCHEMA).wrap(file); - assertThatThrownBy( - () -> new MapBackedContentStats(SCHEMA, METRICS_CONFIG).wrap(file).statsFor(99)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage( - "Cannot convert stats for field ID 99: unknown, not a scalar, or not in metrics config"); + assertThat(stats.containsFieldInMaps(99)).isTrue(); + assertThat(stats.statsFor(99)).isNull(); + assertThat(stats.type().fields()).isEmpty(); + assertThat(stats.fieldStats()).isEmpty(); } @Test void copyNotSupported() { - MapBackedContentStats stats = - new MapBackedContentStats(SCHEMA, METRICS_CONFIG).wrap(FILE_WITH_STATS); + MapBackedContentStats stats = new MapBackedContentStats(SCHEMA).wrap(FILE_WITH_STATS); assertThatThrownBy(stats::copy) .isInstanceOf(UnsupportedOperationException.class) @@ -197,8 +236,7 @@ void copyNotSupported() { @Test void fieldStatsCopyNotSupported() { - MapBackedContentStats stats = - new MapBackedContentStats(SCHEMA, METRICS_CONFIG).wrap(FILE_WITH_STATS); + MapBackedContentStats stats = new MapBackedContentStats(SCHEMA).wrap(FILE_WITH_STATS); assertThatThrownBy(stats.statsFor(1)::copy) .isInstanceOf(UnsupportedOperationException.class) @@ -207,7 +245,7 @@ void fieldStatsCopyNotSupported() { @Test void reuseRebindsBounds() { - MapBackedContentStats stats = new MapBackedContentStats(SCHEMA, METRICS_CONFIG); + MapBackedContentStats stats = new MapBackedContentStats(SCHEMA); stats.wrap(FILE_WITH_STATS); assertThat(stats.statsFor(1).lowerBound()).isEqualTo(1); diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java index 28340a77e89b..168fd257ef97 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java @@ -71,13 +71,15 @@ class TestTrackedFileAdapters { // these are populated by readers using the setter with the position of the field. private static final int MANIFEST_LOCATION_ORDINAL = Tracking.schema().fields().size(); private static final int MANIFEST_POSITION_ORDINAL = Tracking.schema().fields().size() + 1; + // row_position follows the data file schema and stores the manifest position. + private static final int DATA_FILE_POS_ORDINAL = + DataFile.getType(Types.StructType.of()).fields().size(); private static final Schema TABLE_SCHEMA = new Schema( optional(1, "id", Types.IntegerType.get()), optional(2, "score", Types.FloatType.get()), optional(3, "geom", Types.GeometryType.crs84())); - private static final MetricsConfig METRICS_CONFIG = MetricsTestUtil.from(Map.of(), TABLE_SCHEMA); private static final Types.StructType CONTENT_STATS_TYPE = StatsUtil.statsReadSchema(TABLE_SCHEMA, ImmutableList.of(1, 2, 3)); private static final FieldStats ID_STATS = @@ -165,6 +167,11 @@ class TestTrackedFileAdapters { null, null); + static { + assignManifestPosition(DATA_FILE, MANIFEST_LOCATION, MANIFEST_POS); + assignManifestPosition(DATA_FILE_WITH_METRICS, MANIFEST_LOCATION, MANIFEST_POS); + } + @Test void dataFileAdapterDelegation() { TrackingStruct tracking = @@ -724,8 +731,7 @@ void unknownSpecIdThrows() { @Test void dataTrackedFileAdapterHasNoTracking() { - TrackedFile result = - TrackedFileAdapters.forDataFile(TABLE_SCHEMA, METRICS_CONFIG).wrap(DATA_FILE); + TrackedFile result = TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(DATA_FILE); assertThat(result.tracking()).isNull(); assertWrappedDataFileMatchesFileFields(result, DATA_FILE); @@ -736,56 +742,56 @@ void dataTrackedFileAdapterHasNoTracking() { @Test void dataTrackedFileAdapterFromAddedManifestEntry() { - TrackedFile result = - TrackedFileAdapters.forDataFile(TABLE_SCHEMA, METRICS_CONFIG).wrap(addedEntry(DATA_FILE)); + TrackedFile result = TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(addedEntry(DATA_FILE)); assertThat(result.tracking().status()).isEqualTo(EntryStatus.ADDED); assertThat(result.tracking().snapshotId()).isEqualTo(SNAPSHOT_ID); assertThat(result.tracking().dataSequenceNumber()).isNull(); assertThat(result.tracking().fileSequenceNumber()).isNull(); assertThat(result.tracking().firstRowId()).isNull(); + assertManifestPosition(result.tracking(), DATA_FILE); assertWrappedDataFileMatchesFileFields(result, DATA_FILE); } @Test void dataTrackedFileAdapterFromAddedManifestEntryWithNullSnapshotId() { TrackedFile result = - TrackedFileAdapters.forDataFile(TABLE_SCHEMA, METRICS_CONFIG) - .wrap(newEntry().wrapAppend(null, DATA_FILE)); + TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(newEntry().wrapAppend(null, DATA_FILE)); assertThat(result.tracking().status()).isEqualTo(EntryStatus.ADDED); assertThat(result.tracking().snapshotId()).isNull(); + assertManifestPosition(result.tracking(), DATA_FILE); assertWrappedDataFileMatchesFileFields(result, DATA_FILE); } @Test void dataTrackedFileAdapterFromExistingManifestEntry() { TrackedFile result = - TrackedFileAdapters.forDataFile(TABLE_SCHEMA, METRICS_CONFIG) - .wrap(existingEntry(DATA_FILE)); + TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(existingEntry(DATA_FILE)); assertThat(result.tracking().status()).isEqualTo(EntryStatus.EXISTING); assertThat(result.tracking().snapshotId()).isEqualTo(SNAPSHOT_ID); assertThat(result.tracking().dataSequenceNumber()).isEqualTo(DATA_SEQUENCE_NUMBER); assertThat(result.tracking().fileSequenceNumber()).isEqualTo(FILE_SEQUENCE_NUMBER); + assertManifestPosition(result.tracking(), DATA_FILE); assertWrappedDataFileMatchesFileFields(result, DATA_FILE); } @Test void dataTrackedFileAdapterFromDeletedManifestEntry() { TrackedFile result = - TrackedFileAdapters.forDataFile(TABLE_SCHEMA, METRICS_CONFIG).wrap(deletedEntry(DATA_FILE)); + TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(deletedEntry(DATA_FILE)); assertThat(result.tracking().status()).isEqualTo(EntryStatus.DELETED); assertThat(result.tracking().snapshotId()).isEqualTo(SNAPSHOT_ID); assertThat(result.tracking().dataSequenceNumber()).isEqualTo(DATA_SEQUENCE_NUMBER); assertThat(result.tracking().fileSequenceNumber()).isEqualTo(FILE_SEQUENCE_NUMBER); + assertManifestPosition(result.tracking(), DATA_FILE); } @Test void dataTrackedFileAdapterReuse() { - TrackedFileAdapters.DataTrackedFile adapter = - TrackedFileAdapters.forDataFile(TABLE_SCHEMA, METRICS_CONFIG); + TrackedFileAdapters.DataTrackedFile adapter = TrackedFileAdapters.forDataFile(TABLE_SCHEMA); adapter.wrap(DATA_FILE); assertThat(adapter.location()).isEqualTo(DATA_FILE_LOCATION); @@ -804,18 +810,19 @@ void dataTrackedFileAdapterReuse() { ImmutableList.of(0L), null, null); + assignManifestPosition(file2, "s3://bucket/table/manifest-2.parquet", 8L); adapter.wrap(existingEntry(file2)); assertThat(adapter.location()).isEqualTo("s3://bucket/data/file2.parquet"); assertThat(adapter.recordCount()).isEqualTo(200L); assertThat(adapter.tracking().status()).isEqualTo(EntryStatus.EXISTING); + assertManifestPosition(adapter.tracking(), file2); assertThat(adapter.fileSizeInBytes()).isEqualTo(2048L); assertThat(adapter.location()).isNotEqualTo(DATA_FILE.location()); } @Test void dataTrackedFileAdapterRejectsNullFile() { - TrackedFileAdapters.DataTrackedFile adapter = - TrackedFileAdapters.forDataFile(TABLE_SCHEMA, METRICS_CONFIG); + TrackedFileAdapters.DataTrackedFile adapter = TrackedFileAdapters.forDataFile(TABLE_SCHEMA); assertThatThrownBy(() -> adapter.wrap((DataFile) null)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("Invalid file: null"); @@ -823,8 +830,7 @@ void dataTrackedFileAdapterRejectsNullFile() { @Test void dataTrackedFileAdapterContentStats() { - TrackedFile result = - TrackedFileAdapters.forDataFile(TABLE_SCHEMA, METRICS_CONFIG).wrap(DATA_FILE_WITH_METRICS); + TrackedFile result = TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(DATA_FILE_WITH_METRICS); ContentStats stats = result.contentStats(); assertThat(stats).isNotNull(); @@ -838,8 +844,7 @@ void dataTrackedFileAdapterContentStats() { @Test void dataTrackedFileAdapterWithoutMetricsHasNoContentStats() { - TrackedFile result = - TrackedFileAdapters.forDataFile(TABLE_SCHEMA, METRICS_CONFIG).wrap(DATA_FILE); + TrackedFile result = TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(DATA_FILE); assertThat(result.contentStats()).isNull(); } @@ -858,8 +863,7 @@ void dataTrackedFileAdapterKeepsPartitionTuple() { ImmutableList.of(0L), null, null); - TrackedFile result = - TrackedFileAdapters.forDataFile(TABLE_SCHEMA, METRICS_CONFIG).wrap(partitioned); + TrackedFile result = TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(partitioned); assertThat(result.partition().get(0, String.class)).isEqualTo("books"); assertThat( @@ -875,8 +879,7 @@ void dataFileDoubleWrapRoundTrip() { Map specs = ImmutableMap.of(UNPARTITIONED_SPEC.specId(), UNPARTITIONED_SPEC); - TrackedFile tracked = - TrackedFileAdapters.forDataFile(TABLE_SCHEMA, METRICS_CONFIG).wrap(existingEntry(source)); + TrackedFile tracked = TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(existingEntry(source)); DataFile roundTripped = TrackedFileAdapters.asDataFile(tracked, specs); @@ -893,6 +896,8 @@ void dataFileDoubleWrapRoundTrip() { assertThat(roundTripped.dataSequenceNumber()).isEqualTo(DATA_SEQUENCE_NUMBER); assertThat(roundTripped.fileSequenceNumber()).isEqualTo(FILE_SEQUENCE_NUMBER); assertThat(roundTripped.firstRowId()).isNull(); + assertThat(roundTripped.manifestLocation()).isEqualTo(source.manifestLocation()); + assertThat(roundTripped.pos()).isEqualTo(source.pos()); assertThat(roundTripped.valueCounts()).containsAllEntriesOf(source.valueCounts()); assertThat(roundTripped.nullValueCounts()).containsAllEntriesOf(source.nullValueCounts()); assertThat(roundTripped.nanValueCounts()).isNull(); @@ -906,8 +911,7 @@ void dataTrackedFileAdapterUnwrapsToOriginalTrackedFile() { TrackedFile original = dummyTrackedFile(FileContent.DATA); DataFile adapted = TrackedFileAdapters.asDataFile(original, UNPARTITIONED); - TrackedFile result = - TrackedFileAdapters.forDataFile(TABLE_SCHEMA, METRICS_CONFIG).wrap(adapted); + TrackedFile result = TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(adapted); assertThat(result).isSameAs(original); } @@ -931,7 +935,8 @@ void dataManifestTrackedFileAdapter() { assertThat(result.contentType()).isEqualTo(FileContent.DATA_MANIFEST); assertThat(result.formatVersion()).isZero(); assertThat(result.location()).isEqualTo(MANIFEST_LOCATION); - assertThat(result.tracking().status()).isNull(); + assertThat(result.fileFormat()).isEqualTo(FileFormat.AVRO); + assertThat(result.tracking().status()).isEqualTo(EntryStatus.EXISTING); assertThat(result.tracking().snapshotId()).isEqualTo(SNAPSHOT_ID); assertThat(result.tracking().firstRowId()).isNull(); assertThat(result.recordCount()).isEqualTo(6L); @@ -954,7 +959,7 @@ void deleteManifestTrackedFileAdapter() { assertThat(result.contentType()).isEqualTo(FileContent.DELETE_MANIFEST); assertThat(result.formatVersion()).isZero(); assertThat(result.recordCount()).isEqualTo(6L); - assertThat(result.tracking().status()).isNull(); + assertThat(result.tracking().status()).isEqualTo(EntryStatus.EXISTING); assertThat(result.tracking().firstRowId()).isNull(); } @@ -973,6 +978,50 @@ void manifestTrackedFileAdapterFailsWhenAddedFilesCountMissing() { .hasMessageContaining("missing added files count"); } + @Test + void manifestTrackedFileAdapterRejectsNullReplacedFilesCount() { + ManifestFile manifest = manifestWithCounts(1, 1, 0, null, 0); + + assertThatThrownBy(() -> TrackedFileAdapters.forManifestFile().wrap(manifest)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage( + "Cannot convert manifest %s: replaced files count must be 0 for v3 or earlier manifests, was null", + MANIFEST_LOCATION); + } + + @Test + void manifestTrackedFileAdapterRejectsNonZeroReplacedFilesCount() { + ManifestFile manifest = manifestWithCounts(1, 1, 0, 1, 0); + + assertThatThrownBy(() -> TrackedFileAdapters.forManifestFile().wrap(manifest)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage( + "Cannot convert manifest %s: replaced files count must be 0 for v3 or earlier manifests, was 1", + MANIFEST_LOCATION); + } + + @Test + void manifestTrackedFileAdapterRejectsNullModifiedFilesCount() { + ManifestFile manifest = manifestWithCounts(1, 1, 0, 0, null); + + assertThatThrownBy(() -> TrackedFileAdapters.forManifestFile().wrap(manifest)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage( + "Cannot convert manifest %s: modified files count must be 0 for v3 or earlier manifests, was null", + MANIFEST_LOCATION); + } + + @Test + void manifestTrackedFileAdapterRejectsNonZeroModifiedFilesCount() { + ManifestFile manifest = manifestWithCounts(1, 1, 0, 0, 1); + + assertThatThrownBy(() -> TrackedFileAdapters.forManifestFile().wrap(manifest)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage( + "Cannot convert manifest %s: modified files count must be 0 for v3 or earlier manifests, was 1", + MANIFEST_LOCATION); + } + @Test void manifestTrackedFileAdapterFailsWhenAddedRowsCountMissing() { ManifestFile manifest = writeManifestFile(ManifestContent.DATA, 5L, 4L, null); @@ -998,6 +1047,23 @@ private static void assertWrappedDataFileMatchesFileFields(TrackedFile result, D assertThat(result.equalityIds()).isNull(); } + private static ManifestFile manifestWithCounts( + Integer addedFilesCount, + Integer existingFilesCount, + Integer deletedFilesCount, + Integer replacedFilesCount, + Integer modifiedFilesCount) { + ManifestFile manifest = mock(ManifestFile.class); + when(manifest.path()).thenReturn(MANIFEST_LOCATION); + when(manifest.content()).thenReturn(ManifestContent.DATA); + when(manifest.addedFilesCount()).thenReturn(addedFilesCount); + when(manifest.existingFilesCount()).thenReturn(existingFilesCount); + when(manifest.deletedFilesCount()).thenReturn(deletedFilesCount); + when(manifest.replacedFilesCount()).thenReturn(replacedFilesCount); + when(manifest.modifiedFilesCount()).thenReturn(modifiedFilesCount); + return manifest; + } + private static ManifestFile writeManifestFile(ManifestContent content) { return writeManifestFile(content, 5L, 4L, 200L); } @@ -1045,6 +1111,17 @@ private static ManifestEntry deletedEntry(DataFile file) { return newEntry().wrapDelete(SNAPSHOT_ID, DATA_SEQUENCE_NUMBER, FILE_SEQUENCE_NUMBER, file); } + private static void assignManifestPosition(DataFile file, String location, long manifestPos) { + GenericDataFile dataFile = (GenericDataFile) file; + dataFile.setManifestLocation(location); + dataFile.set(DATA_FILE_POS_ORDINAL, manifestPos); + } + + private static void assertManifestPosition(Tracking tracking, DataFile file) { + assertThat(tracking.manifestLocation()).isEqualTo(file.manifestLocation()); + assertThat(tracking.manifestPos()).isEqualTo(file.pos()); + } + private static void assertNullTrackingFields(ContentFile file) { assertThat(file.pos()).isNull(); assertThat(file.manifestLocation()).isNull(); From 953e59d88c3a33e54a9a6584fa80fa6dc98a33a0 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Mon, 5 Oct 2026 15:28:19 -0700 Subject: [PATCH 06/11] Core: Decode geo bounds through Conversions with the column type. Geometry and geography bounds are already decoded by Conversions when given the column type, so decodeBound no longer needs a single-point special case. Generated-by: Cursor Grok 4.7 Co-authored-by: Cursor --- .../apache/iceberg/MapBackedContentStats.java | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/MapBackedContentStats.java b/core/src/main/java/org/apache/iceberg/MapBackedContentStats.java index 4f8d7504323d..567f92d59780 100644 --- a/core/src/main/java/org/apache/iceberg/MapBackedContentStats.java +++ b/core/src/main/java/org/apache/iceberg/MapBackedContentStats.java @@ -19,11 +19,9 @@ package org.apache.iceberg; import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.util.Map; import java.util.Objects; import java.util.Set; -import org.apache.iceberg.geospatial.GeospatialBound; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.relocated.com.google.common.collect.Maps; @@ -180,19 +178,7 @@ private T decodeBound(Map bounds) { return null; } - ByteBuffer buffer = bounds.get(fieldId); - if (columnType.typeId() == Type.TypeID.GEOMETRY - || columnType.typeId() == Type.TypeID.GEOGRAPHY) { - // File-map geometry and geography bounds are single-point encoded. - if (buffer == null) { - return null; - } - - ByteBuffer tmp = buffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); - return (T) GeospatialBound.fromByteBuffer(tmp); - } - - return (T) Conversions.fromByteBuffer(boundType, buffer); + return (T) Conversions.fromByteBuffer(columnType, bounds.get(fieldId)); } @Override From 1bd756d535732da70c5df76b144e2b5bb3d7eee0 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Mon, 5 Oct 2026 22:29:03 -0700 Subject: [PATCH 07/11] Core: Return map-backed stats without building the stats type. Look columns up by field id so field stats can be read from the maps without constructing the content-stats type first. Generated-by: Cursor Grok 4.7 Co-authored-by: Cursor --- .../apache/iceberg/MapBackedContentStats.java | 20 +++++++++---------- .../iceberg/TestMapBackedContentStats.java | 19 ++++++++++-------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/MapBackedContentStats.java b/core/src/main/java/org/apache/iceberg/MapBackedContentStats.java index 567f92d59780..e78299821ae8 100644 --- a/core/src/main/java/org/apache/iceberg/MapBackedContentStats.java +++ b/core/src/main/java/org/apache/iceberg/MapBackedContentStats.java @@ -62,16 +62,13 @@ MapBackedContentStats wrap(ContentFile file) { @Override public Iterable> fieldStats() { - return Iterables.filter( - Iterables.transform( - type().fields(), field -> statsFor(StatsUtil.toFieldId(field.fieldId()))), - Objects::nonNull); + return Iterables.filter(Iterables.transform(statsFieldIds(), this::statsFor), Objects::nonNull); } @Override @SuppressWarnings("unchecked") public FieldStats statsFor(int fieldId) { - if (!containsFieldInMaps(fieldId) || type().field(StatsUtil.toBaseId(fieldId)) == null) { + if (!containsFieldInMaps(fieldId) || tableSchema.findField(fieldId) == null) { return null; } @@ -113,7 +110,8 @@ boolean containsFieldInMaps(int fieldId) { } private Set statsFieldIds() { - Set ids = Sets.newTreeSet(); + Set ids = + Sets.newHashSetWithExpectedSize(valueCounts == null ? 0 : valueCounts.size()); addKeys(ids, valueCounts); addKeys(ids, nullValueCounts); addKeys(ids, nanValueCounts); @@ -141,15 +139,15 @@ private class MapBackedFieldStats implements FieldStats { private final Type boundType; MapBackedFieldStats(int fieldId) { - Types.NestedField field = type.field(StatsUtil.toBaseId(fieldId)); - Preconditions.checkArgument( - field != null, "Cannot convert stats for field ID %s: unknown or not a scalar", fieldId); Types.NestedField column = tableSchema.findField(fieldId); Preconditions.checkArgument(column != null, "Missing column for field ID %s", fieldId); this.fieldId = fieldId; - this.struct = field.type().asStructType(); this.columnType = column.type(); - this.boundType = struct.fieldType(StatsUtil.LOWER_BOUND_NAME); + int baseId = StatsUtil.toBaseId(fieldId); + this.struct = StatsUtil.fieldStatsStruct(columnType, baseId, MetricsModes.Full.get()); + Types.NestedField lowerBound = + struct == null ? null : struct.field(baseId + StatsUtil.LOWER_BOUND_OFFSET); + this.boundType = lowerBound == null ? null : lowerBound.type(); } @Override diff --git a/core/src/test/java/org/apache/iceberg/TestMapBackedContentStats.java b/core/src/test/java/org/apache/iceberg/TestMapBackedContentStats.java index b43e8f4c2f6e..eac556387ceb 100644 --- a/core/src/test/java/org/apache/iceberg/TestMapBackedContentStats.java +++ b/core/src/test/java/org/apache/iceberg/TestMapBackedContentStats.java @@ -69,20 +69,22 @@ class TestMapBackedContentStats { void contentStatsTypeBuiltLazilyFromMapIds() { MapBackedContentStats stats = new MapBackedContentStats(SCHEMA); - assertThat(stats.type()).isEqualTo(StatsUtil.statsReadSchema(SCHEMA, ImmutableList.of())); + assertThat(stats.type().fields()).isEmpty(); stats.wrap(FILE_WITH_STATS); - assertThat(stats.type()) - .isEqualTo(StatsUtil.statsReadSchema(SCHEMA, ImmutableList.of(1, 2, 3, 4))); + assertThat(stats.type().fields()) + .extracting(field -> StatsUtil.toFieldId(field.fieldId())) + .containsExactlyInAnyOrder(1, 2, 3, 4); } @Test void wrapInvalidatesType() { MapBackedContentStats stats = new MapBackedContentStats(SCHEMA).wrap(FILE_WITH_STATS); Types.StructType firstType = stats.type(); - assertThat(firstType) - .isEqualTo(StatsUtil.statsReadSchema(SCHEMA, ImmutableList.of(1, 2, 3, 4))); + assertThat(firstType.fields()) + .extracting(field -> StatsUtil.toFieldId(field.fieldId())) + .containsExactlyInAnyOrder(1, 2, 3, 4); DataFile file2 = dataFile( @@ -93,9 +95,10 @@ void wrapInvalidatesType() { ImmutableMap.of(1, buf(Types.IntegerType.get(), 5000))); stats.wrap(file2); - assertThat(stats.type()) - .isEqualTo(StatsUtil.statsReadSchema(SCHEMA, ImmutableList.of(1))) - .isNotEqualTo(firstType); + assertThat(stats.type().fields()) + .extracting(field -> StatsUtil.toFieldId(field.fieldId())) + .containsExactlyInAnyOrder(1); + assertThat(stats.type()).isNotEqualTo(firstType); } @Test From 3008d1549fdd34feae15656bf615642ea27963eb Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Tue, 6 Oct 2026 12:10:44 -0700 Subject: [PATCH 08/11] Core: Reuse field stats across file wrap Keep cached field stats when rebinding a file, and return null from a factory when the field has no stats struct. Look up the bound type by name. Generated-by: Cursor Grok 4.7 Co-authored-by: Cursor --- .../apache/iceberg/MapBackedContentStats.java | 49 +++--- .../iceberg/TestMapBackedContentStats.java | 149 ++++++++++++++++++ 2 files changed, 180 insertions(+), 18 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/MapBackedContentStats.java b/core/src/main/java/org/apache/iceberg/MapBackedContentStats.java index e78299821ae8..2a41a8f646aa 100644 --- a/core/src/main/java/org/apache/iceberg/MapBackedContentStats.java +++ b/core/src/main/java/org/apache/iceberg/MapBackedContentStats.java @@ -56,7 +56,6 @@ MapBackedContentStats wrap(ContentFile file) { this.lowerBounds = file.lowerBounds(); this.upperBounds = file.upperBounds(); this.type = null; - statsById.clear(); return this; } @@ -68,17 +67,36 @@ public Iterable> fieldStats() { @Override @SuppressWarnings("unchecked") public FieldStats statsFor(int fieldId) { - if (!containsFieldInMaps(fieldId) || tableSchema.findField(fieldId) == null) { + // Schema is fixed for this instance; wrap() rebinds the metric maps. An id absent + // from the current maps is left uncached so a later file can still surface it. + if (!containsFieldInMaps(fieldId)) { return null; } - FieldStats fieldStats = statsById.get(fieldId); - if (fieldStats == null) { - fieldStats = new MapBackedFieldStats<>(fieldId); - statsById.put(fieldId, fieldStats); + if (!statsById.containsKey(fieldId)) { + statsById.put(fieldId, createFieldStats(fieldId)); } - return (FieldStats) fieldStats; + return (FieldStats) statsById.get(fieldId); + } + + FieldStats createFieldStats(int fieldId) { + Types.NestedField field = tableSchema.findField(fieldId); + // A file can carry metrics for an id this schema does not have. Skip it. + if (field == null) { + return null; + } + + Type fieldType = field.type(); + Types.StructType struct = + StatsUtil.fieldStatsStruct(fieldType, StatsUtil.toBaseId(fieldId), MetricsModes.Full.get()); + // null means a struct, list, or map, or an id outside the stats window. The schema is bound + // once during wrapper creation, so the cached null stays valid across wrap(). + if (struct == null) { + return null; + } + + return new MapBackedFieldStats<>(fieldId, fieldType, struct); } @Override @@ -135,19 +153,14 @@ private static boolean containsId(Map map, int id) { private class MapBackedFieldStats implements FieldStats { private final int fieldId; private final Types.StructType struct; - private final Type columnType; + private final Type fieldType; private final Type boundType; - MapBackedFieldStats(int fieldId) { - Types.NestedField column = tableSchema.findField(fieldId); - Preconditions.checkArgument(column != null, "Missing column for field ID %s", fieldId); + private MapBackedFieldStats(int fieldId, Type fieldType, Types.StructType struct) { this.fieldId = fieldId; - this.columnType = column.type(); - int baseId = StatsUtil.toBaseId(fieldId); - this.struct = StatsUtil.fieldStatsStruct(columnType, baseId, MetricsModes.Full.get()); - Types.NestedField lowerBound = - struct == null ? null : struct.field(baseId + StatsUtil.LOWER_BOUND_OFFSET); - this.boundType = lowerBound == null ? null : lowerBound.type(); + this.fieldType = fieldType; + this.struct = struct; + this.boundType = struct.fieldType(StatsUtil.LOWER_BOUND_NAME); } @Override @@ -176,7 +189,7 @@ private T decodeBound(Map bounds) { return null; } - return (T) Conversions.fromByteBuffer(columnType, bounds.get(fieldId)); + return (T) Conversions.fromByteBuffer(fieldType, bounds.get(fieldId)); } @Override diff --git a/core/src/test/java/org/apache/iceberg/TestMapBackedContentStats.java b/core/src/test/java/org/apache/iceberg/TestMapBackedContentStats.java index eac556387ceb..246aeca8b7ed 100644 --- a/core/src/test/java/org/apache/iceberg/TestMapBackedContentStats.java +++ b/core/src/test/java/org/apache/iceberg/TestMapBackedContentStats.java @@ -268,6 +268,155 @@ void reuseRebindsBounds() { assertThat(stats.statsFor(2)).isNull(); } + @Test + void wrapReusesFieldStats() { + MapBackedContentStats stats = new MapBackedContentStats(SCHEMA); + stats.wrap(FILE_WITH_STATS); + FieldStats id = stats.statsFor(1); + FieldStats score = stats.statsFor(2); + FieldStats ts = stats.statsFor(3); + FieldStats name = stats.statsFor(4); + + DataFile file2 = + dataFile( + ImmutableMap.of(1, 50L, 2, 40L, 3, 20L, 4, 30L), + ImmutableMap.of(1, 9L, 2, 8L, 3, 7L, 4, 6L), + ImmutableMap.of(2, 11L), + ImmutableMap.of( + 1, buf(Types.IntegerType.get(), 500), + 2, buf(Types.FloatType.get(), 2.5f), + 3, buf(Types.LongType.get(), 200L), + 4, buf(Types.StringType.get(), "bbb")), + ImmutableMap.of( + 1, buf(Types.IntegerType.get(), 5000), + 2, buf(Types.FloatType.get(), 8.5f), + 3, buf(Types.LongType.get(), 800L), + 4, buf(Types.StringType.get(), "yyy"))); + stats.wrap(file2); + + assertThat(stats.statsFor(1)).isSameAs(id); + assertThat(id.valueCount()).isEqualTo(50L); + assertThat(id.nullValueCount()).isEqualTo(9L); + assertThat(id.lowerBound()).isEqualTo(500); + assertThat(id.upperBound()).isEqualTo(5000); + + assertThat(stats.statsFor(2)).isSameAs(score); + assertThat(score.valueCount()).isEqualTo(40L); + assertThat(score.nullValueCount()).isEqualTo(8L); + assertThat(score.nanValueCount()).isEqualTo(11L); + assertThat(score.lowerBound()).isEqualTo(2.5f); + assertThat(score.upperBound()).isEqualTo(8.5f); + + assertThat(stats.statsFor(3)).isSameAs(ts); + assertThat(ts.valueCount()).isEqualTo(20L); + assertThat(ts.nullValueCount()).isEqualTo(7L); + assertThat(ts.lowerBound()).isEqualTo(200L); + assertThat(ts.upperBound()).isEqualTo(800L); + + assertThat(stats.statsFor(4)).isSameAs(name); + assertThat(name.valueCount()).isEqualTo(30L); + assertThat(name.nullValueCount()).isEqualTo(6L); + assertThat(name.lowerBound().toString()).isEqualTo("bbb"); + assertThat(name.upperBound().toString()).isEqualTo("yyy"); + } + + @Test + void absentIdIsRereadWhenPresentAgain() { + MapBackedContentStats stats = new MapBackedContentStats(SCHEMA); + stats.wrap(FILE_WITH_STATS); + FieldStats id = stats.statsFor(1); + assertThat(id).isNotNull(); + + stats.wrap( + dataFile( + ImmutableMap.of(2, 10L), + ImmutableMap.of(), + ImmutableMap.of(), + ImmutableMap.of(2, buf(Types.FloatType.get(), 0.0f)), + ImmutableMap.of(2, buf(Types.FloatType.get(), 1.0f)))); + assertThat(stats.statsFor(1)).isNull(); + + stats.wrap( + dataFile( + ImmutableMap.of(1, 7L), + ImmutableMap.of(), + ImmutableMap.of(), + ImmutableMap.of(1, buf(Types.IntegerType.get(), 42)), + ImmutableMap.of(1, buf(Types.IntegerType.get(), 43)))); + assertThat(stats.statsFor(1)).isSameAs(id); + assertThat(id.lowerBound()).isEqualTo(42); + assertThat(id.upperBound()).isEqualTo(43); + assertThat(id.valueCount()).isEqualTo(7L); + } + + @Test + void outOfRangeFieldStatsAreNullAndCached() { + int fieldId = 999_950; + Schema schema = + new Schema(Types.NestedField.optional(fieldId, "too_high", Types.IntegerType.get())); + DataFile file = + dataFile( + ImmutableMap.of(fieldId, 1L), + ImmutableMap.of(), + ImmutableMap.of(), + ImmutableMap.of(fieldId, buf(Types.IntegerType.get(), 1)), + ImmutableMap.of(fieldId, buf(Types.IntegerType.get(), 2))); + CountingContentStats stats = new CountingContentStats(schema); + stats.wrap(file); + + assertThat(stats.statsFor(fieldId)).isNull(); + assertThat(stats.creates).isEqualTo(1); + + DataFile file2 = + dataFile( + ImmutableMap.of(fieldId, 9L), + ImmutableMap.of(), + ImmutableMap.of(), + ImmutableMap.of(fieldId, buf(Types.IntegerType.get(), 30)), + ImmutableMap.of(fieldId, buf(Types.IntegerType.get(), 40))); + stats.wrap(file2); + + assertThat(stats.statsFor(fieldId)).isNull(); + assertThat(stats.creates).isEqualTo(1); + assertThat(stats.fieldStats()).isEmpty(); + assertThat(stats.creates).isEqualTo(1); + } + + @Test + void listElementFieldStats() { + Schema schema = + new Schema( + Types.NestedField.required( + 1, "nums", Types.ListType.ofRequired(2, Types.IntegerType.get()))); + DataFile file = + dataFile( + ImmutableMap.of(2, 4L), + ImmutableMap.of(), + ImmutableMap.of(), + ImmutableMap.of(2, buf(Types.IntegerType.get(), 1)), + ImmutableMap.of(2, buf(Types.IntegerType.get(), 9))); + MapBackedContentStats stats = new MapBackedContentStats(schema).wrap(file); + + FieldStats element = stats.statsFor(2); + assertThat(element.lowerBound()).isEqualTo(1); + assertThat(element.upperBound()).isEqualTo(9); + assertThat(stats.fieldStats()).extracting(FieldStats::fieldId).containsExactly(2); + } + + private static final class CountingContentStats extends MapBackedContentStats { + private int creates; + + private CountingContentStats(Schema tableSchema) { + super(tableSchema); + } + + @Override + FieldStats createFieldStats(int fieldId) { + creates += 1; + return super.createFieldStats(fieldId); + } + } + private static ByteBuffer buf(Type type, Object value) { return Conversions.toByteBuffer(type, value); } From 494c61e890719c4d4dd9f4f5a5fdde7bc50bc251 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Tue, 6 Oct 2026 18:39:06 -0700 Subject: [PATCH 09/11] Core: Tighten content stats and tracked file tests Assert mapped values instead of implementation details, matching review feedback on the v4 write-direction wrappers. Generated-by: Cursor Grok 4.7 Co-authored-by: Cursor --- .../iceberg/TestMapBackedContentStats.java | 243 +++++++----------- .../iceberg/TestTrackedFileAdapters.java | 213 ++++++--------- 2 files changed, 161 insertions(+), 295 deletions(-) diff --git a/core/src/test/java/org/apache/iceberg/TestMapBackedContentStats.java b/core/src/test/java/org/apache/iceberg/TestMapBackedContentStats.java index 246aeca8b7ed..33a10f815a02 100644 --- a/core/src/test/java/org/apache/iceberg/TestMapBackedContentStats.java +++ b/core/src/test/java/org/apache/iceberg/TestMapBackedContentStats.java @@ -21,16 +21,24 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import java.math.BigDecimal; import java.nio.ByteBuffer; +import java.util.Comparator; +import java.util.List; import java.util.Map; +import java.util.UUID; +import java.util.stream.Stream; import org.apache.iceberg.geospatial.GeospatialBound; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; +import org.apache.iceberg.types.Comparators; import org.apache.iceberg.types.Conversions; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; class TestMapBackedContentStats { @@ -46,11 +54,12 @@ class TestMapBackedContentStats { new PartitionData(PartitionSpec.unpartitioned().partitionType()); /** - * Stats on fields 1-4; field 5 is absent. Field 3 has no value_count entry so the absent-count - * getter throws. + * Stats on fields 1-4; field 5 is absent. Field 3 has no value-count entry, so valueCount() + * throws on unboxing. */ private static final DataFile FILE_WITH_STATS = dataFile( + 100L, ImmutableMap.of(1, 100L, 2, 100L, 4, 100L), ImmutableMap.of(2, 5L, 3, 1L, 4, 2L), ImmutableMap.of(2, 3L), @@ -66,7 +75,7 @@ class TestMapBackedContentStats { 4, buf(Types.StringType.get(), "zzz"))); @Test - void contentStatsTypeBuiltLazilyFromMapIds() { + void typeMatchesIdsPresentInMaps() { MapBackedContentStats stats = new MapBackedContentStats(SCHEMA); assertThat(stats.type().fields()).isEmpty(); @@ -74,8 +83,8 @@ void contentStatsTypeBuiltLazilyFromMapIds() { stats.wrap(FILE_WITH_STATS); assertThat(stats.type().fields()) - .extracting(field -> StatsUtil.toFieldId(field.fieldId())) - .containsExactlyInAnyOrder(1, 2, 3, 4); + .containsExactlyInAnyOrderElementsOf( + StatsUtil.statsReadSchema(SCHEMA, List.of(1, 2, 3, 4)).fields()); } @Test @@ -83,11 +92,12 @@ void wrapInvalidatesType() { MapBackedContentStats stats = new MapBackedContentStats(SCHEMA).wrap(FILE_WITH_STATS); Types.StructType firstType = stats.type(); assertThat(firstType.fields()) - .extracting(field -> StatsUtil.toFieldId(field.fieldId())) - .containsExactlyInAnyOrder(1, 2, 3, 4); + .containsExactlyInAnyOrderElementsOf( + StatsUtil.statsReadSchema(SCHEMA, List.of(1, 2, 3, 4)).fields()); DataFile file2 = dataFile( + 100L, ImmutableMap.of(1, 50L), ImmutableMap.of(), ImmutableMap.of(), @@ -96,32 +106,57 @@ void wrapInvalidatesType() { stats.wrap(file2); assertThat(stats.type().fields()) - .extracting(field -> StatsUtil.toFieldId(field.fieldId())) - .containsExactlyInAnyOrder(1); + .containsExactlyInAnyOrderElementsOf( + StatsUtil.statsReadSchema(SCHEMA, List.of(1)).fields()); assertThat(stats.type()).isNotEqualTo(firstType); } - @Test - void boundDecodingPerType() { - MapBackedContentStats stats = new MapBackedContentStats(SCHEMA).wrap(FILE_WITH_STATS); - - FieldStats id = stats.statsFor(1); - assertThat(id.lowerBound()).isInstanceOf(Integer.class).isEqualTo(1); - assertThat(id.upperBound()).isInstanceOf(Integer.class).isEqualTo(1000); + @ParameterizedTest + @MethodSource("typesAndBounds") + void boundDecodingPerType(Type type, Object lower, Object upper) { + int fieldId = 1; + Schema schema = new Schema(Types.NestedField.optional(fieldId, "col", type)); + DataFile file = + dataFile( + 100L, + ImmutableMap.of(fieldId, 1L), + ImmutableMap.of(), + ImmutableMap.of(), + bound(fieldId, Conversions.toByteBuffer(type, lower)), + bound(fieldId, Conversions.toByteBuffer(type, upper))); + FieldStats stats = new MapBackedContentStats(schema).wrap(file).statsFor(fieldId); - FieldStats score = stats.statsFor(2); - assertThat(score.lowerBound()).isInstanceOf(Float.class).isEqualTo(1.5f); - assertThat(score.upperBound()).isInstanceOf(Float.class).isEqualTo(9.5f); + Comparator comparator = Comparators.forType(type.asPrimitiveType()); + assertThat(comparator.compare(stats.lowerBound(), lower)).isZero(); + assertThat(comparator.compare(stats.upperBound(), upper)).isZero(); + } - FieldStats ts = stats.statsFor(3); - assertThat(ts.lowerBound()).isInstanceOf(Long.class).isEqualTo(100L); - assertThat(ts.upperBound()).isInstanceOf(Long.class).isEqualTo(999L); - - FieldStats name = stats.statsFor(4); - assertThat(name.lowerBound()).isInstanceOf(CharSequence.class); - assertThat(name.lowerBound().toString()).isEqualTo("aaa"); - assertThat(name.upperBound()).isInstanceOf(CharSequence.class); - assertThat(name.upperBound().toString()).isEqualTo("zzz"); + private static Stream typesAndBounds() { + return Stream.of( + Arguments.of(Types.BooleanType.get(), false, true), + Arguments.of(Types.IntegerType.get(), -5, 100), + Arguments.of(Types.LongType.get(), 0L, 1_000L), + Arguments.of(Types.FloatType.get(), 1.5f, 9.5f), + Arguments.of(Types.DoubleType.get(), 0.0d, 25.0d), + Arguments.of(Types.DateType.get(), 100, 200), + Arguments.of(Types.TimeType.get(), 1_000L, 2_000L), + Arguments.of(Types.TimestampType.withZone(), 111L, 222L), + Arguments.of(Types.TimestampNanoType.withZone(), 111L, 222L), + Arguments.of(Types.StringType.get(), "a", "z"), + Arguments.of( + Types.UUIDType.get(), + UUID.fromString("07ceab48-62b2-4219-9172-856e687c92ad"), + UUID.fromString("a9a7c24d-2869-4c66-8803-b1f6a36256ed")), + Arguments.of( + Types.FixedType.ofLength(4), + ByteBuffer.wrap(new byte[] {0, 1, 2, 3}), + ByteBuffer.wrap(new byte[] {4, 5, 6, 7})), + Arguments.of( + Types.BinaryType.get(), + ByteBuffer.wrap(new byte[] {1, 2}), + ByteBuffer.wrap(new byte[] {3, 4, 5})), + Arguments.of(Types.DecimalType.of(9, 2), new BigDecimal("1.23"), new BigDecimal("9.99")), + Arguments.of(Types.UnknownType.get(), null, null)); } @Test @@ -131,6 +166,7 @@ void geoBoundsDecode() { Schema schema = new Schema(Types.NestedField.optional(10, "geom", Types.GeometryType.crs84())); DataFile file = dataFile( + 100L, ImmutableMap.of(10, 26L), ImmutableMap.of(), ImmutableMap.of(), @@ -147,6 +183,7 @@ void geoBoundsDecode() { void missingBoundsDecodeToNull() { DataFile file = dataFile( + 100L, ImmutableMap.of(1, 100L), ImmutableMap.of(), ImmutableMap.of(), @@ -211,6 +248,7 @@ void fieldWithoutStatsIsExcluded() { void unknownFieldIdInMaps() { DataFile file = dataFile( + 100L, ImmutableMap.of(99, 1L), ImmutableMap.of(), ImmutableMap.of(), @@ -225,36 +263,17 @@ void unknownFieldIdInMaps() { } @Test - void copyNotSupported() { - MapBackedContentStats stats = new MapBackedContentStats(SCHEMA).wrap(FILE_WITH_STATS); - - assertThatThrownBy(stats::copy) - .isInstanceOf(UnsupportedOperationException.class) - .hasMessage("copy is not implemented"); - - assertThatThrownBy(() -> stats.copy(ImmutableSet.of(1))) - .isInstanceOf(UnsupportedOperationException.class) - .hasMessage("copy is not implemented"); - } - - @Test - void fieldStatsCopyNotSupported() { - MapBackedContentStats stats = new MapBackedContentStats(SCHEMA).wrap(FILE_WITH_STATS); - - assertThatThrownBy(stats.statsFor(1)::copy) - .isInstanceOf(UnsupportedOperationException.class) - .hasMessage("copy is not implemented"); - } - - @Test - void reuseRebindsBounds() { + void reuseRebindsFields() { MapBackedContentStats stats = new MapBackedContentStats(SCHEMA); stats.wrap(FILE_WITH_STATS); assertThat(stats.statsFor(1).lowerBound()).isEqualTo(1); + assertThat(stats.statsFor(1).upperBound()).isEqualTo(1000); + assertThat(stats.statsFor(1).valueCount()).isEqualTo(100L); DataFile file2 = dataFile( + 100L, ImmutableMap.of(1, 50L), ImmutableMap.of(), ImmutableMap.of(), @@ -268,67 +287,15 @@ void reuseRebindsBounds() { assertThat(stats.statsFor(2)).isNull(); } - @Test - void wrapReusesFieldStats() { - MapBackedContentStats stats = new MapBackedContentStats(SCHEMA); - stats.wrap(FILE_WITH_STATS); - FieldStats id = stats.statsFor(1); - FieldStats score = stats.statsFor(2); - FieldStats ts = stats.statsFor(3); - FieldStats name = stats.statsFor(4); - - DataFile file2 = - dataFile( - ImmutableMap.of(1, 50L, 2, 40L, 3, 20L, 4, 30L), - ImmutableMap.of(1, 9L, 2, 8L, 3, 7L, 4, 6L), - ImmutableMap.of(2, 11L), - ImmutableMap.of( - 1, buf(Types.IntegerType.get(), 500), - 2, buf(Types.FloatType.get(), 2.5f), - 3, buf(Types.LongType.get(), 200L), - 4, buf(Types.StringType.get(), "bbb")), - ImmutableMap.of( - 1, buf(Types.IntegerType.get(), 5000), - 2, buf(Types.FloatType.get(), 8.5f), - 3, buf(Types.LongType.get(), 800L), - 4, buf(Types.StringType.get(), "yyy"))); - stats.wrap(file2); - - assertThat(stats.statsFor(1)).isSameAs(id); - assertThat(id.valueCount()).isEqualTo(50L); - assertThat(id.nullValueCount()).isEqualTo(9L); - assertThat(id.lowerBound()).isEqualTo(500); - assertThat(id.upperBound()).isEqualTo(5000); - - assertThat(stats.statsFor(2)).isSameAs(score); - assertThat(score.valueCount()).isEqualTo(40L); - assertThat(score.nullValueCount()).isEqualTo(8L); - assertThat(score.nanValueCount()).isEqualTo(11L); - assertThat(score.lowerBound()).isEqualTo(2.5f); - assertThat(score.upperBound()).isEqualTo(8.5f); - - assertThat(stats.statsFor(3)).isSameAs(ts); - assertThat(ts.valueCount()).isEqualTo(20L); - assertThat(ts.nullValueCount()).isEqualTo(7L); - assertThat(ts.lowerBound()).isEqualTo(200L); - assertThat(ts.upperBound()).isEqualTo(800L); - - assertThat(stats.statsFor(4)).isSameAs(name); - assertThat(name.valueCount()).isEqualTo(30L); - assertThat(name.nullValueCount()).isEqualTo(6L); - assertThat(name.lowerBound().toString()).isEqualTo("bbb"); - assertThat(name.upperBound().toString()).isEqualTo("yyy"); - } - @Test void absentIdIsRereadWhenPresentAgain() { MapBackedContentStats stats = new MapBackedContentStats(SCHEMA); stats.wrap(FILE_WITH_STATS); - FieldStats id = stats.statsFor(1); - assertThat(id).isNotNull(); + assertThat(stats.statsFor(1)).isNotNull(); stats.wrap( dataFile( + 100L, ImmutableMap.of(2, 10L), ImmutableMap.of(), ImmutableMap.of(), @@ -338,50 +305,18 @@ void absentIdIsRereadWhenPresentAgain() { stats.wrap( dataFile( + 100L, ImmutableMap.of(1, 7L), ImmutableMap.of(), ImmutableMap.of(), ImmutableMap.of(1, buf(Types.IntegerType.get(), 42)), ImmutableMap.of(1, buf(Types.IntegerType.get(), 43)))); - assertThat(stats.statsFor(1)).isSameAs(id); + FieldStats id = stats.statsFor(1); assertThat(id.lowerBound()).isEqualTo(42); assertThat(id.upperBound()).isEqualTo(43); assertThat(id.valueCount()).isEqualTo(7L); } - @Test - void outOfRangeFieldStatsAreNullAndCached() { - int fieldId = 999_950; - Schema schema = - new Schema(Types.NestedField.optional(fieldId, "too_high", Types.IntegerType.get())); - DataFile file = - dataFile( - ImmutableMap.of(fieldId, 1L), - ImmutableMap.of(), - ImmutableMap.of(), - ImmutableMap.of(fieldId, buf(Types.IntegerType.get(), 1)), - ImmutableMap.of(fieldId, buf(Types.IntegerType.get(), 2))); - CountingContentStats stats = new CountingContentStats(schema); - stats.wrap(file); - - assertThat(stats.statsFor(fieldId)).isNull(); - assertThat(stats.creates).isEqualTo(1); - - DataFile file2 = - dataFile( - ImmutableMap.of(fieldId, 9L), - ImmutableMap.of(), - ImmutableMap.of(), - ImmutableMap.of(fieldId, buf(Types.IntegerType.get(), 30)), - ImmutableMap.of(fieldId, buf(Types.IntegerType.get(), 40))); - stats.wrap(file2); - - assertThat(stats.statsFor(fieldId)).isNull(); - assertThat(stats.creates).isEqualTo(1); - assertThat(stats.fieldStats()).isEmpty(); - assertThat(stats.creates).isEqualTo(1); - } - @Test void listElementFieldStats() { Schema schema = @@ -390,6 +325,7 @@ void listElementFieldStats() { 1, "nums", Types.ListType.ofRequired(2, Types.IntegerType.get()))); DataFile file = dataFile( + 100L, ImmutableMap.of(2, 4L), ImmutableMap.of(), ImmutableMap.of(), @@ -403,25 +339,16 @@ void listElementFieldStats() { assertThat(stats.fieldStats()).extracting(FieldStats::fieldId).containsExactly(2); } - private static final class CountingContentStats extends MapBackedContentStats { - private int creates; - - private CountingContentStats(Schema tableSchema) { - super(tableSchema); - } - - @Override - FieldStats createFieldStats(int fieldId) { - creates += 1; - return super.createFieldStats(fieldId); - } - } - private static ByteBuffer buf(Type type, Object value) { return Conversions.toByteBuffer(type, value); } + private static Map bound(int fieldId, ByteBuffer value) { + return value == null ? ImmutableMap.of() : ImmutableMap.of(fieldId, value); + } + private static DataFile dataFile( + long recordCount, Map valueCounts, Map nullValueCounts, Map nanValueCounts, @@ -429,7 +356,13 @@ private static DataFile dataFile( Map upperBounds) { Metrics metrics = new Metrics( - 100L, null, valueCounts, nullValueCounts, nanValueCounts, lowerBounds, upperBounds); + recordCount, + null, + valueCounts, + nullValueCounts, + nanValueCounts, + lowerBounds, + upperBounds); return new GenericDataFile( PartitionSpec.unpartitioned().specId(), "s3://bucket/data/file.parquet", diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java index 168fd257ef97..fa9eeb2e8268 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java @@ -30,6 +30,7 @@ import org.apache.iceberg.mumbling.MumblingTestUtil; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.types.Comparators; import org.apache.iceberg.types.Conversions; import org.apache.iceberg.types.Types; import org.junit.jupiter.api.Test; @@ -51,6 +52,16 @@ class TestTrackedFileAdapters { private static final long DATA_SEQUENCE_NUMBER = 10L; private static final long FILE_SEQUENCE_NUMBER = 11L; private static final long FIRST_ROW_ID = 1000L; + private static final int SORT_ORDER_ID = 3; + + private static final long MANIFEST_SEQUENCE_NUMBER = 5L; + private static final long MANIFEST_MIN_SEQUENCE_NUMBER = 4L; + private static final int ADDED_FILES_COUNT = 2; + private static final long ADDED_ROWS_COUNT = 200L; + private static final int EXISTING_FILES_COUNT = 3; + private static final long EXISTING_ROWS_COUNT = 300L; + private static final int DELETED_FILES_COUNT = 1; + private static final long DELETED_ROWS_COUNT = 100L; private static final int UNPARTITIONED_SPEC_ID = PartitionSpec.unpartitioned().specId(); private static final Map UNPARTITIONED = @@ -149,11 +160,11 @@ class TestTrackedFileAdapters { FileFormat.PARQUET, PartitionData.EMPTY, 1024L, - new Metrics(100L, null, null, null, null), - null, + new Metrics(100L), + KEY_METADATA, ImmutableList.of(0L), - null, - null); + SORT_ORDER_ID, + FIRST_ROW_ID); private static final DataFile DATA_FILE_WITH_METRICS = new GenericDataFile( UNPARTITIONED_SPEC.specId(), @@ -246,7 +257,7 @@ void dataFileAdapterDelegation() { @ParameterizedTest @EnumSource(value = FileContent.class, mode = EnumSource.Mode.EXCLUDE, names = "DATA") void dataFileAdapterRejectsNonDataContent(FileContent contentType) { - TrackedFileStruct file = dummyTrackedFile(contentType); + TrackedFileStruct file = trackedFile(contentType); assertThatThrownBy(() -> TrackedFileAdapters.asDataFile(file, UNPARTITIONED)) .isInstanceOf(IllegalArgumentException.class) @@ -326,7 +337,7 @@ void equalityDeleteFileAdapterDelegation() { @ParameterizedTest @EnumSource(value = FileContent.class, mode = EnumSource.Mode.EXCLUDE, names = "EQUALITY_DELETES") void equalityDeleteFileAdapterRejectsNonEqualityContent(FileContent contentType) { - TrackedFileStruct file = dummyTrackedFile(contentType); + TrackedFileStruct file = trackedFile(contentType); assertThatThrownBy(() -> TrackedFileAdapters.asEqualityDeleteFile(file, UNPARTITIONED)) .isInstanceOf(IllegalArgumentException.class) @@ -512,7 +523,7 @@ private static TrackedFile serializableDataEntry() { @ParameterizedTest @EnumSource(value = FileContent.class, mode = EnumSource.Mode.EXCLUDE, names = "DATA") void dvDeleteFileAdapterRejectsNonDataContent(FileContent contentType) { - TrackedFileStruct file = dummyTrackedFile(contentType); + TrackedFileStruct file = trackedFile(contentType); assertThatThrownBy(() -> TrackedFileAdapters.asDVDeleteFile(file, UNPARTITIONED)) .isInstanceOf(IllegalArgumentException.class) @@ -521,7 +532,7 @@ void dvDeleteFileAdapterRejectsNonDataContent(FileContent contentType) { @Test void dvDeleteFileAdapterRejectsNullDeletionVector() { - TrackedFileStruct file = dummyTrackedFile(FileContent.DATA); + TrackedFileStruct file = trackedFile(FileContent.DATA); assertThatThrownBy(() -> TrackedFileAdapters.asDVDeleteFile(file, UNPARTITIONED)) .isInstanceOf(IllegalArgumentException.class) @@ -583,13 +594,6 @@ void manifestFileAdapterDelegation(FileContent contentType) { .hasMessage("v4 manifests are not bound to a single partition spec"); } - @Test - void manifestFileAdapterKeepsZeroFormatVersion() { - TrackedFile original = dummyTrackedFile(FileContent.DATA_MANIFEST, 0); - - assertThat(TrackedFileAdapters.asManifestFile(original).formatVersion()).isZero(); - } - @Test void manifestFileAdapterCopy() { TrackedFile file = Mockito.mock(TrackedFile.class); @@ -611,7 +615,7 @@ void manifestFileAdapterCopy() { mode = EnumSource.Mode.EXCLUDE, names = {"DATA_MANIFEST", "DELETE_MANIFEST"}) void manifestFileAdapterRejectsNonManifestContent(FileContent contentType) { - TrackedFileStruct file = dummyTrackedFile(contentType); + TrackedFileStruct file = trackedFile(contentType); assertThatThrownBy(() -> TrackedFileAdapters.asManifestFile(file)) .isInstanceOf(IllegalArgumentException.class) @@ -630,7 +634,7 @@ void dataFileWithoutDeletionVectorReturnsNull() { @Test void nullContentStatsReturnsNullStats() { - TrackedFileStruct file = dummyTrackedFile(FileContent.DATA); + TrackedFileStruct file = trackedFile(FileContent.DATA); DataFile dataFile = TrackedFileAdapters.asDataFile(file, UNPARTITIONED); @@ -646,10 +650,10 @@ void nullTrackingReturnsNullTrackingFields() { // Files read before manifest inheritance have no tracking; tracking-derived fields must be // null rather than throwing. assertNullTrackingFields( - TrackedFileAdapters.asDataFile(dummyTrackedFile(FileContent.DATA), UNPARTITIONED)); + TrackedFileAdapters.asDataFile(trackedFile(FileContent.DATA), UNPARTITIONED)); assertNullTrackingFields( TrackedFileAdapters.asEqualityDeleteFile( - dummyTrackedFile(FileContent.EQUALITY_DELETES), UNPARTITIONED)); + trackedFile(FileContent.EQUALITY_DELETES), UNPARTITIONED)); TrackedFileStruct fileWithDV = new TrackedFileStruct( @@ -674,7 +678,7 @@ void nullTrackingReturnsNullTrackingFields() { @Test void unpartitionedFilePartitionIsEmpty() { - TrackedFileStruct file = dummyTrackedFile(FileContent.DATA); + TrackedFileStruct file = trackedFile(FileContent.DATA); DataFile dataFile = TrackedFileAdapters.asDataFile(file, UNPARTITIONED); @@ -685,7 +689,7 @@ void unpartitionedFilePartitionIsEmpty() { @Test void nullSpecIdResolvesToUnpartitionedSpec() { PartitionSpec unpartitioned = PartitionSpec.builderFor(new Schema()).withSpecId(5).build(); - TrackedFileStruct file = dummyTrackedFile(FileContent.DATA); + TrackedFileStruct file = trackedFile(FileContent.DATA); DataFile dataFile = TrackedFileAdapters.asDataFile(file, specsById(unpartitioned)); @@ -696,7 +700,7 @@ void nullSpecIdResolvesToUnpartitionedSpec() { void nullSpecIdThrowsWhenNoUnpartitionedSpec() { Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); PartitionSpec partitioned = PartitionSpec.builderFor(schema).identity("id").build(); - TrackedFileStruct file = dummyTrackedFile(FileContent.DATA); + TrackedFileStruct file = trackedFile(FileContent.DATA); assertThatThrownBy(() -> TrackedFileAdapters.asDataFile(file, specsById(partitioned))) .isInstanceOf(IllegalArgumentException.class) @@ -730,7 +734,7 @@ void unknownSpecIdThrows() { } @Test - void dataTrackedFileAdapterHasNoTracking() { + void dataTrackedFileAdapterFromDataFile() { TrackedFile result = TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(DATA_FILE); assertThat(result.tracking()).isNull(); @@ -740,62 +744,30 @@ void dataTrackedFileAdapterHasNoTracking() { .hasMessage("Format version is assigned at write time"); } - @Test - void dataTrackedFileAdapterFromAddedManifestEntry() { - TrackedFile result = TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(addedEntry(DATA_FILE)); - - assertThat(result.tracking().status()).isEqualTo(EntryStatus.ADDED); - assertThat(result.tracking().snapshotId()).isEqualTo(SNAPSHOT_ID); - assertThat(result.tracking().dataSequenceNumber()).isNull(); - assertThat(result.tracking().fileSequenceNumber()).isNull(); - assertThat(result.tracking().firstRowId()).isNull(); - assertManifestPosition(result.tracking(), DATA_FILE); - assertWrappedDataFileMatchesFileFields(result, DATA_FILE); - } - - @Test - void dataTrackedFileAdapterFromAddedManifestEntryWithNullSnapshotId() { - TrackedFile result = - TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(newEntry().wrapAppend(null, DATA_FILE)); - - assertThat(result.tracking().status()).isEqualTo(EntryStatus.ADDED); - assertThat(result.tracking().snapshotId()).isNull(); - assertManifestPosition(result.tracking(), DATA_FILE); - assertWrappedDataFileMatchesFileFields(result, DATA_FILE); - } - @Test void dataTrackedFileAdapterFromExistingManifestEntry() { TrackedFile result = - TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(existingEntry(DATA_FILE)); + TrackedFileAdapters.forDataFile(TABLE_SCHEMA) + .wrap( + newEntry() + .wrapExisting( + SNAPSHOT_ID, DATA_SEQUENCE_NUMBER, FILE_SEQUENCE_NUMBER, DATA_FILE)); assertThat(result.tracking().status()).isEqualTo(EntryStatus.EXISTING); assertThat(result.tracking().snapshotId()).isEqualTo(SNAPSHOT_ID); assertThat(result.tracking().dataSequenceNumber()).isEqualTo(DATA_SEQUENCE_NUMBER); assertThat(result.tracking().fileSequenceNumber()).isEqualTo(FILE_SEQUENCE_NUMBER); + assertThat(result.tracking().firstRowId()).isEqualTo(FIRST_ROW_ID); assertManifestPosition(result.tracking(), DATA_FILE); assertWrappedDataFileMatchesFileFields(result, DATA_FILE); } - @Test - void dataTrackedFileAdapterFromDeletedManifestEntry() { - TrackedFile result = - TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(deletedEntry(DATA_FILE)); - - assertThat(result.tracking().status()).isEqualTo(EntryStatus.DELETED); - assertThat(result.tracking().snapshotId()).isEqualTo(SNAPSHOT_ID); - assertThat(result.tracking().dataSequenceNumber()).isEqualTo(DATA_SEQUENCE_NUMBER); - assertThat(result.tracking().fileSequenceNumber()).isEqualTo(FILE_SEQUENCE_NUMBER); - assertManifestPosition(result.tracking(), DATA_FILE); - } - @Test void dataTrackedFileAdapterReuse() { TrackedFileAdapters.DataTrackedFile adapter = TrackedFileAdapters.forDataFile(TABLE_SCHEMA); adapter.wrap(DATA_FILE); - assertThat(adapter.location()).isEqualTo(DATA_FILE_LOCATION); - assertThat(adapter.recordCount()).isEqualTo(100L); + assertWrappedDataFileMatchesFileFields(adapter, DATA_FILE); assertThat(adapter.tracking()).isNull(); DataFile file2 = @@ -811,13 +783,11 @@ void dataTrackedFileAdapterReuse() { null, null); assignManifestPosition(file2, "s3://bucket/table/manifest-2.parquet", 8L); - adapter.wrap(existingEntry(file2)); - assertThat(adapter.location()).isEqualTo("s3://bucket/data/file2.parquet"); - assertThat(adapter.recordCount()).isEqualTo(200L); + adapter.wrap( + newEntry().wrapExisting(SNAPSHOT_ID, DATA_SEQUENCE_NUMBER, FILE_SEQUENCE_NUMBER, file2)); + assertWrappedDataFileMatchesFileFields(adapter, file2); assertThat(adapter.tracking().status()).isEqualTo(EntryStatus.EXISTING); assertManifestPosition(adapter.tracking(), file2); - assertThat(adapter.fileSizeInBytes()).isEqualTo(2048L); - assertThat(adapter.location()).isNotEqualTo(DATA_FILE.location()); } @Test @@ -865,50 +835,25 @@ void dataTrackedFileAdapterKeepsPartitionTuple() { null); TrackedFile result = TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(partitioned); - assertThat(result.partition().get(0, String.class)).isEqualTo("books"); - assertThat( - TrackedFileAdapters.asDataFile(result, specsById(PARTITIONED_SPEC)) - .partition() - .get(0, String.class)) - .isEqualTo("books"); + assertThat(result.partition()) + .usingComparator(Comparators.forType(PARTITIONED_SPEC.partitionType())) + .isEqualTo(PARTITION); + assertThat(TrackedFileAdapters.asDataFile(result, specsById(PARTITIONED_SPEC)).partition()) + .usingComparator(Comparators.forType(PARTITIONED_SPEC.partitionType())) + .isEqualTo(PARTITION); } @Test - void dataFileDoubleWrapRoundTrip() { - DataFile source = DATA_FILE_WITH_METRICS; - Map specs = - ImmutableMap.of(UNPARTITIONED_SPEC.specId(), UNPARTITIONED_SPEC); - - TrackedFile tracked = TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(existingEntry(source)); - - DataFile roundTripped = TrackedFileAdapters.asDataFile(tracked, specs); - - assertThat(roundTripped.content()).isEqualTo(FileContent.DATA); - assertThat(roundTripped.location()).isEqualTo(source.location()); - assertThat(roundTripped.format()).isEqualTo(source.format()); - assertThat(roundTripped.recordCount()).isEqualTo(source.recordCount()); - assertThat(roundTripped.fileSizeInBytes()).isEqualTo(source.fileSizeInBytes()); - assertThat(roundTripped.specId()).isEqualTo(source.specId()); - assertThat(roundTripped.partition()).isEqualTo(source.partition()); - assertThat(roundTripped.sortOrderId()).isEqualTo(source.sortOrderId()); - assertThat(roundTripped.splitOffsets()).isEqualTo(source.splitOffsets()); - assertThat(roundTripped.keyMetadata()).isEqualTo(source.keyMetadata()); - assertThat(roundTripped.dataSequenceNumber()).isEqualTo(DATA_SEQUENCE_NUMBER); - assertThat(roundTripped.fileSequenceNumber()).isEqualTo(FILE_SEQUENCE_NUMBER); - assertThat(roundTripped.firstRowId()).isNull(); - assertThat(roundTripped.manifestLocation()).isEqualTo(source.manifestLocation()); - assertThat(roundTripped.pos()).isEqualTo(source.pos()); - assertThat(roundTripped.valueCounts()).containsAllEntriesOf(source.valueCounts()); - assertThat(roundTripped.nullValueCounts()).containsAllEntriesOf(source.nullValueCounts()); - assertThat(roundTripped.nanValueCounts()).isNull(); - assertThat(roundTripped.lowerBounds()).containsAllEntriesOf(source.lowerBounds()); - assertThat(roundTripped.upperBounds()).containsAllEntriesOf(source.upperBounds()); - assertThat(roundTripped.columnSizes()).isNull(); + void trackedFileDoubleWrapRoundTrip() { + TrackedFile source = trackedFile(FileContent.DATA); + DataFile dataFile = TrackedFileAdapters.asDataFile(source, UNPARTITIONED); + TrackedFile roundTripped = TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(dataFile); + assertThat(roundTripped).isSameAs(source); } @Test void dataTrackedFileAdapterUnwrapsToOriginalTrackedFile() { - TrackedFile original = dummyTrackedFile(FileContent.DATA); + TrackedFile original = trackedFile(FileContent.DATA); DataFile adapted = TrackedFileAdapters.asDataFile(original, UNPARTITIONED); TrackedFile result = TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(adapted); @@ -918,7 +863,7 @@ void dataTrackedFileAdapterUnwrapsToOriginalTrackedFile() { @Test void manifestTrackedFileAdapterUnwrapsToOriginalTrackedFile() { - TrackedFile original = dummyTrackedFile(FileContent.DATA_MANIFEST, 0); + TrackedFile original = trackedFile(FileContent.DATA_MANIFEST, 0); ManifestFile adapted = TrackedFileAdapters.asManifestFile(original); TrackedFile result = TrackedFileAdapters.forManifestFile().wrap(adapted); @@ -939,12 +884,13 @@ void dataManifestTrackedFileAdapter() { assertThat(result.tracking().status()).isEqualTo(EntryStatus.EXISTING); assertThat(result.tracking().snapshotId()).isEqualTo(SNAPSHOT_ID); assertThat(result.tracking().firstRowId()).isNull(); - assertThat(result.recordCount()).isEqualTo(6L); + assertThat(result.recordCount()) + .isEqualTo(ADDED_FILES_COUNT + EXISTING_FILES_COUNT + DELETED_FILES_COUNT); assertThat(result.manifestInfo()).isNotNull(); - assertThat(result.manifestInfo().addedFilesCount()).isEqualTo(2); - assertThat(result.manifestInfo().existingFilesCount()).isEqualTo(3); - assertThat(result.manifestInfo().deletedFilesCount()).isEqualTo(1); - assertThat(result.manifestInfo().addedRowsCount()).isEqualTo(200L); + assertThat(result.manifestInfo().addedFilesCount()).isEqualTo(ADDED_FILES_COUNT); + assertThat(result.manifestInfo().existingFilesCount()).isEqualTo(EXISTING_FILES_COUNT); + assertThat(result.manifestInfo().deletedFilesCount()).isEqualTo(DELETED_FILES_COUNT); + assertThat(result.manifestInfo().addedRowsCount()).isEqualTo(ADDED_ROWS_COUNT); assertThat(result.manifestInfo().replacedFilesCount()).isEqualTo(0); assertThat(result.manifestInfo().replacedRowsCount()).isEqualTo(0L); assertThat(result.manifestInfo().modifiedFilesCount()).isEqualTo(0); @@ -958,7 +904,8 @@ void deleteManifestTrackedFileAdapter() { assertThat(result.contentType()).isEqualTo(FileContent.DELETE_MANIFEST); assertThat(result.formatVersion()).isZero(); - assertThat(result.recordCount()).isEqualTo(6L); + assertThat(result.recordCount()) + .isEqualTo(ADDED_FILES_COUNT + EXISTING_FILES_COUNT + DELETED_FILES_COUNT); assertThat(result.tracking().status()).isEqualTo(EntryStatus.EXISTING); assertThat(result.tracking().firstRowId()).isNull(); } @@ -1024,7 +971,9 @@ void manifestTrackedFileAdapterRejectsNonZeroModifiedFilesCount() { @Test void manifestTrackedFileAdapterFailsWhenAddedRowsCountMissing() { - ManifestFile manifest = writeManifestFile(ManifestContent.DATA, 5L, 4L, null); + ManifestFile manifest = + writeManifestFile( + ManifestContent.DATA, MANIFEST_SEQUENCE_NUMBER, MANIFEST_MIN_SEQUENCE_NUMBER, null); TrackedFile tracked = TrackedFileAdapters.forManifestFile().wrap(manifest); assertThatThrownBy(() -> tracked.manifestInfo().addedRowsCount()) @@ -1065,12 +1014,8 @@ private static ManifestFile manifestWithCounts( } private static ManifestFile writeManifestFile(ManifestContent content) { - return writeManifestFile(content, 5L, 4L, 200L); - } - - private static ManifestFile writeManifestFile( - ManifestContent content, long sequenceNumber, long minSequenceNumber) { - return writeManifestFile(content, sequenceNumber, minSequenceNumber, 200L); + return writeManifestFile( + content, MANIFEST_SEQUENCE_NUMBER, MANIFEST_MIN_SEQUENCE_NUMBER, ADDED_ROWS_COUNT); } private static ManifestFile writeManifestFile( @@ -1078,7 +1023,7 @@ private static ManifestFile writeManifestFile( List partitions = ImmutableList.of(); return new GenericManifestFile( MANIFEST_LOCATION, - 2048L, + MANIFEST_FILE_SIZE, UNPARTITIONED_SPEC.specId(), content, sequenceNumber, @@ -1086,12 +1031,12 @@ private static ManifestFile writeManifestFile( SNAPSHOT_ID, partitions, null, - 2, + ADDED_FILES_COUNT, addedRowsCount, - 3, - 300L, - 1, - 100L, + EXISTING_FILES_COUNT, + EXISTING_ROWS_COUNT, + DELETED_FILES_COUNT, + DELETED_ROWS_COUNT, null); } @@ -1099,18 +1044,6 @@ private static GenericManifestEntry newEntry() { return new GenericManifestEntry<>(ManifestEntry.getSchema(PARTITION_TYPE).asStruct()); } - private static ManifestEntry addedEntry(DataFile file) { - return newEntry().wrapAppend(SNAPSHOT_ID, file); - } - - private static ManifestEntry existingEntry(DataFile file) { - return newEntry().wrapExisting(SNAPSHOT_ID, DATA_SEQUENCE_NUMBER, FILE_SEQUENCE_NUMBER, file); - } - - private static ManifestEntry deletedEntry(DataFile file) { - return newEntry().wrapDelete(SNAPSHOT_ID, DATA_SEQUENCE_NUMBER, FILE_SEQUENCE_NUMBER, file); - } - private static void assignManifestPosition(DataFile file, String location, long manifestPos) { GenericDataFile dataFile = (GenericDataFile) file; dataFile.setManifestLocation(location); @@ -1141,12 +1074,12 @@ private static PartitionData partition(String category) { return partition; } - /** Minimal file with no tracking, used by the rejection and null-tracking tests. */ - private static TrackedFileStruct dummyTrackedFile(FileContent contentType) { - return dummyTrackedFile(contentType, FORMAT_VERSION_V4); + /** Minimal file for the rejection and null-tracking tests. */ + private static TrackedFileStruct trackedFile(FileContent contentType) { + return trackedFile(contentType, FORMAT_VERSION_V4); } - private static TrackedFileStruct dummyTrackedFile(FileContent contentType, int formatVersion) { + private static TrackedFileStruct trackedFile(FileContent contentType, int formatVersion) { return new TrackedFileStruct( null, contentType, From 0a19d5afda22071afc80285e5f2d2b5a33002e26 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Tue, 6 Oct 2026 19:10:54 -0700 Subject: [PATCH 10/11] Core: Make field-stats helpers private and shorten manifest errors The helpers are only used inside MapBackedContentStats. The manifest count errors stay valid for a null count. Generated-by: Cursor Grok 4.7 Co-authored-by: Cursor --- .../org/apache/iceberg/MapBackedContentStats.java | 4 ++-- .../java/org/apache/iceberg/TrackedFileAdapters.java | 4 ++-- .../apache/iceberg/TestMapBackedContentStats.java | 2 -- .../org/apache/iceberg/TestTrackedFileAdapters.java | 12 ++++-------- 4 files changed, 8 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/MapBackedContentStats.java b/core/src/main/java/org/apache/iceberg/MapBackedContentStats.java index 2a41a8f646aa..66e9b6694f5f 100644 --- a/core/src/main/java/org/apache/iceberg/MapBackedContentStats.java +++ b/core/src/main/java/org/apache/iceberg/MapBackedContentStats.java @@ -80,7 +80,7 @@ public FieldStats statsFor(int fieldId) { return (FieldStats) statsById.get(fieldId); } - FieldStats createFieldStats(int fieldId) { + private FieldStats createFieldStats(int fieldId) { Types.NestedField field = tableSchema.findField(fieldId); // A file can carry metrics for an id this schema does not have. Skip it. if (field == null) { @@ -118,7 +118,7 @@ public ContentStats copy(Set fieldIds) { throw new UnsupportedOperationException("copy is not implemented"); } - boolean containsFieldInMaps(int fieldId) { + private boolean containsFieldInMaps(int fieldId) { return containsId(valueCounts, fieldId) || containsId(nullValueCounts, fieldId) || containsId(nanValueCounts, fieldId) diff --git a/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java b/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java index 280dc05b55ab..dc5fec4879b0 100644 --- a/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java +++ b/core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java @@ -1093,12 +1093,12 @@ private static long manifestRecordCount(ManifestFile manifest) { manifest.path()); Preconditions.checkArgument( Objects.equals(manifest.replacedFilesCount(), 0), - "Cannot convert manifest %s: replaced files count must be 0 for v3 or earlier manifests, was %s", + "Cannot convert manifest %s: Invalid replaced file count: %s", manifest.path(), manifest.replacedFilesCount()); Preconditions.checkArgument( Objects.equals(manifest.modifiedFilesCount(), 0), - "Cannot convert manifest %s: modified files count must be 0 for v3 or earlier manifests, was %s", + "Cannot convert manifest %s: Invalid modified file count: %s", manifest.path(), manifest.modifiedFilesCount()); return (long) manifest.addedFilesCount() diff --git a/core/src/test/java/org/apache/iceberg/TestMapBackedContentStats.java b/core/src/test/java/org/apache/iceberg/TestMapBackedContentStats.java index 33a10f815a02..0ce5d9601ac7 100644 --- a/core/src/test/java/org/apache/iceberg/TestMapBackedContentStats.java +++ b/core/src/test/java/org/apache/iceberg/TestMapBackedContentStats.java @@ -236,7 +236,6 @@ void countsAndTightBounds() { void fieldWithoutStatsIsExcluded() { MapBackedContentStats stats = new MapBackedContentStats(SCHEMA).wrap(FILE_WITH_STATS); - assertThat(stats.containsFieldInMaps(5)).isFalse(); assertThat(stats.type().field(StatsUtil.toBaseId(5))).isNull(); assertThat(stats.statsFor(5)).isNull(); assertThat(stats.fieldStats()) @@ -256,7 +255,6 @@ void unknownFieldIdInMaps() { ImmutableMap.of()); MapBackedContentStats stats = new MapBackedContentStats(SCHEMA).wrap(file); - assertThat(stats.containsFieldInMaps(99)).isTrue(); assertThat(stats.statsFor(99)).isNull(); assertThat(stats.type().fields()).isEmpty(); assertThat(stats.fieldStats()).isEmpty(); diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java index fa9eeb2e8268..29263338ecb2 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java @@ -932,8 +932,7 @@ void manifestTrackedFileAdapterRejectsNullReplacedFilesCount() { assertThatThrownBy(() -> TrackedFileAdapters.forManifestFile().wrap(manifest)) .isInstanceOf(IllegalArgumentException.class) .hasMessage( - "Cannot convert manifest %s: replaced files count must be 0 for v3 or earlier manifests, was null", - MANIFEST_LOCATION); + "Cannot convert manifest %s: Invalid replaced file count: null", MANIFEST_LOCATION); } @Test @@ -943,8 +942,7 @@ void manifestTrackedFileAdapterRejectsNonZeroReplacedFilesCount() { assertThatThrownBy(() -> TrackedFileAdapters.forManifestFile().wrap(manifest)) .isInstanceOf(IllegalArgumentException.class) .hasMessage( - "Cannot convert manifest %s: replaced files count must be 0 for v3 or earlier manifests, was 1", - MANIFEST_LOCATION); + "Cannot convert manifest %s: Invalid replaced file count: 1", MANIFEST_LOCATION); } @Test @@ -954,8 +952,7 @@ void manifestTrackedFileAdapterRejectsNullModifiedFilesCount() { assertThatThrownBy(() -> TrackedFileAdapters.forManifestFile().wrap(manifest)) .isInstanceOf(IllegalArgumentException.class) .hasMessage( - "Cannot convert manifest %s: modified files count must be 0 for v3 or earlier manifests, was null", - MANIFEST_LOCATION); + "Cannot convert manifest %s: Invalid modified file count: null", MANIFEST_LOCATION); } @Test @@ -965,8 +962,7 @@ void manifestTrackedFileAdapterRejectsNonZeroModifiedFilesCount() { assertThatThrownBy(() -> TrackedFileAdapters.forManifestFile().wrap(manifest)) .isInstanceOf(IllegalArgumentException.class) .hasMessage( - "Cannot convert manifest %s: modified files count must be 0 for v3 or earlier manifests, was 1", - MANIFEST_LOCATION); + "Cannot convert manifest %s: Invalid modified file count: 1", MANIFEST_LOCATION); } @Test From a658f97fc77db33780fbb1a20b8d42d6af3ec916 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Wed, 7 Oct 2026 09:33:06 -0700 Subject: [PATCH 11/11] minor test cleanup --- .../apache/iceberg/TestMapBackedContentStats.java | 14 ++++++++++---- .../apache/iceberg/TestTrackedFileAdapters.java | 14 +------------- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/core/src/test/java/org/apache/iceberg/TestMapBackedContentStats.java b/core/src/test/java/org/apache/iceberg/TestMapBackedContentStats.java index 0ce5d9601ac7..ca8e92dbcae0 100644 --- a/core/src/test/java/org/apache/iceberg/TestMapBackedContentStats.java +++ b/core/src/test/java/org/apache/iceberg/TestMapBackedContentStats.java @@ -230,17 +230,23 @@ void countsAndTightBounds() { .hasMessageContaining("Long.longValue()"); assertThat(ts.hasNullValueCount()).isTrue(); assertThat(ts.nullValueCount()).isEqualTo(1L); + + FieldStats name = stats.statsFor(4); + assertThat(name.hasValueCount()).isTrue(); + assertThat(name.valueCount()).isEqualTo(100L); + assertThat(name.hasNullValueCount()).isTrue(); + assertThat(name.nullValueCount()).isEqualTo(2L); + assertThat(name.hasNanValueCount()).isFalse(); + assertThatThrownBy(name::nanValueCount) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("Long.longValue()"); } @Test void fieldWithoutStatsIsExcluded() { MapBackedContentStats stats = new MapBackedContentStats(SCHEMA).wrap(FILE_WITH_STATS); - assertThat(stats.type().field(StatsUtil.toBaseId(5))).isNull(); assertThat(stats.statsFor(5)).isNull(); - assertThat(stats.fieldStats()) - .extracting(FieldStats::fieldId) - .containsExactlyInAnyOrder(1, 2, 3, 4); } @Test diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java index 29263338ecb2..3c6799989766 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java @@ -844,30 +844,18 @@ void dataTrackedFileAdapterKeepsPartitionTuple() { } @Test - void trackedFileDoubleWrapRoundTrip() { + void dataTrackedFileAdapterUnwrapsToOriginalTrackedFile() { TrackedFile source = trackedFile(FileContent.DATA); DataFile dataFile = TrackedFileAdapters.asDataFile(source, UNPARTITIONED); TrackedFile roundTripped = TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(dataFile); assertThat(roundTripped).isSameAs(source); } - @Test - void dataTrackedFileAdapterUnwrapsToOriginalTrackedFile() { - TrackedFile original = trackedFile(FileContent.DATA); - DataFile adapted = TrackedFileAdapters.asDataFile(original, UNPARTITIONED); - - TrackedFile result = TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(adapted); - - assertThat(result).isSameAs(original); - } - @Test void manifestTrackedFileAdapterUnwrapsToOriginalTrackedFile() { TrackedFile original = trackedFile(FileContent.DATA_MANIFEST, 0); ManifestFile adapted = TrackedFileAdapters.asManifestFile(original); - TrackedFile result = TrackedFileAdapters.forManifestFile().wrap(adapted); - assertThat(result).isSameAs(original); assertThat(result.formatVersion()).isZero(); }