diff --git a/astra/src/main/java/com/slack/astra/chunk/ChunkInfo.java b/astra/src/main/java/com/slack/astra/chunk/ChunkInfo.java index b7c09e598f..81fe0ddba7 100644 --- a/astra/src/main/java/com/slack/astra/chunk/ChunkInfo.java +++ b/astra/src/main/java/com/slack/astra/chunk/ChunkInfo.java @@ -3,7 +3,6 @@ import static com.slack.astra.util.ArgValidationUtils.ensureTrue; import com.slack.astra.metadata.snapshot.SnapshotMetadata; -import com.slack.astra.proto.metadata.Metadata; import java.util.Objects; /** @@ -31,19 +30,16 @@ public static ChunkInfo fromSnapshotMetadata(SnapshotMetadata snapshotMetadata) snapshotMetadata.endTimeEpochMs, snapshotMetadata.maxOffset, snapshotMetadata.partitionId, - snapshotMetadata.snapshotPath, snapshotMetadata.sizeInBytesOnDisk); } public static SnapshotMetadata toSnapshotMetadata(ChunkInfo chunkInfo, String chunkPrefix) { return new SnapshotMetadata( chunkPrefix + chunkInfo.chunkId, - chunkInfo.snapshotPath, chunkInfo.getDataStartTimeEpochMs(), chunkInfo.getDataEndTimeEpochMs(), chunkInfo.maxOffset, chunkInfo.kafkaPartitionId, - Metadata.IndexType.LOGS_LUCENE9, chunkInfo.sizeInBytesOnDisk); } @@ -77,14 +73,10 @@ public static SnapshotMetadata toSnapshotMetadata(ChunkInfo chunkInfo, String ch // indexing and snapshotting and is not useful afterwards. private long chunkSnapshotTimeEpochMs; - // Path to S3 snapshot. - private String snapshotPath; - // Size of chunk on disk in bytes private long sizeInBytesOnDisk; - public ChunkInfo( - String chunkId, long chunkCreationTimeEpochMs, String kafkaPartitionId, String snapshotPath) { + public ChunkInfo(String chunkId, long chunkCreationTimeEpochMs, String kafkaPartitionId) { // TODO: Should we set the snapshot time to creation time also? this( chunkId, @@ -95,7 +87,6 @@ public ChunkInfo( 0, DEFAULT_MAX_OFFSET, kafkaPartitionId, - snapshotPath, 0); } @@ -108,7 +99,6 @@ public ChunkInfo( long chunkSnapshotTimeEpochMs, long maxOffset, String kafkaPartitionId, - String snapshotPath, long sizeInBytesOnDisk) { ensureTrue(chunkId != null && !chunkId.isEmpty(), "Invalid chunk dataset name " + chunkId); ensureTrue( @@ -124,7 +114,6 @@ public ChunkInfo( this.chunkSnapshotTimeEpochMs = chunkSnapshotTimeEpochMs; this.maxOffset = maxOffset; this.kafkaPartitionId = kafkaPartitionId; - this.snapshotPath = snapshotPath; this.sizeInBytesOnDisk = sizeInBytesOnDisk; } @@ -164,14 +153,6 @@ public void setChunkLastUpdatedTimeEpochMs(long chunkLastUpdatedTimeEpochMs) { this.chunkLastUpdatedTimeEpochMs = chunkLastUpdatedTimeEpochMs; } - public void setSnapshotPath(String snapshotPath) { - this.snapshotPath = snapshotPath; - } - - public String getSnapshotPath() { - return snapshotPath; - } - public long getSizeInBytesOnDisk() { return sizeInBytesOnDisk; } @@ -238,8 +219,6 @@ public String toString() { + dataEndTimeEpochMs + ", chunkSnapshotTimeEpochMs=" + chunkSnapshotTimeEpochMs - + ", snapshotPath='" - + snapshotPath + ", sizeInBytesOnDisk='" + sizeInBytesOnDisk + '}'; @@ -258,7 +237,6 @@ public boolean equals(Object o) { && chunkSnapshotTimeEpochMs == chunkInfo.chunkSnapshotTimeEpochMs && Objects.equals(chunkId, chunkInfo.chunkId) && Objects.equals(kafkaPartitionId, chunkInfo.kafkaPartitionId) - && Objects.equals(snapshotPath, chunkInfo.snapshotPath) && sizeInBytesOnDisk == chunkInfo.sizeInBytesOnDisk; } @@ -272,7 +250,6 @@ public int hashCode() { chunkLastUpdatedTimeEpochMs, dataStartTimeEpochMs, dataEndTimeEpochMs, - chunkSnapshotTimeEpochMs, - snapshotPath); + chunkSnapshotTimeEpochMs); } } diff --git a/astra/src/main/java/com/slack/astra/chunk/IndexingChunkImpl.java b/astra/src/main/java/com/slack/astra/chunk/IndexingChunkImpl.java index 70fc76e74d..718a6a4a46 100644 --- a/astra/src/main/java/com/slack/astra/chunk/IndexingChunkImpl.java +++ b/astra/src/main/java/com/slack/astra/chunk/IndexingChunkImpl.java @@ -6,7 +6,6 @@ import com.slack.astra.metadata.search.SearchMetadataStore; import com.slack.astra.metadata.snapshot.SnapshotMetadata; import com.slack.astra.metadata.snapshot.SnapshotMetadataStore; -import com.slack.astra.proto.metadata.Metadata; import io.micrometer.core.instrument.MeterRegistry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -66,18 +65,16 @@ public void postSnapshot() { SnapshotMetadata nonLiveSnapshotMetadata = toSnapshotMetadata(chunkInfo, ""); snapshotMetadataStore.createSync(nonLiveSnapshotMetadata); - // Update the live snapshot. Keep the same snapshotId and snapshotPath to + // Update the live snapshot. Keep the same snapshotId to // ensure it's a live snapshot. SnapshotMetadata updatedSnapshotMetadata = new SnapshotMetadata( liveSnapshotMetadata.snapshotId, - liveSnapshotMetadata.snapshotPath, chunkInfo.getDataStartTimeEpochMs(), chunkInfo.getDataEndTimeEpochMs(), chunkInfo.getMaxOffset(), chunkInfo.getKafkaPartitionId(), - Metadata.IndexType.LOGS_LUCENE9, - chunkInfo.getSizeInBytesOnDisk()); + liveSnapshotMetadata.sizeInBytesOnDisk); snapshotMetadataStore.updateSync(updatedSnapshotMetadata); liveSnapshotMetadata = updatedSnapshotMetadata; diff --git a/astra/src/main/java/com/slack/astra/chunk/ReadOnlyChunkImpl.java b/astra/src/main/java/com/slack/astra/chunk/ReadOnlyChunkImpl.java index 8dc1b5bd3c..4ec25ef396 100644 --- a/astra/src/main/java/com/slack/astra/chunk/ReadOnlyChunkImpl.java +++ b/astra/src/main/java/com/slack/astra/chunk/ReadOnlyChunkImpl.java @@ -29,7 +29,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.time.Instant; -import java.util.List; import java.util.Map; import java.util.Objects; import java.util.UUID; @@ -152,7 +151,6 @@ public ReadOnlyChunkImpl( Metadata.CacheSlotMetadata.CacheSlotState.FREE, "", Instant.now().toEpochMilli(), - List.of(Metadata.IndexType.LOGS_LUCENE9), searchContext.hostname, replicaSet); cacheSlotMetadataStore.createSync(cacheSlotMetadata); @@ -265,9 +263,8 @@ public void downloadChunkData() { long durationNanos = assignmentTimer.stop(chunkAssignmentTimerSuccess); LOG.info( - "Downloaded chunk with snapshot id '{}' at path '{}' in {} seconds, was {}", + "Downloaded chunk with snapshot id '{}' in {} seconds, was {}", snapshotMetadata.snapshotId, - snapshotMetadata.snapshotPath, TimeUnit.SECONDS.convert(durationNanos, TimeUnit.NANOSECONDS), FileUtils.byteCountToDisplaySize(FileUtils.sizeOfDirectory(dataDirectory.toFile()))); } catch (Exception e) { @@ -408,9 +405,8 @@ private void handleChunkAssignment(CacheSlotMetadata cacheSlotMetadata) { long durationNanos = assignmentTimer.stop(chunkAssignmentTimerSuccess); LOG.debug( - "Downloaded chunk with snapshot id '{}' at path '{}' in {} seconds, was {}", + "Downloaded chunk with snapshot id '{}' in {} seconds, was {}", snapshotMetadata.snapshotId, - snapshotMetadata.snapshotPath, TimeUnit.SECONDS.convert(durationNanos, TimeUnit.NANOSECONDS), FileUtils.byteCountToDisplaySize(FileUtils.sizeOfDirectory(dataDirectory.toFile()))); } catch (Exception e) { diff --git a/astra/src/main/java/com/slack/astra/chunk/ReadWriteChunk.java b/astra/src/main/java/com/slack/astra/chunk/ReadWriteChunk.java index 310527ef35..d91a6c33be 100644 --- a/astra/src/main/java/com/slack/astra/chunk/ReadWriteChunk.java +++ b/astra/src/main/java/com/slack/astra/chunk/ReadWriteChunk.java @@ -68,7 +68,7 @@ public abstract class ReadWriteChunk implements Chunk { public static final String INDEX_FILES_UPLOAD = "index_files_upload"; public static final String INDEX_FILES_UPLOAD_FAILED = "index_files_upload_failed"; public static final String SNAPSHOT_TIMER = "snapshot.timer"; - public static final String LIVE_SNAPSHOT_PREFIX = SnapshotMetadata.LIVE_SNAPSHOT_PATH + "_"; + public static final String LIVE_SNAPSHOT_PREFIX = "LIVE_"; public static final String SCHEMA_FILE_NAME = "schema.json"; private final LogStore logStore; @@ -111,8 +111,7 @@ protected ReadWriteChunk( new ChunkInfo( chunkDataPrefix + "_" + chunkCreationTime.getEpochSecond() + "_" + logStoreId, chunkCreationTime.toEpochMilli(), - kafkaPartitionId, - SnapshotMetadata.LIVE_SNAPSHOT_PATH); + kafkaPartitionId); readOnly = false; this.meterRegistry = meterRegistry; @@ -253,7 +252,6 @@ public boolean snapshotToS3(ChunkStore chunkStore) { chunkStore.upload(chunkInfo.chunkId, dirPath); snapshotTimer.stop(meterRegistry.timer(SNAPSHOT_TIMER)); - chunkInfo.setSnapshotPath(chunkStore.getRemotePath(chunkInfo.chunkId)); chunkInfo.setSizeInBytesOnDisk(totalBytes); logger.info("Finished RW chunk snapshot to S3 {}.", chunkInfo); return true; diff --git a/astra/src/main/java/com/slack/astra/chunkManager/ChunkManagerBase.java b/astra/src/main/java/com/slack/astra/chunkManager/ChunkManagerBase.java index 0d7b75f320..77fd4c2996 100644 --- a/astra/src/main/java/com/slack/astra/chunkManager/ChunkManagerBase.java +++ b/astra/src/main/java/com/slack/astra/chunkManager/ChunkManagerBase.java @@ -98,7 +98,6 @@ public SearchResult query(SearchQuery query, Duration queryTimeout) { Tracing.currentTracer() .startScopedSpan("ChunkManagerBase.chunkQuery"); span.tag("chunkId", chunk.id()); - span.tag("chunkSnapshotPath", chunk.info().getSnapshotPath()); concurrentQueries.acquire(); try { return chunk.query(query); diff --git a/astra/src/main/java/com/slack/astra/clusterManager/ReplicaCreationService.java b/astra/src/main/java/com/slack/astra/clusterManager/ReplicaCreationService.java index 48ace911a6..e603b25490 100644 --- a/astra/src/main/java/com/slack/astra/clusterManager/ReplicaCreationService.java +++ b/astra/src/main/java/com/slack/astra/clusterManager/ReplicaCreationService.java @@ -18,7 +18,6 @@ import com.slack.astra.metadata.snapshot.SnapshotMetadata; import com.slack.astra.metadata.snapshot.SnapshotMetadataStore; import com.slack.astra.proto.config.AstraConfigs; -import com.slack.astra.proto.metadata.Metadata; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Timer; @@ -177,7 +176,7 @@ protected Map createReplicasForUnassignedSnapshots() { .filter( snapshotMetadata -> snapshotMetadata.endTimeEpochMs > snapshotExpiration - && !SnapshotMetadata.isLive(snapshotMetadata) + && !snapshotMetadata.isLive() && !existingReplicas.contains(snapshotMetadata.snapshotId)) .map( (snapshotMetadata) -> { @@ -249,7 +248,6 @@ public static ReplicaMetadata replicaMetadataFromSnapshotId( replicaSet, Instant.now().toEpochMilli(), expireAfter.toEpochMilli(), - isRestored, - Metadata.IndexType.LOGS_LUCENE9); + isRestored); } } diff --git a/astra/src/main/java/com/slack/astra/clusterManager/SnapshotDeletionService.java b/astra/src/main/java/com/slack/astra/clusterManager/SnapshotDeletionService.java index 98657b7e38..d9676c7d5e 100644 --- a/astra/src/main/java/com/slack/astra/clusterManager/SnapshotDeletionService.java +++ b/astra/src/main/java/com/slack/astra/clusterManager/SnapshotDeletionService.java @@ -14,7 +14,6 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.slack.astra.blobfs.ChunkStore; import com.slack.astra.metadata.replica.ReplicaMetadataStore; -import com.slack.astra.metadata.snapshot.SnapshotMetadata; import com.slack.astra.metadata.snapshot.SnapshotMetadataStore; import com.slack.astra.proto.config.AstraConfigs; import io.micrometer.core.instrument.Counter; @@ -175,7 +174,7 @@ protected int deleteExpiredSnapshotsWithoutReplicas() { // served from the indexers. To avoid the whole headache of managing all the // different states we could be in, we should just disable the deletion of live // snapshots whole-cloth. We clean those up when a node boots anyhow - .filter(snapshotMetadata -> !SnapshotMetadata.isLive(snapshotMetadata)) + .filter(snapshotMetadata -> !snapshotMetadata.isLive()) .map( snapshotMetadata -> { ListenableFuture future = diff --git a/astra/src/main/java/com/slack/astra/metadata/cache/CacheSlotMetadata.java b/astra/src/main/java/com/slack/astra/metadata/cache/CacheSlotMetadata.java index cc7db5c06d..c06b07cf88 100644 --- a/astra/src/main/java/com/slack/astra/metadata/cache/CacheSlotMetadata.java +++ b/astra/src/main/java/com/slack/astra/metadata/cache/CacheSlotMetadata.java @@ -4,8 +4,6 @@ import com.slack.astra.metadata.core.AstraPartitionedMetadata; import com.slack.astra.proto.metadata.Metadata; -import java.util.Collections; -import java.util.List; import java.util.Objects; /** @@ -18,23 +16,18 @@ public class CacheSlotMetadata extends AstraPartitionedMetadata { public final Metadata.CacheSlotMetadata.CacheSlotState cacheSlotState; public final String replicaId; public final long updatedTimeEpochMs; - public final List supportedIndexTypes; public CacheSlotMetadata( String name, Metadata.CacheSlotMetadata.CacheSlotState cacheSlotState, String replicaId, long updatedTimeEpochMs, - List supportedIndexTypes, String hostname, String replicaSet) { super(name); checkArgument(hostname != null && !hostname.isEmpty(), "Hostname cannot be null or empty"); checkArgument(cacheSlotState != null, "Cache slot state cannot be null"); checkArgument(updatedTimeEpochMs > 0, "Updated time must be greater than 0"); - checkArgument( - supportedIndexTypes != null && !supportedIndexTypes.isEmpty(), - "supported index types shouldn't be empty"); if (cacheSlotState.equals(Metadata.CacheSlotMetadata.CacheSlotState.FREE)) { checkArgument( replicaId != null && replicaId.isEmpty(), @@ -50,7 +43,6 @@ public CacheSlotMetadata( this.cacheSlotState = cacheSlotState; this.replicaId = replicaId; this.updatedTimeEpochMs = updatedTimeEpochMs; - this.supportedIndexTypes = Collections.unmodifiableList(supportedIndexTypes); } public String getHostname() { @@ -67,8 +59,7 @@ public boolean equals(Object o) { if (!hostname.equals(that.hostname)) return false; if (!Objects.equals(replicaSet, that.replicaSet)) return false; if (cacheSlotState != that.cacheSlotState) return false; - if (!replicaId.equals(that.replicaId)) return false; - return supportedIndexTypes.equals(that.supportedIndexTypes); + return replicaId.equals(that.replicaId); } @Override @@ -79,7 +70,6 @@ public int hashCode() { result = 31 * result + cacheSlotState.hashCode(); result = 31 * result + replicaId.hashCode(); result = 31 * result + (int) (updatedTimeEpochMs ^ (updatedTimeEpochMs >>> 32)); - result = 31 * result + supportedIndexTypes.hashCode(); return result; } @@ -99,8 +89,6 @@ public String toString() { + '\'' + ", updatedTimeEpochMs=" + updatedTimeEpochMs - + ", supportedIndexTypes=" - + supportedIndexTypes + ", name='" + name + '\'' diff --git a/astra/src/main/java/com/slack/astra/metadata/cache/CacheSlotMetadataSerializer.java b/astra/src/main/java/com/slack/astra/metadata/cache/CacheSlotMetadataSerializer.java index 67b263d289..900f512568 100644 --- a/astra/src/main/java/com/slack/astra/metadata/cache/CacheSlotMetadataSerializer.java +++ b/astra/src/main/java/com/slack/astra/metadata/cache/CacheSlotMetadataSerializer.java @@ -14,7 +14,6 @@ private static CacheSlotMetadata fromCacheSlotMetadataProto( cacheSlotMetadataProto.getCacheSlotState().name()), cacheSlotMetadataProto.getReplicaId(), cacheSlotMetadataProto.getUpdatedTimeEpochMs(), - cacheSlotMetadataProto.getSupportedIndexTypesList(), cacheSlotMetadataProto.getHostname(), cacheSlotMetadataProto.getReplicaSet()); } @@ -25,7 +24,6 @@ private static Metadata.CacheSlotMetadata toCacheSlotMetadataProto(CacheSlotMeta .setReplicaId(metadata.replicaId) .setCacheSlotState(metadata.cacheSlotState) .setUpdatedTimeEpochMs(metadata.updatedTimeEpochMs) - .addAllSupportedIndexTypes(metadata.supportedIndexTypes) .setHostname(metadata.hostname) .setReplicaSet(metadata.replicaSet) .build(); diff --git a/astra/src/main/java/com/slack/astra/metadata/cache/CacheSlotMetadataStore.java b/astra/src/main/java/com/slack/astra/metadata/cache/CacheSlotMetadataStore.java index eb331082db..ad870148ca 100644 --- a/astra/src/main/java/com/slack/astra/metadata/cache/CacheSlotMetadataStore.java +++ b/astra/src/main/java/com/slack/astra/metadata/cache/CacheSlotMetadataStore.java @@ -50,7 +50,6 @@ public ListenableFuture updateCacheSlotStateStateWithReplicaId( newState, replicaId, Instant.now().toEpochMilli(), - cacheSlotMetadata.supportedIndexTypes, cacheSlotMetadata.hostname, cacheSlotMetadata.replicaSet); // todo - consider refactoring this to return a completable future instead diff --git a/astra/src/main/java/com/slack/astra/metadata/replica/ReplicaMetadata.java b/astra/src/main/java/com/slack/astra/metadata/replica/ReplicaMetadata.java index 2ac71627ad..c1c163ab20 100644 --- a/astra/src/main/java/com/slack/astra/metadata/replica/ReplicaMetadata.java +++ b/astra/src/main/java/com/slack/astra/metadata/replica/ReplicaMetadata.java @@ -3,7 +3,6 @@ import static com.google.common.base.Preconditions.checkArgument; import com.slack.astra.metadata.core.AstraPartitionedMetadata; -import com.slack.astra.proto.metadata.Metadata; import java.time.Instant; import java.time.ZoneOffset; import java.time.ZonedDateTime; @@ -23,7 +22,6 @@ public class ReplicaMetadata extends AstraPartitionedMetadata { public final long createdTimeEpochMs; public final long expireAfterEpochMs; public boolean isRestored; - public final Metadata.IndexType indexType; public ReplicaMetadata( String name, @@ -31,8 +29,7 @@ public ReplicaMetadata( String replicaSet, long createdTimeEpochMs, long expireAfterEpochMs, - boolean isRestored, - Metadata.IndexType indexType) { + boolean isRestored) { super(name); checkArgument(createdTimeEpochMs > 0, "Created time must be greater than 0"); checkArgument(expireAfterEpochMs >= 0, "Expiration time must be greater than or equal to 0"); @@ -44,7 +41,6 @@ public ReplicaMetadata( this.createdTimeEpochMs = createdTimeEpochMs; this.expireAfterEpochMs = expireAfterEpochMs; this.isRestored = isRestored; - this.indexType = indexType; } public String getSnapshotId() { @@ -77,8 +73,7 @@ public boolean equals(Object o) { if (expireAfterEpochMs != that.expireAfterEpochMs) return false; if (isRestored != that.isRestored) return false; if (!snapshotId.equals(that.snapshotId)) return false; - if (!replicaSet.equals(that.replicaSet)) return false; - return indexType == that.indexType; + return replicaSet.equals(that.replicaSet); } @Override @@ -89,7 +84,6 @@ public int hashCode() { result = 31 * result + (int) (createdTimeEpochMs ^ (createdTimeEpochMs >>> 32)); result = 31 * result + (int) (expireAfterEpochMs ^ (expireAfterEpochMs >>> 32)); result = 31 * result + (isRestored ? 1 : 0); - result = 31 * result + indexType.hashCode(); return result; } @@ -108,8 +102,6 @@ public String toString() { + expireAfterEpochMs + ", isRestored=" + isRestored - + ", indexType=" - + indexType + ", name='" + name + '\'' diff --git a/astra/src/main/java/com/slack/astra/metadata/replica/ReplicaMetadataSerializer.java b/astra/src/main/java/com/slack/astra/metadata/replica/ReplicaMetadataSerializer.java index 773e2ea514..66d0a9cf12 100644 --- a/astra/src/main/java/com/slack/astra/metadata/replica/ReplicaMetadataSerializer.java +++ b/astra/src/main/java/com/slack/astra/metadata/replica/ReplicaMetadataSerializer.java @@ -14,8 +14,7 @@ private static ReplicaMetadata fromReplicaMetadataProto( replicaMetadataProto.getReplicaSet(), replicaMetadataProto.getCreatedTimeEpochMs(), replicaMetadataProto.getExpireAfterEpochMs(), - replicaMetadataProto.getIsRestored(), - Metadata.IndexType.LOGS_LUCENE9); + replicaMetadataProto.getIsRestored()); } private static Metadata.ReplicaMetadata toReplicaMetadataProto(ReplicaMetadata metadata) { @@ -26,7 +25,6 @@ private static Metadata.ReplicaMetadata toReplicaMetadataProto(ReplicaMetadata m .setCreatedTimeEpochMs(metadata.createdTimeEpochMs) .setExpireAfterEpochMs(metadata.expireAfterEpochMs) .setIsRestored(metadata.isRestored) - .setIndexType(metadata.indexType) .build(); } diff --git a/astra/src/main/java/com/slack/astra/metadata/snapshot/SnapshotMetadata.java b/astra/src/main/java/com/slack/astra/metadata/snapshot/SnapshotMetadata.java index 89b29c272a..0e21374230 100644 --- a/astra/src/main/java/com/slack/astra/metadata/snapshot/SnapshotMetadata.java +++ b/astra/src/main/java/com/slack/astra/metadata/snapshot/SnapshotMetadata.java @@ -3,7 +3,6 @@ import static com.google.common.base.Preconditions.checkArgument; import com.slack.astra.metadata.core.AstraPartitionedMetadata; -import com.slack.astra.proto.metadata.Metadata; import java.time.Instant; import java.time.ZoneOffset; import java.time.ZonedDateTime; @@ -23,55 +22,38 @@ * here. */ public class SnapshotMetadata extends AstraPartitionedMetadata { - public static final String LIVE_SNAPSHOT_PATH = "LIVE"; - - public static boolean isLive(SnapshotMetadata snapshotMetadata) { - return snapshotMetadata.snapshotPath.equals(LIVE_SNAPSHOT_PATH); - } - - // snapshot path is deprecated in favor of storing these to a well-known path - // @see ChunkStore.getPath - @Deprecated public final String snapshotPath; - public final String snapshotId; public final long startTimeEpochMs; public final long endTimeEpochMs; public final long maxOffset; public final String partitionId; - public final Metadata.IndexType indexType; public long sizeInBytesOnDisk; public SnapshotMetadata( String snapshotId, - String snapshotPath, long startTimeEpochMs, long endTimeEpochMs, long maxOffset, String partitionId, - Metadata.IndexType indexType, long sizeInBytesOnDisk) { this( snapshotId, - snapshotPath, snapshotId, startTimeEpochMs, endTimeEpochMs, maxOffset, partitionId, - sizeInBytesOnDisk, - indexType); + sizeInBytesOnDisk); } private SnapshotMetadata( String name, - String snapshotPath, String snapshotId, long startTimeEpochMs, long endTimeEpochMs, long maxOffset, String partitionId, - long sizeInBytesOnDisk, - Metadata.IndexType indexType) { + long sizeInBytesOnDisk) { super(name); checkArgument(snapshotId != null && !snapshotId.isEmpty(), "snapshotId can't be null or empty"); checkArgument(startTimeEpochMs > 0, "start time should be greater than zero."); @@ -82,65 +64,45 @@ private SnapshotMetadata( checkArgument(maxOffset >= 0, "max offset should be greater than or equal to zero."); checkArgument( partitionId != null && !partitionId.isEmpty(), "partitionId can't be null or empty"); - checkArgument( - snapshotPath != null && !snapshotPath.isEmpty(), "snapshotPath can't be null or empty"); - this.snapshotPath = snapshotPath; this.snapshotId = snapshotId; this.startTimeEpochMs = startTimeEpochMs; this.endTimeEpochMs = endTimeEpochMs; this.maxOffset = maxOffset; this.partitionId = partitionId; - this.indexType = indexType; this.sizeInBytesOnDisk = sizeInBytesOnDisk; } @Override - public boolean equals(Object o) { + public final boolean equals(Object o) { if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + if (!(o instanceof SnapshotMetadata that)) return false; if (!super.equals(o)) return false; - SnapshotMetadata that = (SnapshotMetadata) o; - - if (startTimeEpochMs != that.startTimeEpochMs) return false; - if (endTimeEpochMs != that.endTimeEpochMs) return false; - if (maxOffset != that.maxOffset) return false; - if (snapshotPath != null ? !snapshotPath.equals(that.snapshotPath) : that.snapshotPath != null) - return false; - if (snapshotId != null ? !snapshotId.equals(that.snapshotId) : that.snapshotId != null) - return false; - if (partitionId != null ? !partitionId.equals(that.partitionId) : that.partitionId != null) - return false; - if (sizeInBytesOnDisk != that.sizeInBytesOnDisk) return false; - return indexType == that.indexType; + return startTimeEpochMs == that.startTimeEpochMs + && endTimeEpochMs == that.endTimeEpochMs + && maxOffset == that.maxOffset + && sizeInBytesOnDisk == that.sizeInBytesOnDisk + && snapshotId.equals(that.snapshotId) + && partitionId.equals(that.partitionId); } @Override public int hashCode() { int result = super.hashCode(); - result = 31 * result + (snapshotPath != null ? snapshotPath.hashCode() : 0); - result = 31 * result + (snapshotId != null ? snapshotId.hashCode() : 0); - result = 31 * result + (int) (startTimeEpochMs ^ (startTimeEpochMs >>> 32)); - result = 31 * result + (int) (endTimeEpochMs ^ (endTimeEpochMs >>> 32)); - result = 31 * result + (int) (maxOffset ^ (maxOffset >>> 32)); - result = 31 * result + (partitionId != null ? partitionId.hashCode() : 0); - result = 31 * result + (indexType != null ? indexType.hashCode() : 0); + result = 31 * result + snapshotId.hashCode(); + result = 31 * result + Long.hashCode(startTimeEpochMs); + result = 31 * result + Long.hashCode(endTimeEpochMs); + result = 31 * result + Long.hashCode(maxOffset); + result = 31 * result + partitionId.hashCode(); result = 31 * result + Long.hashCode(sizeInBytesOnDisk); return result; } @Override public String toString() { - // Include name from super class in the toString method to simplify debugging. return "SnapshotMetadata{" - + "name='" - + name - + '\'' - + ", snapshotPath='" - + snapshotPath - + '\'' - + ", snapshotId='" + + "snapshotId='" + snapshotId + '\'' + ", startTimeEpochMs=" @@ -152,16 +114,17 @@ public String toString() { + ", partitionId='" + partitionId + '\'' - + ", indexType=" - + indexType + ", sizeInBytesOnDisk=" + sizeInBytesOnDisk + + ", name='" + + name + + '\'' + '}'; } @Override public String getPartition() { - if (isLive(this)) { + if (isLive()) { // this keeps all the live snapshots in a single partition - this is important as their stored // startTimeEpochMs is not stable, and will be updated. This would cause an update to a live // node to fail with a partitioned metadata store as it cannot change the path of the znode. @@ -174,4 +137,11 @@ public String getPartition() { snapshotTime.getLong(ChronoField.HOUR_OF_DAY)); } } + + // todo - this is better than the previous version of storing a static "LIVE" string to a path + // variable but not by a lot. The "isLive" functionality should be reconsidered more broadly. + // The ideal way is likely to reconsider the ZK type for "LIVE" snapshots + public boolean isLive() { + return this.sizeInBytesOnDisk == 0; + } } diff --git a/astra/src/main/java/com/slack/astra/metadata/snapshot/SnapshotMetadataSerializer.java b/astra/src/main/java/com/slack/astra/metadata/snapshot/SnapshotMetadataSerializer.java index f2224adfb3..3fb5918e46 100644 --- a/astra/src/main/java/com/slack/astra/metadata/snapshot/SnapshotMetadataSerializer.java +++ b/astra/src/main/java/com/slack/astra/metadata/snapshot/SnapshotMetadataSerializer.java @@ -11,12 +11,10 @@ private static Metadata.SnapshotMetadata toSnapshotMetadataProto( return Metadata.SnapshotMetadata.newBuilder() .setName(snapshotMetadata.name) .setSnapshotId(snapshotMetadata.snapshotId) - .setSnapshotPath(snapshotMetadata.snapshotPath) .setStartTimeEpochMs(snapshotMetadata.startTimeEpochMs) .setEndTimeEpochMs(snapshotMetadata.endTimeEpochMs) .setPartitionId(snapshotMetadata.partitionId) .setMaxOffset(snapshotMetadata.maxOffset) - .setIndexType(snapshotMetadata.indexType) .setSizeInBytes(snapshotMetadata.sizeInBytesOnDisk) .build(); } @@ -25,12 +23,10 @@ private static SnapshotMetadata fromSnapshotMetadataProto( Metadata.SnapshotMetadata protoSnapshotMetadata) { return new SnapshotMetadata( protoSnapshotMetadata.getSnapshotId(), - protoSnapshotMetadata.getSnapshotPath(), protoSnapshotMetadata.getStartTimeEpochMs(), protoSnapshotMetadata.getEndTimeEpochMs(), protoSnapshotMetadata.getMaxOffset(), protoSnapshotMetadata.getPartitionId(), - Metadata.IndexType.LOGS_LUCENE9, protoSnapshotMetadata.getSizeInBytes()); } diff --git a/astra/src/main/proto/metadata.proto b/astra/src/main/proto/metadata.proto index a8e6f2b247..180d539b64 100644 --- a/astra/src/main/proto/metadata.proto +++ b/astra/src/main/proto/metadata.proto @@ -4,11 +4,6 @@ package slack.proto.astra; option java_package = "com.slack.astra.proto.metadata"; -// Type of index used to store the data. -enum IndexType { - LOGS_LUCENE9 = 0; -}; - message CacheSlotMetadata { enum CacheSlotState { FREE = 0; @@ -32,7 +27,7 @@ message CacheSlotMetadata { int64 updated_time_epoch_ms = 4; // Index types supported by cache slot. - repeated IndexType supported_index_types = 5; + // **removed Aug 2024** repeated IndexType supported_index_types = 5; // Unique string identifying the host string hostname = 6; @@ -59,7 +54,7 @@ message ReplicaMetadata { bool isRestored = 6; - IndexType index_type = 7; + // **removed Aug 2024** IndexType index_type = 7; } message SnapshotMetadata { @@ -83,7 +78,7 @@ message SnapshotMetadata { int64 max_offset = 6; // The type of index used to store this data. - IndexType index_type = 8; + // **removed Aug 2024** IndexType index_type = 8; // Size of the snapshot in bytes int64 sizeInBytes = 9; diff --git a/astra/src/test/java/com/slack/astra/chunk/ChunkInfoTest.java b/astra/src/test/java/com/slack/astra/chunk/ChunkInfoTest.java index c547d6993b..c75a4e2e3d 100644 --- a/astra/src/test/java/com/slack/astra/chunk/ChunkInfoTest.java +++ b/astra/src/test/java/com/slack/astra/chunk/ChunkInfoTest.java @@ -15,13 +15,11 @@ public class ChunkInfoTest { private static final String TEST_KAFKA_PARTITION_ID = "10"; private static final String TEST_CHUNK_NAME = "testChunkInfo1"; - private static final String TEST_SNAPSHOT_PATH = "testSnapshotPath"; @Test public void testChunkInfoCreation() { final long chunkCreationTime = 1000; - final ChunkInfo info = - new ChunkInfo(TEST_CHUNK_NAME, 1000, TEST_KAFKA_PARTITION_ID, TEST_SNAPSHOT_PATH); + final ChunkInfo info = new ChunkInfo(TEST_CHUNK_NAME, 1000, TEST_KAFKA_PARTITION_ID); assertThat(info.getChunkCreationTimeEpochMs()).isEqualTo(chunkCreationTime); assertThat(info.getChunkLastUpdatedTimeEpochMs()).isEqualTo(chunkCreationTime); assertThat(info.getDataStartTimeEpochMs()).isEqualTo(chunkCreationTime); @@ -29,7 +27,6 @@ public void testChunkInfoCreation() { assertThat(info.getChunkSnapshotTimeEpochMs()).isEqualTo(0); assertThat(info.getMaxOffset()).isEqualTo(DEFAULT_MAX_OFFSET); assertThat(info.getKafkaPartitionId()).isEqualTo(TEST_KAFKA_PARTITION_ID); - assertThat(info.getSnapshotPath()).isEqualTo(TEST_SNAPSHOT_PATH); } @Test @@ -37,11 +34,7 @@ public void testChunkDataTimeRange() { final LocalDateTime startTime = LocalDateTime.of(2020, 10, 1, 10, 10, 0); final long chunkCreationTimeEpochMilli = startTime.toInstant(ZoneOffset.UTC).toEpochMilli(); final ChunkInfo info = - new ChunkInfo( - TEST_CHUNK_NAME, - chunkCreationTimeEpochMilli, - TEST_KAFKA_PARTITION_ID, - TEST_SNAPSHOT_PATH); + new ChunkInfo(TEST_CHUNK_NAME, chunkCreationTimeEpochMilli, TEST_KAFKA_PARTITION_ID); assertThat(info.getChunkCreationTimeEpochMs()).isEqualTo(chunkCreationTimeEpochMilli); assertThat(info.getChunkLastUpdatedTimeEpochMs()).isEqualTo(chunkCreationTimeEpochMilli); assertThat(info.getDataStartTimeEpochMs()).isEqualTo(chunkCreationTimeEpochMilli); @@ -49,7 +42,6 @@ public void testChunkDataTimeRange() { assertThat(info.getChunkSnapshotTimeEpochMs()).isEqualTo(0); assertThat(info.getMaxOffset()).isEqualTo(DEFAULT_MAX_OFFSET); assertThat(info.getKafkaPartitionId()).isEqualTo(TEST_KAFKA_PARTITION_ID); - assertThat(info.getSnapshotPath()).isEqualTo(TEST_SNAPSHOT_PATH); // Add message with same time range. info.updateDataTimeRange(chunkCreationTimeEpochMilli); @@ -115,8 +107,7 @@ public void testUnInitializedChunkDataInRange() { final LocalDateTime startTime = LocalDateTime.of(2020, 10, 1, 10, 10, 0); final long chunkCreationTimeSecs = startTime.toInstant(ZoneOffset.UTC).toEpochMilli(); final ChunkInfo info = - new ChunkInfo( - TEST_CHUNK_NAME, chunkCreationTimeSecs, TEST_KAFKA_PARTITION_ID, TEST_SNAPSHOT_PATH); + new ChunkInfo(TEST_CHUNK_NAME, chunkCreationTimeSecs, TEST_KAFKA_PARTITION_ID); assertThat(info.getChunkCreationTimeEpochMs()).isEqualTo(chunkCreationTimeSecs); assertThat(info.getChunkLastUpdatedTimeEpochMs()).isEqualTo(chunkCreationTimeSecs); assertThat(info.getDataStartTimeEpochMs()).isEqualTo(chunkCreationTimeSecs); @@ -133,7 +124,6 @@ public void testUnInitializedChunkDataInRange() { assertThat(info.containsDataInTimeRange(1000, chunkCreationTimeSecs + 1)).isTrue(); assertThat(info.getMaxOffset()).isEqualTo(DEFAULT_MAX_OFFSET); assertThat(info.getKafkaPartitionId()).isEqualTo(TEST_KAFKA_PARTITION_ID); - assertThat(info.getSnapshotPath()).isEqualTo(TEST_SNAPSHOT_PATH); } @Test @@ -141,8 +131,7 @@ public void testChunkDataInRange() { final LocalDateTime startTime = LocalDateTime.of(2020, 10, 1, 10, 10, 0); final long chunkCreationTimeMs = startTime.toInstant(ZoneOffset.UTC).toEpochMilli(); final ChunkInfo info = - new ChunkInfo( - TEST_CHUNK_NAME, chunkCreationTimeMs, TEST_KAFKA_PARTITION_ID, TEST_SNAPSHOT_PATH); + new ChunkInfo(TEST_CHUNK_NAME, chunkCreationTimeMs, TEST_KAFKA_PARTITION_ID); assertThat(info.getChunkCreationTimeEpochMs()).isEqualTo(chunkCreationTimeMs); assertThat(info.getChunkLastUpdatedTimeEpochMs()).isEqualTo(chunkCreationTimeMs); assertThat(info.getDataStartTimeEpochMs()).isEqualTo(chunkCreationTimeMs); @@ -256,8 +245,7 @@ public void testChunkDataInRange() { @Test public void testNegativeStartTimeInDateRange() { - final ChunkInfo info = - new ChunkInfo(TEST_CHUNK_NAME, 1000, TEST_KAFKA_PARTITION_ID, TEST_SNAPSHOT_PATH); + final ChunkInfo info = new ChunkInfo(TEST_CHUNK_NAME, 1000, TEST_KAFKA_PARTITION_ID); info.updateDataTimeRange(980); info.updateDataTimeRange(1020); @@ -267,8 +255,7 @@ public void testNegativeStartTimeInDateRange() { @Test public void testNegativeEndTimeInDateRange() { - final ChunkInfo info = - new ChunkInfo(TEST_CHUNK_NAME, 1000, TEST_KAFKA_PARTITION_ID, TEST_SNAPSHOT_PATH); + final ChunkInfo info = new ChunkInfo(TEST_CHUNK_NAME, 1000, TEST_KAFKA_PARTITION_ID); info.updateDataTimeRange(980); info.updateDataTimeRange(1020); @@ -278,8 +265,7 @@ public void testNegativeEndTimeInDateRange() { @Test public void testNegativeIntervalInDateRange() { - final ChunkInfo info = - new ChunkInfo(TEST_CHUNK_NAME, 1000, TEST_KAFKA_PARTITION_ID, TEST_SNAPSHOT_PATH); + final ChunkInfo info = new ChunkInfo(TEST_CHUNK_NAME, 1000, TEST_KAFKA_PARTITION_ID); info.updateDataTimeRange(980); info.updateDataTimeRange(1020); @@ -290,32 +276,30 @@ public void testNegativeIntervalInDateRange() { @Test public void testInvalidChunkName() { assertThatExceptionOfType(IllegalArgumentException.class) - .isThrownBy(() -> new ChunkInfo(null, 100, TEST_KAFKA_PARTITION_ID, TEST_SNAPSHOT_PATH)); + .isThrownBy(() -> new ChunkInfo(null, 100, TEST_KAFKA_PARTITION_ID)); } @Test public void testEmptyChunkName() { assertThatExceptionOfType(IllegalArgumentException.class) - .isThrownBy(() -> new ChunkInfo("", 100, TEST_KAFKA_PARTITION_ID, TEST_SNAPSHOT_PATH)); + .isThrownBy(() -> new ChunkInfo("", 100, TEST_KAFKA_PARTITION_ID)); } @Test public void testNegativeChunkCreationTime() { assertThatExceptionOfType(IllegalArgumentException.class) - .isThrownBy( - () -> new ChunkInfo(TEST_CHUNK_NAME, -1, TEST_KAFKA_PARTITION_ID, TEST_SNAPSHOT_PATH)); + .isThrownBy(() -> new ChunkInfo(TEST_CHUNK_NAME, -1, TEST_KAFKA_PARTITION_ID)); } @Test public void testEmptyKafkaPartitionId() { assertThatExceptionOfType(IllegalArgumentException.class) - .isThrownBy(() -> new ChunkInfo(TEST_CHUNK_NAME, 100, "", TEST_SNAPSHOT_PATH)); + .isThrownBy(() -> new ChunkInfo(TEST_CHUNK_NAME, 100, "")); } @Test public void testMaxOffset() { - ChunkInfo chunkInfo = - new ChunkInfo(TEST_CHUNK_NAME, 100, TEST_KAFKA_PARTITION_ID, TEST_SNAPSHOT_PATH); + ChunkInfo chunkInfo = new ChunkInfo(TEST_CHUNK_NAME, 100, TEST_KAFKA_PARTITION_ID); assertThat(chunkInfo.getMaxOffset()).isEqualTo(DEFAULT_MAX_OFFSET); chunkInfo.updateMaxOffset(100); assertThat(chunkInfo.getMaxOffset()).isEqualTo(100); @@ -333,16 +317,6 @@ public void testMaxOffset() { assertThat(chunkInfo.getMaxOffset()).isEqualTo(104); } - @Test - public void testSnapshotPathCanBeUpdated() { - ChunkInfo chunkInfo = - new ChunkInfo(TEST_CHUNK_NAME, 10000, TEST_KAFKA_PARTITION_ID, TEST_SNAPSHOT_PATH); - assertThat(chunkInfo.getSnapshotPath()).isEqualTo(TEST_SNAPSHOT_PATH); - String testPath = "/path"; - chunkInfo.setSnapshotPath(testPath); - assertThat(chunkInfo.getSnapshotPath()).isEqualTo(testPath); - } - @Test public void snapshotMetadataConversion() { long dataStart = 101; @@ -357,7 +331,6 @@ public void snapshotMetadataConversion() { dataEnd, 1000, TEST_KAFKA_PARTITION_ID, - TEST_SNAPSHOT_PATH, 0); assertThat(fromSnapshotMetadata(toSnapshotMetadata(chunkInfo, ""))).isEqualTo(chunkInfo); } diff --git a/astra/src/test/java/com/slack/astra/chunk/IndexingChunkImplTest.java b/astra/src/test/java/com/slack/astra/chunk/IndexingChunkImplTest.java index 056ce028d9..dc254480f8 100644 --- a/astra/src/test/java/com/slack/astra/chunk/IndexingChunkImplTest.java +++ b/astra/src/test/java/com/slack/astra/chunk/IndexingChunkImplTest.java @@ -82,8 +82,6 @@ private static void testBeforeSnapshotState( assertThat(beforeSearchNodes.size()).isEqualTo(1); assertThat(beforeSearchNodes.get(0).url).contains(TEST_HOST); assertThat(beforeSearchNodes.get(0).url).contains(String.valueOf(TEST_PORT)); - - assertThat(beforeSearchNodes.get(0).snapshotName).contains(SnapshotMetadata.LIVE_SNAPSHOT_PATH); } @Nested @@ -638,7 +636,6 @@ public void testSnapshotToNonExistentS3BucketFails() // Snapshot to S3 without creating the s3 bucket. assertThat(chunk.snapshotToS3(chunkStore)).isFalse(); - assertThat(chunk.info().getSnapshotPath()).isEqualTo(SnapshotMetadata.LIVE_SNAPSHOT_PATH); // Metadata checks List afterSnapshots = @@ -647,15 +644,11 @@ public void testSnapshotToNonExistentS3BucketFails() assertThat(afterSnapshots.get(0).partitionId).isEqualTo(TEST_KAFKA_PARTITION_ID); assertThat(afterSnapshots.get(0).maxOffset).isEqualTo(0); - assertThat(afterSnapshots.get(0).snapshotPath).isEqualTo(SnapshotMetadata.LIVE_SNAPSHOT_PATH); - List afterSearchNodes = AstraMetadataTestUtils.listSyncUncached(searchMetadataStore); assertThat(afterSearchNodes.size()).isEqualTo(1); assertThat(afterSearchNodes.get(0).url).contains(TEST_HOST); assertThat(afterSearchNodes.get(0).url).contains(String.valueOf(TEST_PORT)); - assertThat(afterSearchNodes.get(0).snapshotName) - .contains(SnapshotMetadata.LIVE_SNAPSHOT_PATH); } @SuppressWarnings("OptionalGetWithoutIsPresent") @@ -702,9 +695,7 @@ public void testSnapshotToS3UsingChunkApi() throws Exception { ChunkStore chunkStore = new ChunkStore(s3AsyncClient, bucket); // Snapshot to S3 - assertThat(chunk.info().getSnapshotPath()).isEqualTo(SnapshotMetadata.LIVE_SNAPSHOT_PATH); assertThat(chunk.snapshotToS3(chunkStore)).isTrue(); - assertThat(chunk.info().getSnapshotPath()).isNotEmpty(); // depending on heap and CFS files this can be 5 or 19. assertThat(getCount(INDEX_FILES_UPLOAD, registry)).isGreaterThan(5); @@ -733,21 +724,16 @@ public void testSnapshotToS3UsingChunkApi() throws Exception { assertThat(afterSnapshots.size()).isEqualTo(2); assertThat(afterSnapshots).contains(ChunkInfo.toSnapshotMetadata(chunk.info(), "")); SnapshotMetadata liveSnapshot = - afterSnapshots.stream() - .filter(s -> s.snapshotPath.equals(SnapshotMetadata.LIVE_SNAPSHOT_PATH)) - .findFirst() - .get(); + afterSnapshots.stream().filter(SnapshotMetadata::isLive).findFirst().get(); assertThat(liveSnapshot.partitionId).isEqualTo(TEST_KAFKA_PARTITION_ID); assertThat(liveSnapshot.maxOffset).isEqualTo(offset - 1); - assertThat(liveSnapshot.snapshotPath).isEqualTo(SnapshotMetadata.LIVE_SNAPSHOT_PATH); + assertThat(liveSnapshot.isLive()).isTrue(); List afterSearchNodes = AstraMetadataTestUtils.listSyncUncached(searchMetadataStore); assertThat(afterSearchNodes.size()).isEqualTo(1); assertThat(afterSearchNodes.get(0).url).contains(TEST_HOST); assertThat(afterSearchNodes.get(0).url).contains(String.valueOf(TEST_PORT)); - assertThat(afterSearchNodes.get(0).snapshotName) - .contains(SnapshotMetadata.LIVE_SNAPSHOT_PATH); // Check total size of objects uploaded was correctly tracked assertThat(chunk.info().getSizeInBytesOnDisk()) diff --git a/astra/src/test/java/com/slack/astra/chunk/ReadOnlyChunkImplTest.java b/astra/src/test/java/com/slack/astra/chunk/ReadOnlyChunkImplTest.java index d9efa033be..da0f6bbc0d 100644 --- a/astra/src/test/java/com/slack/astra/chunk/ReadOnlyChunkImplTest.java +++ b/astra/src/test/java/com/slack/astra/chunk/ReadOnlyChunkImplTest.java @@ -7,7 +7,6 @@ import static com.slack.astra.logstore.LuceneIndexStoreImpl.MESSAGES_FAILED_COUNTER; import static com.slack.astra.logstore.LuceneIndexStoreImpl.MESSAGES_RECEIVED_COUNTER; import static com.slack.astra.logstore.LuceneIndexStoreImpl.REFRESHES_TIMER; -import static com.slack.astra.proto.metadata.Metadata.IndexType.LOGS_LUCENE9; import static com.slack.astra.testlib.MetricsUtil.getCount; import static com.slack.astra.testlib.MetricsUtil.getTimerCount; import static com.slack.astra.testlib.TemporaryLogStoreAndSearcherExtension.addMessages; @@ -606,7 +605,6 @@ private void assignReplicaToChunk( Metadata.CacheSlotMetadata.CacheSlotState.ASSIGNED, replicaId, Instant.now().toEpochMilli(), - List.of(LOGS_LUCENE9), readOnlyChunk.searchContext.hostname, "rep1"); cacheSlotMetadataStore.updateAsync(updatedCacheSlotMetadata); @@ -619,12 +617,10 @@ private void initializeZkSnapshot( snapshotMetadataStore.createSync( new SnapshotMetadata( snapshotId, - "path", Instant.now().minus(1, ChronoUnit.MINUTES).toEpochMilli(), Instant.now().toEpochMilli(), 1, "partitionId", - LOGS_LUCENE9, sizeInBytesOnDisk)); } @@ -639,8 +635,7 @@ private void initializeZkReplica( "rep1", Instant.now().toEpochMilli(), Instant.now().plusSeconds(60).toEpochMilli(), - false, - LOGS_LUCENE9)); + false)); } private void initializeBlobStorageWithIndex(String snapshotId) throws Exception { diff --git a/astra/src/test/java/com/slack/astra/chunk/RecoveryChunkImplTest.java b/astra/src/test/java/com/slack/astra/chunk/RecoveryChunkImplTest.java index 1b93ec09af..d9710d4350 100644 --- a/astra/src/test/java/com/slack/astra/chunk/RecoveryChunkImplTest.java +++ b/astra/src/test/java/com/slack/astra/chunk/RecoveryChunkImplTest.java @@ -620,7 +620,6 @@ public void testSnapshotToNonExistentS3BucketFails() { // Snapshot to S3 without creating the s3 bucket. assertThat(chunk.snapshotToS3(chunkStore)).isFalse(); - assertThat(chunk.info().getSnapshotPath()).isEqualTo(SnapshotMetadata.LIVE_SNAPSHOT_PATH); // No live snapshot or search metadata is published since the S3 snapshot failed. assertThat(AstraMetadataTestUtils.listSyncUncached(snapshotMetadataStore)).isEmpty(); @@ -671,9 +670,7 @@ public void testSnapshotToS3UsingChunkApi() throws Exception { ChunkStore chunkStore = new ChunkStore(s3AsyncClient, bucket); // Snapshot to S3 - assertThat(chunk.info().getSnapshotPath()).isEqualTo(SnapshotMetadata.LIVE_SNAPSHOT_PATH); assertThat(chunk.snapshotToS3(chunkStore)).isTrue(); - assertThat(chunk.info().getSnapshotPath()).isNotEmpty(); // depending on heap and CFS files this can be 5 or 19. assertThat(getCount(INDEX_FILES_UPLOAD, registry)).isGreaterThan(5); diff --git a/astra/src/test/java/com/slack/astra/chunkManager/CachingChunkManagerTest.java b/astra/src/test/java/com/slack/astra/chunkManager/CachingChunkManagerTest.java index 713989cb1f..67420041d8 100644 --- a/astra/src/test/java/com/slack/astra/chunkManager/CachingChunkManagerTest.java +++ b/astra/src/test/java/com/slack/astra/chunkManager/CachingChunkManagerTest.java @@ -159,9 +159,7 @@ private CachingChunkManager initChunkManager() throws TimeoutExcepti private CacheNodeAssignment initAssignment(String snapshotId) throws Exception { cacheNodeAssignmentStore = new CacheNodeAssignmentStore(curatorFramework); snapshotMetadataStore = new SnapshotMetadataStore(curatorFramework); - snapshotMetadataStore.createSync( - new SnapshotMetadata( - snapshotId, TEST_S3_BUCKET, 1, 1, 0, "abcd", Metadata.IndexType.LOGS_LUCENE9, 29)); + snapshotMetadataStore.createSync(new SnapshotMetadata(snapshotId, 1, 1, 0, "abcd", 29)); CacheNodeAssignment newAssignment = new CacheNodeAssignment( "abcd", diff --git a/astra/src/test/java/com/slack/astra/chunkManager/IndexingChunkManagerTest.java b/astra/src/test/java/com/slack/astra/chunkManager/IndexingChunkManagerTest.java index e0fc0deade..e162601d8c 100644 --- a/astra/src/test/java/com/slack/astra/chunkManager/IndexingChunkManagerTest.java +++ b/astra/src/test/java/com/slack/astra/chunkManager/IndexingChunkManagerTest.java @@ -430,10 +430,9 @@ public void testAddMessage() throws Exception { List liveSnapshots = fetchLiveSnapshot(snapshots); assertThat(liveSnapshots.size()).isEqualTo(1); assertThat(fetchNonLiveSnapshot(snapshots)).isEmpty(); - assertThat(snapshots.get(0).snapshotPath).startsWith(SnapshotMetadata.LIVE_SNAPSHOT_PATH); + assertThat(snapshots.get(0).isLive()).isTrue(); assertThat(snapshots.get(0).maxOffset).isEqualTo(0); assertThat(snapshots.get(0).partitionId).isEqualTo(TEST_KAFKA_PARTITION_ID); - assertThat(snapshots.get(0).snapshotId).startsWith(SnapshotMetadata.LIVE_SNAPSHOT_PATH); assertThat(snapshots.get(0).startTimeEpochMs) .isCloseTo(creationTime.toEpochMilli(), Offset.offset(5000L)); assertThat(snapshots.get(0).endTimeEpochMs).isEqualTo(MAX_FUTURE_TIME); @@ -442,7 +441,6 @@ public void testAddMessage() throws Exception { assertThat(searchNodes.size()).isEqualTo(1); assertThat(searchNodes.get(0).url).contains(TEST_HOST); assertThat(searchNodes.get(0).url).contains(String.valueOf(TEST_PORT)); - assertThat(searchNodes.get(0).snapshotName).contains(SnapshotMetadata.LIVE_SNAPSHOT_PATH); // Add a message with a very high offset. final int veryHighOffset = 1000; @@ -1267,7 +1265,7 @@ public void testSuccessfulRollOverFinishesOnClose() throws Exception { assertThat(fetchLiveSnapshot(snapshots)).isEmpty(); assertThat(snapshots.get(0).maxOffset).isEqualTo(offset - 1); assertThat(snapshots.get(0).endTimeEpochMs).isLessThan(MAX_FUTURE_TIME); - assertThat(snapshots.get(0).snapshotId).doesNotContain(SnapshotMetadata.LIVE_SNAPSHOT_PATH); + assertThat(snapshots.get(0).isLive()).isFalse(); searchMetadataStore.close(); snapshotMetadataStore.close(); } diff --git a/astra/src/test/java/com/slack/astra/clusterManager/CacheNodeAssignmentServiceTest.java b/astra/src/test/java/com/slack/astra/clusterManager/CacheNodeAssignmentServiceTest.java index 994cec8021..c67e4c3b53 100644 --- a/astra/src/test/java/com/slack/astra/clusterManager/CacheNodeAssignmentServiceTest.java +++ b/astra/src/test/java/com/slack/astra/clusterManager/CacheNodeAssignmentServiceTest.java @@ -2,7 +2,6 @@ import static com.slack.astra.clusterManager.CacheNodeAssignmentService.assign; import static com.slack.astra.clusterManager.CacheNodeAssignmentService.sortSnapshotsByReplicaCreationTime; -import static com.slack.astra.proto.metadata.Metadata.IndexType.LOGS_LUCENE9; import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; import static org.mockito.Mockito.spy; @@ -103,8 +102,7 @@ public void testBasicLifecycle() throws Exception { cacheNodeMetadataStore.createSync(cacheNodeMetadata); SnapshotMetadata snapshotMetadata = - new SnapshotMetadata( - "snapshot" + i, "snapshot" + i, 1L, 2L, 10L, "abcd", LOGS_LUCENE9, 5); + new SnapshotMetadata("snapshot" + i, 1L, 2L, 10L, "abcd", 5); snapshotMetadataStore.createSync(snapshotMetadata); ReplicaMetadata replicaMetadata = @@ -114,8 +112,7 @@ public void testBasicLifecycle() throws Exception { "rep1", 1L, Instant.now().plus(15, ChronoUnit.MINUTES).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataStore.createSync(replicaMetadata); } CacheNodeAssignmentService cacheNodeAssignmentService = @@ -161,15 +158,7 @@ public void testExpiredReplicasMarkedForEvictionLifecycle() throws TimeoutExcept cacheNodeAssignmentStore.createSync(newAssignment); SnapshotMetadata snapshotMetadata = - new SnapshotMetadata( - String.format(snapshotKey, i), - String.format(snapshotKey, i), - 1L, - 2L, - 10L, - "abcd", - LOGS_LUCENE9, - 5); + new SnapshotMetadata(String.format(snapshotKey, i), 1L, 2L, 10L, "abcd", 5); snapshotMetadataStore.createSync(snapshotMetadata); ReplicaMetadata replicaMetadata = @@ -179,8 +168,7 @@ public void testExpiredReplicasMarkedForEvictionLifecycle() throws TimeoutExcept replicaSet, 1L, Instant.now().minusSeconds(120).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataStore.createSync(replicaMetadata); } @@ -225,15 +213,7 @@ public void testEvictExpiredReplicasOnly() { cacheNodeAssignmentStore.createSync(newAssignment); SnapshotMetadata snapshotMetadata = - new SnapshotMetadata( - String.format(snapshotKey, i), - String.format(snapshotKey, i), - 1L, - 2L, - 10L, - "abcd", - LOGS_LUCENE9, - 5); + new SnapshotMetadata(String.format(snapshotKey, i), 1L, 2L, 10L, "abcd", 5); snapshotMetadataStore.createSync(snapshotMetadata); ReplicaMetadata replicaMetadata = @@ -243,8 +223,7 @@ public void testEvictExpiredReplicasOnly() { replicaSet, 1L, Instant.now().minusSeconds(120).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataStore.createSync(replicaMetadata); } @@ -297,20 +276,13 @@ public void testShouldntCreateDuplicateAssignments() throws TimeoutException { cacheNodeAssignmentStore.createSync(existingAssignment); // Create snapshot1 and store it in the store - SnapshotMetadata snapshotMetadata = - new SnapshotMetadata(snapshotId, snapshotId, 1L, 2L, 10L, "abcd", LOGS_LUCENE9, 5); + SnapshotMetadata snapshotMetadata = new SnapshotMetadata(snapshotId, 1L, 2L, 10L, "abcd", 5); snapshotMetadataStore.createSync(snapshotMetadata); // Create a replica for snapshot1 ReplicaMetadata replicaMetadata = new ReplicaMetadata( - "replica1", - snapshotId, - replicaSet, - 1L, - Instant.now().toEpochMilli(), - false, - LOGS_LUCENE9); + "replica1", snapshotId, replicaSet, 1L, Instant.now().toEpochMilli(), false); replicaMetadataStore.createSync(replicaMetadata); CacheNodeAssignmentService cacheNodeAssignmentService = @@ -505,8 +477,7 @@ public void testComplyWithMaxConcurrentAssignments() { cacheNodeMetadataStore.createSync(cacheNodeMetadata); SnapshotMetadata snapshotMetadata = - new SnapshotMetadata( - "snapshot" + i, "snapshot" + i, 1L, 2L, 10L, "abcd", LOGS_LUCENE9, 1); + new SnapshotMetadata("snapshot" + i, 1L, 2L, 10L, "abcd", 1); snapshotMetadataStore.createSync(snapshotMetadata); ReplicaMetadata replicaMetadata = @@ -516,8 +487,7 @@ public void testComplyWithMaxConcurrentAssignments() { "rep1", 1L, Instant.now().plus(15, ChronoUnit.MINUTES).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataStore.createSync(replicaMetadata); } CacheNodeAssignmentService cacheNodeAssignmentService = @@ -561,15 +531,7 @@ public void testAssignMostRecentSnapshotsFirst() { Instant now = Instant.now(); for (int i = 0; i < 6; i++) { SnapshotMetadata snapshotMetadata = - new SnapshotMetadata( - String.format(SNAPSHOT_ID_KEY, i), - String.format(SNAPSHOT_ID_KEY, i), - 1L, - 2L, - 5L, - "abcd", - LOGS_LUCENE9, - 10); + new SnapshotMetadata(String.format(SNAPSHOT_ID_KEY, i), 1L, 2L, 5L, "abcd", 10); snapshotMetadataStore.createSync(snapshotMetadata); ReplicaMetadata replicaMetadata = @@ -579,8 +541,7 @@ public void testAssignMostRecentSnapshotsFirst() { "rep1", now.plus(15 * i, ChronoUnit.MINUTES).toEpochMilli(), now.plus(15, ChronoUnit.MINUTES).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataStore.createSync(replicaMetadata); } @@ -624,8 +585,7 @@ public void testSortSnapshotsByReplicaCreationTime() { for (int i = 0; i < 4; i++) { String snapshotId = String.format(SNAPSHOT_ID_KEY, i); - SnapshotMetadata snapshotMetadata = - new SnapshotMetadata(snapshotId, snapshotId, 1L, 2L, 10L, "abcd", LOGS_LUCENE9, 1); + SnapshotMetadata snapshotMetadata = new SnapshotMetadata(snapshotId, 1L, 2L, 10L, "abcd", 1); snapshots.add(snapshotMetadata); ReplicaMetadata replicaMetadata = @@ -635,8 +595,7 @@ public void testSortSnapshotsByReplicaCreationTime() { "rep1", 10 + i, Instant.now().plus(15, ChronoUnit.MINUTES).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicas.put(snapshotId, replicaMetadata); } Collections.shuffle(snapshots); @@ -667,15 +626,7 @@ public void testPacksMostEmptyBinsFirst() { Instant now = Instant.now(); for (int i = 0; i < 3; i++) { SnapshotMetadata snapshotMetadata = - new SnapshotMetadata( - String.format(SNAPSHOT_ID_KEY, i), - String.format(SNAPSHOT_ID_KEY, i), - 1L, - 2L, - 5L, - "abcd", - LOGS_LUCENE9, - 10); + new SnapshotMetadata(String.format(SNAPSHOT_ID_KEY, i), 1L, 2L, 5L, "abcd", 10); snapshotMetadataStore.createSync(snapshotMetadata); ReplicaMetadata replicaMetadata = @@ -685,8 +636,7 @@ public void testPacksMostEmptyBinsFirst() { "rep1", now.plus(15 * i, ChronoUnit.MINUTES).toEpochMilli(), now.plus(15, ChronoUnit.MINUTES).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataStore.createSync(replicaMetadata); } @@ -734,8 +684,7 @@ private static List makeSnapshotsWithSizes(List sizes List snapshots = new ArrayList<>(); for (int i = 0; i < sizes.size(); i++) { Integer size = sizes.get(i); - snapshots.add( - new SnapshotMetadata("snapshot" + i, "/" + i, 1, 2 * 1000, 3, "a", LOGS_LUCENE9, size)); + snapshots.add(new SnapshotMetadata("snapshot" + i, 1, 2 * 1000, 3, "a", size)); } return snapshots; } diff --git a/astra/src/test/java/com/slack/astra/clusterManager/ClusterHpaMetricServiceTest.java b/astra/src/test/java/com/slack/astra/clusterManager/ClusterHpaMetricServiceTest.java index 4241cde782..def9aac1a6 100644 --- a/astra/src/test/java/com/slack/astra/clusterManager/ClusterHpaMetricServiceTest.java +++ b/astra/src/test/java/com/slack/astra/clusterManager/ClusterHpaMetricServiceTest.java @@ -1,7 +1,6 @@ package com.slack.astra.clusterManager; import static com.slack.astra.clusterManager.ClusterHpaMetricService.CACHE_HPA_METRIC_NAME; -import static com.slack.astra.proto.metadata.Metadata.IndexType.LOGS_LUCENE9; import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; import static org.mockito.Mockito.spy; @@ -120,9 +119,9 @@ void oneReplicasetScaledown() { when(replicaMetadataStore.listSync()) .thenReturn( List.of( - new ReplicaMetadata("foo", "foo", "rep1", 1L, 0L, false, LOGS_LUCENE9), - new ReplicaMetadata("bar", "bar", "rep2", 1L, 0L, false, LOGS_LUCENE9), - new ReplicaMetadata("baz", "baz", "rep1", 1L, 0L, false, LOGS_LUCENE9))); + new ReplicaMetadata("foo", "foo", "rep1", 1L, 0L, false), + new ReplicaMetadata("bar", "bar", "rep2", 1L, 0L, false), + new ReplicaMetadata("baz", "baz", "rep1", 1L, 0L, false))); when(cacheSlotMetadataStore.listSync()) .thenReturn( @@ -132,7 +131,6 @@ void oneReplicasetScaledown() { Metadata.CacheSlotMetadata.CacheSlotState.FREE, "", 1, - List.of(LOGS_LUCENE9), "localhost", "rep1"), new CacheSlotMetadata( @@ -140,7 +138,6 @@ void oneReplicasetScaledown() { Metadata.CacheSlotMetadata.CacheSlotState.FREE, "", 1, - List.of(LOGS_LUCENE9), "localhost", "rep1"), new CacheSlotMetadata( @@ -148,7 +145,6 @@ void oneReplicasetScaledown() { Metadata.CacheSlotMetadata.CacheSlotState.FREE, "", 1, - List.of(LOGS_LUCENE9), "localhost", "rep2"), new CacheSlotMetadata( @@ -156,7 +152,6 @@ void oneReplicasetScaledown() { Metadata.CacheSlotMetadata.CacheSlotState.FREE, "", 1, - List.of(LOGS_LUCENE9), "localhost", "rep1"))); @@ -206,8 +201,8 @@ void twoReplicasetScaledown() { when(replicaMetadataStore.listSync()) .thenReturn( List.of( - new ReplicaMetadata("foo", "foo", "rep1", 1L, 0L, false, LOGS_LUCENE9), - new ReplicaMetadata("bar", "bar", "rep2", 1L, 0L, false, LOGS_LUCENE9))); + new ReplicaMetadata("foo", "foo", "rep1", 1L, 0L, false), + new ReplicaMetadata("bar", "bar", "rep2", 1L, 0L, false))); when(cacheSlotMetadataStore.listSync()) .thenReturn( @@ -217,7 +212,6 @@ void twoReplicasetScaledown() { Metadata.CacheSlotMetadata.CacheSlotState.FREE, "", 1, - List.of(LOGS_LUCENE9), "localhost", "rep1"), new CacheSlotMetadata( @@ -225,7 +219,6 @@ void twoReplicasetScaledown() { Metadata.CacheSlotMetadata.CacheSlotState.FREE, "", 1, - List.of(LOGS_LUCENE9), "localhost", "rep1"), new CacheSlotMetadata( @@ -233,7 +226,6 @@ void twoReplicasetScaledown() { Metadata.CacheSlotMetadata.CacheSlotState.FREE, "", 1, - List.of(LOGS_LUCENE9), "localhost", "rep2"), new CacheSlotMetadata( @@ -241,7 +233,6 @@ void twoReplicasetScaledown() { Metadata.CacheSlotMetadata.CacheSlotState.FREE, "", 1, - List.of(LOGS_LUCENE9), "localhost", "rep2"))); @@ -294,10 +285,10 @@ void twoReplicasetScaleup() { when(replicaMetadataStore.listSync()) .thenReturn( List.of( - new ReplicaMetadata("foo", "foo", "rep1", 1L, 0L, false, LOGS_LUCENE9), - new ReplicaMetadata("bar", "bar", "rep1", 1L, 0L, false, LOGS_LUCENE9), - new ReplicaMetadata("baz", "bar", "rep2", 1L, 0L, false, LOGS_LUCENE9), - new ReplicaMetadata("bal", "bar", "rep2", 1L, 0L, false, LOGS_LUCENE9))); + new ReplicaMetadata("foo", "foo", "rep1", 1L, 0L, false), + new ReplicaMetadata("bar", "bar", "rep1", 1L, 0L, false), + new ReplicaMetadata("baz", "bar", "rep2", 1L, 0L, false), + new ReplicaMetadata("bal", "bar", "rep2", 1L, 0L, false))); when(cacheSlotMetadataStore.listSync()) .thenReturn( @@ -307,7 +298,6 @@ void twoReplicasetScaleup() { Metadata.CacheSlotMetadata.CacheSlotState.FREE, "", 1, - List.of(LOGS_LUCENE9), "localhost", "rep1"))); @@ -364,17 +354,15 @@ public void testHpaScaleUp() throws Exception { when(snapshotMetadataStore.listSync()) .thenReturn( List.of( - new SnapshotMetadata( - "snapshot2", "snapshot2", 1L, 2L, 5L, "abcd", LOGS_LUCENE9, 10), - new SnapshotMetadata( - "snapshot1", "snapshot1", 1L, 2L, 5L, "abcd", LOGS_LUCENE9, 5))); + new SnapshotMetadata("snapshot2", 1L, 2L, 5L, "abcd", 10), + new SnapshotMetadata("snapshot1", 1L, 2L, 5L, "abcd", 5))); // Register 1 replica associated with the snapshot when(replicaMetadataStore.listSync()) .thenReturn( List.of( - new ReplicaMetadata("replica2", "snapshot2", "rep1", 1L, 2L, false, LOGS_LUCENE9), - new ReplicaMetadata("replica1", "snapshot1", "rep1", 1L, 2L, false, LOGS_LUCENE9))); + new ReplicaMetadata("replica2", "snapshot2", "rep1", 1L, 2L, false), + new ReplicaMetadata("replica1", "snapshot1", "rep1", 1L, 2L, false))); // Register 2 cache nodes with lots of capacity when(cacheNodeMetadataStore.listSync()) @@ -423,16 +411,11 @@ public void testHpaScaleDown() throws Exception { // Register 1 snapshot when(snapshotMetadataStore.listSync()) - .thenReturn( - List.of( - new SnapshotMetadata( - "snapshot1", "snapshot1", 1L, 2L, 5L, "abcd", LOGS_LUCENE9, 5))); + .thenReturn(List.of(new SnapshotMetadata("snapshot1", 1L, 2L, 5L, "abcd", 5))); // Register 1 replica associated with the snapshot when(replicaMetadataStore.listSync()) - .thenReturn( - List.of( - new ReplicaMetadata("replica1", "snapshot1", "rep1", 1L, 2L, false, LOGS_LUCENE9))); + .thenReturn(List.of(new ReplicaMetadata("replica1", "snapshot1", "rep1", 1L, 2L, false))); // Register 2 cache nodes with lots of capacity when(cacheNodeMetadataStore.listSync()) @@ -476,17 +459,14 @@ public void testHpaReplicaSetFiltering() throws Exception { // Register snapshot of size 15 when(snapshotMetadataStore.listSync()) - .thenReturn( - List.of( - new SnapshotMetadata( - "snapshot1", "snapshot1", 1L, 2L, 5L, "abcd", LOGS_LUCENE9, 15))); + .thenReturn(List.of(new SnapshotMetadata("snapshot1", 1L, 2L, 5L, "abcd", 15))); // Register 2 replicas for rep1 and rep2 when(replicaMetadataStore.listSync()) .thenReturn( List.of( - new ReplicaMetadata("replica2", "snapshot1", "rep1", 1L, 2L, false, LOGS_LUCENE9), - new ReplicaMetadata("replica1", "snapshot1", "rep2", 1L, 2L, false, LOGS_LUCENE9))); + new ReplicaMetadata("replica2", "snapshot1", "rep1", 1L, 2L, false), + new ReplicaMetadata("replica1", "snapshot1", "rep2", 1L, 2L, false))); // Register 2 cache nodes (rep1, rep2), of size 10 each when(cacheNodeMetadataStore.listSync()) diff --git a/astra/src/test/java/com/slack/astra/clusterManager/ReplicaAssignmentServiceTest.java b/astra/src/test/java/com/slack/astra/clusterManager/ReplicaAssignmentServiceTest.java index 6712329334..13e5921f30 100644 --- a/astra/src/test/java/com/slack/astra/clusterManager/ReplicaAssignmentServiceTest.java +++ b/astra/src/test/java/com/slack/astra/clusterManager/ReplicaAssignmentServiceTest.java @@ -2,7 +2,6 @@ import static com.slack.astra.clusterManager.ReplicaAssignmentService.REPLICA_ASSIGN_PENDING; import static com.slack.astra.clusterManager.ReplicaAssignmentService.REPLICA_ASSIGN_TIMER; -import static com.slack.astra.proto.metadata.Metadata.IndexType.LOGS_LUCENE9; import static com.slack.astra.server.AstraConfig.DEFAULT_START_STOP_DURATION; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.AssertionsForClassTypes.assertThatExceptionOfType; @@ -43,8 +42,6 @@ import org.junit.jupiter.api.Test; public class ReplicaAssignmentServiceTest { - - private static final List SUPPORTED_INDEX_TYPES = List.of(LOGS_LUCENE9); public static final String HOSTNAME = "hostname"; public static final String REPLICA_SET = "rep1"; @@ -220,8 +217,7 @@ public void shouldHandleNoAvailableSlots() { REPLICA_SET, Instant.now().toEpochMilli(), Instant.now().plusSeconds(60).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataList.add(replicaMetadata); replicaMetadataStore.createAsync(replicaMetadata); } @@ -278,7 +274,6 @@ public void shouldHandleNoAvailableReplicas() { Metadata.CacheSlotMetadata.CacheSlotState.FREE, "", Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, HOSTNAME, REPLICA_SET); cacheSlotMetadataList.add(cacheSlotMetadata); @@ -338,8 +333,7 @@ public void shouldHandleAllReplicasAlreadyAssigned() { REPLICA_SET, Instant.now().toEpochMilli(), Instant.now().plusSeconds(60).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataList.add(replicaMetadata); replicaMetadataStore.createAsync(replicaMetadata); } @@ -352,7 +346,6 @@ public void shouldHandleAllReplicasAlreadyAssigned() { Metadata.CacheSlotMetadata.CacheSlotState.ASSIGNED, replicaMetadataList.get(i).name, Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, HOSTNAME, REPLICA_SET); cacheSlotMetadataList.add(cacheSlotMetadata); @@ -366,7 +359,6 @@ public void shouldHandleAllReplicasAlreadyAssigned() { Metadata.CacheSlotMetadata.CacheSlotState.FREE, "", Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, HOSTNAME, REPLICA_SET); cacheSlotMetadataList.add(cacheSlotMetadata); @@ -430,8 +422,7 @@ public void shouldHandleMixOfSlotStates() { REPLICA_SET, Instant.now().toEpochMilli(), Instant.now().plusSeconds(60).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataList.add(replicaMetadata); replicaMetadataStore.createAsync(replicaMetadata); } @@ -443,7 +434,6 @@ public void shouldHandleMixOfSlotStates() { Metadata.CacheSlotMetadata.CacheSlotState.ASSIGNED, replicaMetadataList.get(0).name, Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, HOSTNAME, REPLICA_SET); unmutatedSlots.add(cacheSlotWithAssignment); @@ -455,7 +445,6 @@ public void shouldHandleMixOfSlotStates() { Metadata.CacheSlotMetadata.CacheSlotState.LIVE, replicaMetadataList.get(1).name, Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, HOSTNAME, REPLICA_SET); unmutatedSlots.add(cacheSlotLive); @@ -467,7 +456,6 @@ public void shouldHandleMixOfSlotStates() { Metadata.CacheSlotMetadata.CacheSlotState.EVICTING, replicaMetadataList.get(2).name, Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, HOSTNAME, REPLICA_SET); unmutatedSlots.add(cacheSlotEvicting); @@ -479,7 +467,6 @@ public void shouldHandleMixOfSlotStates() { Metadata.CacheSlotMetadata.CacheSlotState.FREE, "", Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, HOSTNAME, REPLICA_SET); cacheSlotMetadataStore.createAsync(cacheSlotFree); @@ -547,8 +534,7 @@ public void shouldHandleAllSlotsAlreadyAssigned() { REPLICA_SET, Instant.now().toEpochMilli(), Instant.now().plusSeconds(60).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataList.add(replicaMetadata); replicaMetadataStore.createAsync(replicaMetadata); } @@ -561,7 +547,6 @@ public void shouldHandleAllSlotsAlreadyAssigned() { Metadata.CacheSlotMetadata.CacheSlotState.ASSIGNED, replicaMetadataList.get(i).name, Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, HOSTNAME, REPLICA_SET); cacheSlotMetadataList.add(cacheSlotMetadata); @@ -625,8 +610,7 @@ public void shouldHandleExpiredReplicas() { REPLICA_SET, Instant.now().minus(1500, ChronoUnit.MINUTES).toEpochMilli(), Instant.now().minusSeconds(60).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataExpiredList.add(replicaMetadata); replicaMetadataStore.createAsync(replicaMetadata); } @@ -639,8 +623,7 @@ public void shouldHandleExpiredReplicas() { REPLICA_SET, Instant.now().minus(1500, ChronoUnit.MINUTES).toEpochMilli(), 0, - false, - LOGS_LUCENE9); + false); replicaMetadataExpiredList.add(replicaMetadataZero); replicaMetadataStore.createAsync(replicaMetadataZero); @@ -652,8 +635,7 @@ public void shouldHandleExpiredReplicas() { REPLICA_SET, Instant.now().toEpochMilli(), Instant.now().plusSeconds(60).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataStore.createAsync(replicaMetadata); } @@ -664,7 +646,6 @@ public void shouldHandleExpiredReplicas() { Metadata.CacheSlotMetadata.CacheSlotState.FREE, "", Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, HOSTNAME, REPLICA_SET); cacheSlotMetadataStore.createAsync(cacheSlotMetadata); @@ -745,8 +726,7 @@ public void shouldRetryFailedAssignmentOnFollowingRun() { REPLICA_SET, Instant.now().toEpochMilli(), Instant.now().plusSeconds(60).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataStore.createAsync(replicaMetadata); } @@ -757,7 +737,6 @@ public void shouldRetryFailedAssignmentOnFollowingRun() { Metadata.CacheSlotMetadata.CacheSlotState.FREE, "", Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, HOSTNAME, REPLICA_SET); cacheSlotMetadataStore.createAsync(cacheSlotMetadata); @@ -870,8 +849,7 @@ public void shouldHandleTimedOutFutures() { REPLICA_SET, Instant.now().toEpochMilli(), Instant.now().plusSeconds(60).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataStore.createAsync(replicaMetadata); } @@ -882,7 +860,6 @@ public void shouldHandleTimedOutFutures() { Metadata.CacheSlotMetadata.CacheSlotState.FREE, "", Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, HOSTNAME, REPLICA_SET); cacheSlotMetadataStore.createAsync(cacheSlotMetadata); @@ -971,8 +948,7 @@ public void shouldHandleExceptionalFutures() { REPLICA_SET, Instant.now().toEpochMilli(), Instant.now().plusSeconds(60).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataStore.createAsync(replicaMetadata); } @@ -983,7 +959,6 @@ public void shouldHandleExceptionalFutures() { Metadata.CacheSlotMetadata.CacheSlotState.FREE, "", Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, HOSTNAME, REPLICA_SET); cacheSlotMetadataStore.createAsync(cacheSlotMetadata); @@ -1058,7 +1033,6 @@ public void shouldHandleSlotsAvailableFirstLifecycle() throws Exception { Metadata.CacheSlotMetadata.CacheSlotState.FREE, "", Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, HOSTNAME, REPLICA_SET); cacheSlotMetadataStore.createAsync(cacheSlotMetadata); @@ -1086,8 +1060,7 @@ public void shouldHandleSlotsAvailableFirstLifecycle() throws Exception { REPLICA_SET, Instant.now().toEpochMilli(), Instant.now().plusSeconds(60).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataStore.createAsync(replicaMetadata); } @@ -1156,8 +1129,7 @@ public void shouldHandleReplicasAvailableFirstLifecycle() throws Exception { REPLICA_SET, Instant.now().toEpochMilli(), Instant.now().plusSeconds(60).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataList.add(replicaMetadata); replicaMetadataStore.createAsync(replicaMetadata); } @@ -1174,7 +1146,6 @@ public void shouldHandleReplicasAvailableFirstLifecycle() throws Exception { Metadata.CacheSlotMetadata.CacheSlotState.FREE, "", Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, HOSTNAME, REPLICA_SET); cacheSlotMetadataStore.createAsync(cacheSlotMetadata); @@ -1249,81 +1220,7 @@ public void shouldPreferNewerReplicasIfLackingCapacity() throws Exception { REPLICA_SET, now.minus(1, ChronoUnit.HOURS).toEpochMilli(), now.plusSeconds(60).toEpochMilli(), - false, - LOGS_LUCENE9); - replicaMetadataStore.createAsync(olderReplicaMetadata); - - ReplicaMetadata newerReplicaMetadata = - new ReplicaMetadata( - UUID.randomUUID().toString(), - UUID.randomUUID().toString(), - REPLICA_SET, - now.toEpochMilli(), - now.plusSeconds(60).toEpochMilli(), - false, - LOGS_LUCENE9); - replicaMetadataStore.createAsync(newerReplicaMetadata); - - await().until(() -> replicaMetadataStore.listSync().size() == 2); - - replicaAssignmentService.startAsync(); - replicaAssignmentService.awaitRunning(DEFAULT_START_STOP_DURATION); - - CacheSlotMetadata cacheSlotMetadata = - new CacheSlotMetadata( - UUID.randomUUID().toString(), - Metadata.CacheSlotMetadata.CacheSlotState.FREE, - "", - Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, - HOSTNAME, - REPLICA_SET); - - cacheSlotMetadataStore.createAsync(cacheSlotMetadata); - await() - .until( - () -> { - List cacheSlotMetadataList = cacheSlotMetadataStore.listSync(); - return cacheSlotMetadataList.size() == 1 - && cacheSlotMetadataList.get(0).cacheSlotState - == Metadata.CacheSlotMetadata.CacheSlotState.ASSIGNED; - }); - - List assignedCacheSlot = - AstraMetadataTestUtils.listSyncUncached(cacheSlotMetadataStore); - assertThat(assignedCacheSlot.get(0).replicaId).isEqualTo(newerReplicaMetadata.name); - assertThat(assignedCacheSlot.get(0).supportedIndexTypes).containsAll(SUPPORTED_INDEX_TYPES); - } - - @Test - public void assignmentPreservesSupportedIndexTypes() throws Exception { - AstraConfigs.ManagerConfig.ReplicaAssignmentServiceConfig replicaAssignmentServiceConfig = - AstraConfigs.ManagerConfig.ReplicaAssignmentServiceConfig.newBuilder() - .setSchedulePeriodMins(1) - .addAllReplicaSets(List.of(REPLICA_SET)) - .setMaxConcurrentPerNode(2) - .build(); - AstraConfigs.ManagerConfig managerConfig = - AstraConfigs.ManagerConfig.newBuilder() - .setEventAggregationSecs(2) - .setScheduleInitialDelayMins(1) - .setReplicaAssignmentServiceConfig(replicaAssignmentServiceConfig) - .build(); - - ReplicaAssignmentService replicaAssignmentService = - new ReplicaAssignmentService( - cacheSlotMetadataStore, replicaMetadataStore, managerConfig, meterRegistry); - - Instant now = Instant.now(); - ReplicaMetadata olderReplicaMetadata = - new ReplicaMetadata( - UUID.randomUUID().toString(), - UUID.randomUUID().toString(), - REPLICA_SET, - now.minus(1, ChronoUnit.HOURS).toEpochMilli(), - now.plusSeconds(60).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataStore.createAsync(olderReplicaMetadata); ReplicaMetadata newerReplicaMetadata = @@ -1333,8 +1230,7 @@ public void assignmentPreservesSupportedIndexTypes() throws Exception { REPLICA_SET, now.toEpochMilli(), now.plusSeconds(60).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataStore.createAsync(newerReplicaMetadata); await().until(() -> replicaMetadataStore.listSync().size() == 2); @@ -1342,14 +1238,12 @@ public void assignmentPreservesSupportedIndexTypes() throws Exception { replicaAssignmentService.startAsync(); replicaAssignmentService.awaitRunning(DEFAULT_START_STOP_DURATION); - final List suppportedIndexTypes = List.of(LOGS_LUCENE9, LOGS_LUCENE9); CacheSlotMetadata cacheSlotMetadata = new CacheSlotMetadata( UUID.randomUUID().toString(), Metadata.CacheSlotMetadata.CacheSlotState.FREE, "", Instant.now().toEpochMilli(), - suppportedIndexTypes, HOSTNAME, REPLICA_SET); @@ -1366,9 +1260,6 @@ public void assignmentPreservesSupportedIndexTypes() throws Exception { List assignedCacheSlot = AstraMetadataTestUtils.listSyncUncached(cacheSlotMetadataStore); assertThat(assignedCacheSlot.get(0).replicaId).isEqualTo(newerReplicaMetadata.name); - assertThat(assignedCacheSlot.get(0).supportedIndexTypes) - .containsExactlyInAnyOrderElementsOf(suppportedIndexTypes); - assertThat(assignedCacheSlot.size()).isEqualTo(1); } @Test @@ -1403,8 +1294,7 @@ public void shouldNotAssignIfAlreadyLoading() throws Exception { REPLICA_SET, now.minus(1, ChronoUnit.HOURS).toEpochMilli(), now.plusSeconds(60).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataStore.createAsync(expectedUnassignedMetadata); ReplicaMetadata loadingMetadata = @@ -1414,8 +1304,7 @@ public void shouldNotAssignIfAlreadyLoading() throws Exception { REPLICA_SET, now.minus(1, ChronoUnit.HOURS).toEpochMilli(), now.plusSeconds(60).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataStore.createAsync(loadingMetadata); await().until(() -> replicaMetadataStore.listSync().size() == 2); @@ -1426,7 +1315,6 @@ public void shouldNotAssignIfAlreadyLoading() throws Exception { Metadata.CacheSlotMetadata.CacheSlotState.LOADING, loadingMetadata.snapshotId, Instant.now().toEpochMilli(), - List.of(LOGS_LUCENE9, LOGS_LUCENE9), HOSTNAME, REPLICA_SET); cacheSlotMetadataStore.createAsync(cacheSlotMetadata); @@ -1437,7 +1325,6 @@ public void shouldNotAssignIfAlreadyLoading() throws Exception { Metadata.CacheSlotMetadata.CacheSlotState.FREE, "", Instant.now().toEpochMilli(), - List.of(LOGS_LUCENE9, LOGS_LUCENE9), HOSTNAME, REPLICA_SET); cacheSlotMetadataStore.createAsync(freeCacheSlot); @@ -1484,8 +1371,7 @@ public void shouldPreventConcurrentAssignmentsExceedingLimit() throws Exception REPLICA_SET, now.minus(1, ChronoUnit.HOURS).toEpochMilli(), now.plusSeconds(60).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataStore.createAsync(expectedAssignedMetadata1); ReplicaMetadata expectedAssignedMetadata2 = @@ -1495,8 +1381,7 @@ public void shouldPreventConcurrentAssignmentsExceedingLimit() throws Exception REPLICA_SET, now.minus(1, ChronoUnit.HOURS).toEpochMilli(), now.plusSeconds(60).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataStore.createAsync(expectedAssignedMetadata2); ReplicaMetadata expectedUnassignedMetadata = @@ -1506,8 +1391,7 @@ public void shouldPreventConcurrentAssignmentsExceedingLimit() throws Exception REPLICA_SET, now.toEpochMilli(), now.plusSeconds(60).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataStore.createAsync(expectedUnassignedMetadata); await().until(() -> replicaMetadataStore.listSync().size() == 3); @@ -1520,7 +1404,6 @@ public void shouldPreventConcurrentAssignmentsExceedingLimit() throws Exception Metadata.CacheSlotMetadata.CacheSlotState.FREE, "", Instant.now().toEpochMilli(), - List.of(LOGS_LUCENE9, LOGS_LUCENE9), HOSTNAME, REPLICA_SET); cacheSlotMetadataStore.createAsync(cacheSlotMetadata); diff --git a/astra/src/test/java/com/slack/astra/clusterManager/ReplicaCreationServiceTest.java b/astra/src/test/java/com/slack/astra/clusterManager/ReplicaCreationServiceTest.java index 9a11a7b2e1..43ea2bfe84 100644 --- a/astra/src/test/java/com/slack/astra/clusterManager/ReplicaCreationServiceTest.java +++ b/astra/src/test/java/com/slack/astra/clusterManager/ReplicaCreationServiceTest.java @@ -1,6 +1,5 @@ package com.slack.astra.clusterManager; -import static com.slack.astra.proto.metadata.Metadata.IndexType.LOGS_LUCENE9; import static com.slack.astra.server.AstraConfig.DEFAULT_START_STOP_DURATION; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.AssertionsForClassTypes.assertThatExceptionOfType; @@ -89,14 +88,7 @@ public void shouldDoNothingIfReplicasAlreadyExist() throws Exception { SnapshotMetadata snapshotA = new SnapshotMetadata( - "a", - "a", - Instant.now().toEpochMilli() - 1, - Instant.now().toEpochMilli(), - 0, - "a", - LOGS_LUCENE9, - 0); + "a", Instant.now().toEpochMilli() - 1, Instant.now().toEpochMilli(), 0, "a", 0); snapshotMetadataStore.createSync(snapshotA); replicaMetadataStore.createSync( @@ -140,14 +132,7 @@ public void shouldDoNothingIfReplicasAlreadyExist() throws Exception { public void shouldCreateZeroReplicasNoneConfigured() throws Exception { SnapshotMetadata snapshotA = new SnapshotMetadata( - "a", - "a", - Instant.now().toEpochMilli() - 1, - Instant.now().toEpochMilli(), - 0, - "a", - LOGS_LUCENE9, - 0); + "a", Instant.now().toEpochMilli() - 1, Instant.now().toEpochMilli(), 0, "a", 0); snapshotMetadataStore.createSync(snapshotA); AstraConfigs.ManagerConfig.ReplicaCreationServiceConfig replicaCreationServiceConfig = @@ -184,14 +169,7 @@ public void shouldCreateZeroReplicasNoneConfigured() throws Exception { public void shouldCreateFourReplicasIfNoneExist() throws Exception { SnapshotMetadata snapshotA = new SnapshotMetadata( - "a", - "a", - Instant.now().toEpochMilli() - 1, - Instant.now().toEpochMilli(), - 0, - "a", - LOGS_LUCENE9, - 0); + "a", Instant.now().toEpochMilli() - 1, Instant.now().toEpochMilli(), 0, "a", 100); snapshotMetadataStore.createSync(snapshotA); AstraConfigs.ManagerConfig.ReplicaCreationServiceConfig replicaCreationServiceConfig = @@ -237,26 +215,12 @@ public void shouldCreateFourReplicasIfNoneExist() throws Exception { public void shouldNotCreateReplicasForLiveSnapshots() { SnapshotMetadata snapshotNotLive = new SnapshotMetadata( - "a", - "a", - Instant.now().toEpochMilli() - 1, - Instant.now().toEpochMilli(), - 0, - "a", - LOGS_LUCENE9, - 0); + "a", Instant.now().toEpochMilli() - 1, Instant.now().toEpochMilli(), 0, "a", 100); snapshotMetadataStore.createSync(snapshotNotLive); SnapshotMetadata snapshotLive = new SnapshotMetadata( - "b", - SnapshotMetadata.LIVE_SNAPSHOT_PATH, - Instant.now().toEpochMilli() - 1, - Instant.now().toEpochMilli(), - 0, - "b", - LOGS_LUCENE9, - 0); + "b", Instant.now().toEpochMilli() - 1, Instant.now().toEpochMilli(), 0, "b", 0); snapshotMetadataStore.createSync(snapshotLive); AstraConfigs.ManagerConfig.ReplicaCreationServiceConfig replicaCreationServiceConfig = @@ -334,14 +298,12 @@ public void shouldHandleVeryLargeListOfIneligibleSnapshots() { String snapshotId = UUID.randomUUID().toString(); SnapshotMetadata snapshot = new SnapshotMetadata( - snapshotId, snapshotId, Instant.now().minus(1450, ChronoUnit.MINUTES).toEpochMilli(), Instant.now().minus(1441, ChronoUnit.MINUTES).toEpochMilli(), 0, snapshotId, - LOGS_LUCENE9, - 0); + 100); snapshotList.add(snapshot); }); @@ -352,12 +314,10 @@ public void shouldHandleVeryLargeListOfIneligibleSnapshots() { SnapshotMetadata snapshot = new SnapshotMetadata( snapshotId, - SnapshotMetadata.LIVE_SNAPSHOT_PATH, Instant.now().toEpochMilli() - 1, Instant.now().toEpochMilli(), 0, snapshotId, - LOGS_LUCENE9, 0); snapshotList.add(snapshot); }); @@ -369,14 +329,12 @@ public void shouldHandleVeryLargeListOfIneligibleSnapshots() { String snapshotId = UUID.randomUUID().toString(); SnapshotMetadata snapshot = new SnapshotMetadata( - snapshotId, snapshotId, Instant.now().toEpochMilli() - 1, Instant.now().toEpochMilli(), 0, snapshotId, - LOGS_LUCENE9, - 0); + 100); eligibleSnapshots.add(snapshot); }); snapshotList.addAll(eligibleSnapshots); @@ -456,14 +414,7 @@ public void shouldCreateReplicaWhenSnapshotAddedAfterRunning() throws Exception // EventAggregationSecs duration, attempt to create the replicas SnapshotMetadata snapshotA = new SnapshotMetadata( - "a", - "a", - Instant.now().toEpochMilli() - 1, - Instant.now().toEpochMilli(), - 0, - "a", - LOGS_LUCENE9, - 0); + "a", Instant.now().toEpochMilli() - 1, Instant.now().toEpochMilli(), 0, "a", 100); snapshotMetadataStore.createSync(snapshotA); await().until(() -> replicaMetadataStore.listSync().size() == 2); @@ -529,14 +480,7 @@ public void shouldStillCreateReplicaIfFirstAttemptFails() throws Exception { // attempt to create the replicas SnapshotMetadata snapshotA = new SnapshotMetadata( - "a", - "a", - Instant.now().toEpochMilli() - 1, - Instant.now().toEpochMilli(), - 0, - "a", - LOGS_LUCENE9, - 0); + "a", Instant.now().toEpochMilli() - 1, Instant.now().toEpochMilli(), 0, "a", 100); snapshotMetadataStore.createSync(snapshotA); await() @@ -582,14 +526,7 @@ public void shouldStillCreateReplicaIfFirstAttemptFails() throws Exception { public void shouldHandleFailedCreateFutures() { SnapshotMetadata snapshotA = new SnapshotMetadata( - "a", - "a", - Instant.now().toEpochMilli() - 1, - Instant.now().toEpochMilli(), - 0, - "a", - LOGS_LUCENE9, - 0); + "a", Instant.now().toEpochMilli() - 1, Instant.now().toEpochMilli(), 0, "a", 100); snapshotMetadataStore.createSync(snapshotA); await().until(() -> snapshotMetadataStore.listSync().size() == 1); @@ -632,14 +569,7 @@ public void shouldHandleFailedCreateFutures() { public void shouldHandleMixOfSuccessfulFailedZkFutures() { SnapshotMetadata snapshotA = new SnapshotMetadata( - "a", - "a", - Instant.now().toEpochMilli() - 1, - Instant.now().toEpochMilli(), - 0, - "a", - LOGS_LUCENE9, - 0); + "a", Instant.now().toEpochMilli() - 1, Instant.now().toEpochMilli(), 0, "a", 100); snapshotMetadataStore.createSync(snapshotA); await().until(() -> snapshotMetadataStore.listSync().size() == 1); diff --git a/astra/src/test/java/com/slack/astra/clusterManager/ReplicaDeletionServiceTest.java b/astra/src/test/java/com/slack/astra/clusterManager/ReplicaDeletionServiceTest.java index 5175f0ffac..eb13a32cf7 100644 --- a/astra/src/test/java/com/slack/astra/clusterManager/ReplicaDeletionServiceTest.java +++ b/astra/src/test/java/com/slack/astra/clusterManager/ReplicaDeletionServiceTest.java @@ -1,6 +1,5 @@ package com.slack.astra.clusterManager; -import static com.slack.astra.proto.metadata.Metadata.IndexType.LOGS_LUCENE9; import static com.slack.astra.server.AstraConfig.DEFAULT_START_STOP_DURATION; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.AssertionsForClassTypes.assertThatExceptionOfType; @@ -41,7 +40,6 @@ import org.junit.jupiter.api.Test; public class ReplicaDeletionServiceTest { - public static final List SUPPORTED_INDEX_TYPES = List.of(LOGS_LUCENE9); public static final String HOSTNAME = "hostname"; private static final String REPLICA_SET = "rep1"; private TestingServer testingServer; @@ -161,8 +159,7 @@ public void shouldDeleteExpiredReplicaWithoutAssignment() { REPLICA_SET, Instant.now().minusSeconds(30).toEpochMilli(), Instant.now().minusSeconds(10).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataStore.createAsync(replicaMetadata); CacheSlotMetadata cacheSlotMetadata = @@ -171,7 +168,6 @@ public void shouldDeleteExpiredReplicaWithoutAssignment() { Metadata.CacheSlotMetadata.CacheSlotState.FREE, "", Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, HOSTNAME, REPLICA_SET); cacheSlotMetadataStore.createAsync(cacheSlotMetadata); @@ -224,8 +220,7 @@ public void shouldNotDeleteExpiredReplicasWithAssignments() { REPLICA_SET, Instant.now().minusSeconds(30).toEpochMilli(), Instant.now().minusSeconds(10).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataList.add(replicaMetadata); replicaMetadataStore.createAsync(replicaMetadata); } @@ -236,7 +231,6 @@ public void shouldNotDeleteExpiredReplicasWithAssignments() { Metadata.CacheSlotMetadata.CacheSlotState.ASSIGNED, replicaMetadataList.get(0).name, Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, HOSTNAME, REPLICA_SET); cacheSlotMetadataStore.createAsync(cacheSlotMetadataAssigned); @@ -247,7 +241,6 @@ public void shouldNotDeleteExpiredReplicasWithAssignments() { Metadata.CacheSlotMetadata.CacheSlotState.EVICT, replicaMetadataList.get(1).name, Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, HOSTNAME, REPLICA_SET); cacheSlotMetadataStore.createAsync(cacheSlotMetadataEvict); @@ -258,7 +251,6 @@ public void shouldNotDeleteExpiredReplicasWithAssignments() { Metadata.CacheSlotMetadata.CacheSlotState.EVICTING, replicaMetadataList.get(2).name, Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, HOSTNAME, REPLICA_SET); cacheSlotMetadataStore.createAsync(cacheSlotMetadataEvicting); @@ -312,8 +304,7 @@ public void shouldNotDeleteUnexpiredReplicaWithoutAssignment() { REPLICA_SET, Instant.now().minusSeconds(30).toEpochMilli(), Instant.now().plusSeconds(30).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataStore.createAsync(replicaMetadata); CacheSlotMetadata cacheSlotMetadata = @@ -322,7 +313,6 @@ public void shouldNotDeleteUnexpiredReplicaWithoutAssignment() { Metadata.CacheSlotMetadata.CacheSlotState.FREE, "", Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, HOSTNAME, REPLICA_SET); cacheSlotMetadataStore.createAsync(cacheSlotMetadata); @@ -373,8 +363,7 @@ public void shouldRetryFailedDeleteAttempt() { REPLICA_SET, Instant.now().minusSeconds(30).toEpochMilli(), Instant.now().minusSeconds(10).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataStore.createAsync(replicaMetadata); CacheSlotMetadata cacheSlotMetadata = @@ -383,7 +372,6 @@ public void shouldRetryFailedDeleteAttempt() { Metadata.CacheSlotMetadata.CacheSlotState.FREE, "", Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, HOSTNAME, REPLICA_SET); cacheSlotMetadataStore.createAsync(cacheSlotMetadata); @@ -458,8 +446,7 @@ public void shouldHandleMixOfZkSuccessFailure() { REPLICA_SET, Instant.now().minusSeconds(30).toEpochMilli(), Instant.now().minusSeconds(10).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataStore.createAsync(replicaMetadata); } @@ -471,7 +458,6 @@ public void shouldHandleMixOfZkSuccessFailure() { Metadata.CacheSlotMetadata.CacheSlotState.FREE, "", Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, HOSTNAME, REPLICA_SET); cacheSlotMetadataList.add(cacheSlotMetadata); @@ -564,8 +550,7 @@ public void shouldHandleDeletionLifecycle() throws TimeoutException { REPLICA_SET, Instant.now().minusSeconds(30).toEpochMilli(), Instant.now().minusSeconds(10).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataStore.createAsync(replicaMetadataUnassigned); ReplicaMetadata replicaMetadataAssigned = @@ -575,8 +560,7 @@ public void shouldHandleDeletionLifecycle() throws TimeoutException { REPLICA_SET, Instant.now().minusSeconds(30).toEpochMilli(), Instant.now().minusSeconds(10).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataStore.createAsync(replicaMetadataAssigned); CacheSlotMetadata cacheSlotMetadataUnassigned = @@ -585,7 +569,6 @@ public void shouldHandleDeletionLifecycle() throws TimeoutException { Metadata.CacheSlotMetadata.CacheSlotState.FREE, "", Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, HOSTNAME, REPLICA_SET); cacheSlotMetadataStore.createAsync(cacheSlotMetadataUnassigned); @@ -596,7 +579,6 @@ public void shouldHandleDeletionLifecycle() throws TimeoutException { Metadata.CacheSlotMetadata.CacheSlotState.ASSIGNED, replicaMetadataAssigned.name, Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, HOSTNAME, REPLICA_SET); cacheSlotMetadataStore.createAsync(cacheSlotMetadataAssigned); diff --git a/astra/src/test/java/com/slack/astra/clusterManager/ReplicaEvictionServiceTest.java b/astra/src/test/java/com/slack/astra/clusterManager/ReplicaEvictionServiceTest.java index 3259fa7ab2..2b87db2ca6 100644 --- a/astra/src/test/java/com/slack/astra/clusterManager/ReplicaEvictionServiceTest.java +++ b/astra/src/test/java/com/slack/astra/clusterManager/ReplicaEvictionServiceTest.java @@ -1,6 +1,5 @@ package com.slack.astra.clusterManager; -import static com.slack.astra.proto.metadata.Metadata.IndexType.LOGS_LUCENE9; import static com.slack.astra.server.AstraConfig.DEFAULT_START_STOP_DURATION; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.AssertionsForClassTypes.assertThatExceptionOfType; @@ -41,7 +40,6 @@ import org.junit.jupiter.api.Test; public class ReplicaEvictionServiceTest { - private static final List SUPPORTED_INDEX_TYPES = List.of(LOGS_LUCENE9); public static final String HOSTNAME = "hostname"; public static final String REPLICA_SET = "rep1"; private TestingServer testingServer; @@ -164,8 +162,7 @@ public void shouldHandleNoExpiredCacheSlots() { REPLICA_SET, Instant.now().toEpochMilli(), Instant.now().plusSeconds(60).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicas.add(replicaMetadata); replicaMetadataStore.createAsync(replicaMetadata); } @@ -177,7 +174,6 @@ public void shouldHandleNoExpiredCacheSlots() { Metadata.CacheSlotMetadata.CacheSlotState.ASSIGNED, replicas.get(0).name, Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, HOSTNAME, REPLICA_SET); cacheSlots.add(cacheSlotAssigned); @@ -189,7 +185,6 @@ public void shouldHandleNoExpiredCacheSlots() { Metadata.CacheSlotMetadata.CacheSlotState.LIVE, replicas.get(1).name, Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, HOSTNAME, REPLICA_SET); cacheSlots.add(cacheSlotLive); @@ -201,7 +196,6 @@ public void shouldHandleNoExpiredCacheSlots() { Metadata.CacheSlotMetadata.CacheSlotState.LOADING, replicas.get(2).name, Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, HOSTNAME, REPLICA_SET); cacheSlots.add(cacheSlotLoading); @@ -229,88 +223,6 @@ public void shouldHandleNoExpiredCacheSlots() { .isEqualTo(1); } - @Test - public void shouldPreserveSupportedIndexTypesOnEviction() { - AstraConfigs.ManagerConfig.ReplicaEvictionServiceConfig replicaEvictionServiceConfig = - AstraConfigs.ManagerConfig.ReplicaEvictionServiceConfig.newBuilder() - .setSchedulePeriodMins(10) - .build(); - - AstraConfigs.ManagerConfig managerConfig = - AstraConfigs.ManagerConfig.newBuilder() - .setReplicaEvictionServiceConfig(replicaEvictionServiceConfig) - .setEventAggregationSecs(10) - .setScheduleInitialDelayMins(10) - .build(); - - ReplicaEvictionService replicaEvictionService = - new ReplicaEvictionService( - cacheSlotMetadataStore, replicaMetadataStore, managerConfig, meterRegistry); - - ReplicaMetadata replicaMetadata = - new ReplicaMetadata( - UUID.randomUUID().toString(), - UUID.randomUUID().toString(), - REPLICA_SET, - Instant.now().toEpochMilli(), - 0, - false, - LOGS_LUCENE9); - replicaMetadataStore.createAsync(replicaMetadata); - - // TODO: Update list with different index types when we have more types. - final List supportedIndexTypes = List.of(LOGS_LUCENE9, LOGS_LUCENE9); - CacheSlotMetadata cacheSlotMetadata = - new CacheSlotMetadata( - UUID.randomUUID().toString(), - Metadata.CacheSlotMetadata.CacheSlotState.LIVE, - replicaMetadata.name, - Instant.now().toEpochMilli(), - supportedIndexTypes, - HOSTNAME, - REPLICA_SET); - cacheSlotMetadataStore.createAsync(cacheSlotMetadata); - assertThat(cacheSlotMetadata.supportedIndexTypes.size()).isEqualTo(2); - - await().until(() -> replicaMetadataStore.listSync().size() == 1); - await().until(() -> cacheSlotMetadataStore.listSync().size() == 1); - - int replicasMarked = replicaEvictionService.markReplicasForEviction(Instant.now()); - assertThat(replicasMarked).isEqualTo(1); - - assertThat(replicaMetadataStore.listSync()).containsExactly(replicaMetadata); - await() - .until( - () -> - cacheSlotMetadataStore.listSync().stream() - .allMatch( - cacheSlot -> - cacheSlot.cacheSlotState.equals( - Metadata.CacheSlotMetadata.CacheSlotState.EVICT))); - - CacheSlotMetadata updatedCacheSlot = cacheSlotMetadataStore.listSync().get(0); - assertThat(updatedCacheSlot.updatedTimeEpochMs) - .isGreaterThan(cacheSlotMetadata.updatedTimeEpochMs); - assertThat(updatedCacheSlot.cacheSlotState) - .isEqualTo(Metadata.CacheSlotMetadata.CacheSlotState.EVICT); - assertThat(updatedCacheSlot.name).isEqualTo(cacheSlotMetadata.name); - assertThat(updatedCacheSlot.replicaId).isEqualTo(cacheSlotMetadata.replicaId); - assertThat(updatedCacheSlot.supportedIndexTypes) - .containsExactlyInAnyOrderElementsOf(cacheSlotMetadata.supportedIndexTypes); - - assertThat( - MetricsUtil.getCount( - ReplicaEvictionService.REPLICA_MARK_EVICT_SUCCEEDED, meterRegistry)) - .isEqualTo(1); - assertThat( - MetricsUtil.getCount(ReplicaEvictionService.REPLICA_MARK_EVICT_FAILED, meterRegistry)) - .isEqualTo(0); - assertThat( - MetricsUtil.getTimerCount( - ReplicaEvictionService.REPLICA_MARK_EVICT_TIMER, meterRegistry)) - .isEqualTo(1); - } - @Test public void shouldEvictReplicaWithEmptyExpiration() { AstraConfigs.ManagerConfig.ReplicaEvictionServiceConfig replicaEvictionServiceConfig = @@ -336,8 +248,7 @@ public void shouldEvictReplicaWithEmptyExpiration() { REPLICA_SET, Instant.now().toEpochMilli(), 0, - false, - LOGS_LUCENE9); + false); replicaMetadataStore.createAsync(replicaMetadata); CacheSlotMetadata cacheSlotMetadata = @@ -346,7 +257,6 @@ public void shouldEvictReplicaWithEmptyExpiration() { Metadata.CacheSlotMetadata.CacheSlotState.LIVE, replicaMetadata.name, Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, HOSTNAME, REPLICA_SET); cacheSlotMetadataStore.createAsync(cacheSlotMetadata); @@ -413,8 +323,7 @@ public void shouldNotMutateReplicaAlreadyMarkedForEviction() { REPLICA_SET, Instant.now().toEpochMilli(), Instant.now().minusSeconds(60).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataStore.createAsync(replicaMetadata); CacheSlotMetadata cacheSlotMetadata = @@ -423,7 +332,6 @@ public void shouldNotMutateReplicaAlreadyMarkedForEviction() { Metadata.CacheSlotMetadata.CacheSlotState.EVICT, replicaMetadata.name, Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, HOSTNAME, REPLICA_SET); cacheSlotMetadataStore.createAsync(cacheSlotMetadata); @@ -475,8 +383,7 @@ public void shouldNotMutateReplicaAlreadyEvicting() { REPLICA_SET, Instant.now().toEpochMilli(), Instant.now().minusSeconds(60).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataStore.createAsync(replicaMetadata); CacheSlotMetadata cacheSlotMetadata = @@ -485,7 +392,6 @@ public void shouldNotMutateReplicaAlreadyEvicting() { Metadata.CacheSlotMetadata.CacheSlotState.EVICTING, replicaMetadata.name, Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, HOSTNAME, REPLICA_SET); cacheSlotMetadataStore.createAsync(cacheSlotMetadata); @@ -537,8 +443,7 @@ public void shouldRetryFailedEvictionOnNextRun() { REPLICA_SET, Instant.now().toEpochMilli(), Instant.now().minusSeconds(60).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataStore.createAsync(replicaMetadata); CacheSlotMetadata cacheSlotMetadata = @@ -547,7 +452,6 @@ public void shouldRetryFailedEvictionOnNextRun() { Metadata.CacheSlotMetadata.CacheSlotState.LIVE, replicaMetadata.name, Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, HOSTNAME, REPLICA_SET); cacheSlotMetadataStore.createAsync(cacheSlotMetadata); @@ -644,8 +548,7 @@ public void shouldHandleMixOfZkSuccessFailures() { REPLICA_SET, Instant.now().toEpochMilli(), Instant.now().minusSeconds(60).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicas.add(replicaMetadata); replicaMetadataStore.createAsync(replicaMetadata); } @@ -657,7 +560,6 @@ public void shouldHandleMixOfZkSuccessFailures() { Metadata.CacheSlotMetadata.CacheSlotState.LIVE, replicas.get(0).name, Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, HOSTNAME, REPLICA_SET); cacheSlotMetadataStore.createAsync(cacheSlotMetadata); @@ -774,8 +676,7 @@ public void shouldHandleMixOfExpiredAndUnexpiredLifecycle() throws TimeoutExcept REPLICA_SET, Instant.now().toEpochMilli(), Instant.now().minusSeconds(60).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataStore.createAsync(replicaMetadataExpiredOne); CacheSlotMetadata cacheSlotReplicaExpiredOne = new CacheSlotMetadata( @@ -783,7 +684,6 @@ public void shouldHandleMixOfExpiredAndUnexpiredLifecycle() throws TimeoutExcept Metadata.CacheSlotMetadata.CacheSlotState.LIVE, replicaMetadataExpiredOne.name, Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, HOSTNAME, REPLICA_SET); cacheSlotMetadataStore.createAsync(cacheSlotReplicaExpiredOne); @@ -795,8 +695,7 @@ public void shouldHandleMixOfExpiredAndUnexpiredLifecycle() throws TimeoutExcept REPLICA_SET, Instant.now().toEpochMilli(), Instant.now().minusSeconds(60).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataStore.createAsync(replicaMetadataExpiredTwo); CacheSlotMetadata cacheSlotReplicaExpireTwo = new CacheSlotMetadata( @@ -804,7 +703,6 @@ public void shouldHandleMixOfExpiredAndUnexpiredLifecycle() throws TimeoutExcept Metadata.CacheSlotMetadata.CacheSlotState.EVICT, replicaMetadataExpiredTwo.name, Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, HOSTNAME, REPLICA_SET); cacheSlotMetadataStore.createAsync(cacheSlotReplicaExpireTwo); @@ -816,8 +714,7 @@ public void shouldHandleMixOfExpiredAndUnexpiredLifecycle() throws TimeoutExcept REPLICA_SET, Instant.now().toEpochMilli(), Instant.now().plusSeconds(360).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataStore.createAsync(replicaMetadataUnexpiredOne); CacheSlotMetadata cacheSlotReplicaUnexpiredOne = new CacheSlotMetadata( @@ -825,7 +722,6 @@ public void shouldHandleMixOfExpiredAndUnexpiredLifecycle() throws TimeoutExcept Metadata.CacheSlotMetadata.CacheSlotState.LIVE, replicaMetadataUnexpiredOne.name, Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, HOSTNAME, REPLICA_SET); cacheSlotMetadataStore.createAsync(cacheSlotReplicaUnexpiredOne); @@ -837,8 +733,7 @@ public void shouldHandleMixOfExpiredAndUnexpiredLifecycle() throws TimeoutExcept REPLICA_SET, Instant.now().toEpochMilli(), Instant.now().plusSeconds(360).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataStore.createAsync(replicaMetadataUnexpiredTwo); CacheSlotMetadata cacheSlotFree = new CacheSlotMetadata( @@ -846,7 +741,6 @@ public void shouldHandleMixOfExpiredAndUnexpiredLifecycle() throws TimeoutExcept Metadata.CacheSlotMetadata.CacheSlotState.FREE, "", Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, HOSTNAME, REPLICA_SET); cacheSlotMetadataStore.createAsync(cacheSlotFree); diff --git a/astra/src/test/java/com/slack/astra/clusterManager/ReplicaRestoreServiceTest.java b/astra/src/test/java/com/slack/astra/clusterManager/ReplicaRestoreServiceTest.java index f81fc5a494..b3a6f793a2 100644 --- a/astra/src/test/java/com/slack/astra/clusterManager/ReplicaRestoreServiceTest.java +++ b/astra/src/test/java/com/slack/astra/clusterManager/ReplicaRestoreServiceTest.java @@ -1,6 +1,5 @@ package com.slack.astra.clusterManager; -import static com.slack.astra.proto.metadata.Metadata.IndexType.LOGS_LUCENE9; import static org.assertj.core.api.AssertionsForClassTypes.assertThat; import static org.assertj.core.api.AssertionsForClassTypes.assertThatExceptionOfType; import static org.assertj.core.api.AssertionsForClassTypes.fail; @@ -96,8 +95,7 @@ public void shouldHandleDrainingAndAdding() throws Exception { for (int i = 0; i < 10; i++) { long now = Instant.now().toEpochMilli(); String id = "loop" + i; - SnapshotMetadata snapshotIncluded = - new SnapshotMetadata(id, id, now + 10, now + 15, 0, id, LOGS_LUCENE9, 0); + SnapshotMetadata snapshotIncluded = new SnapshotMetadata(id, now + 10, now + 15, 0, id, 0); replicaRestoreService.queueSnapshotsForRestoration(List.of(snapshotIncluded)); Thread.sleep(300); } @@ -140,7 +138,7 @@ public void shouldHandleMultipleSimultaneousRequests() { long now = Instant.now().toEpochMilli(); String id = "loop" + UUID.randomUUID(); SnapshotMetadata snapshotIncluded = - new SnapshotMetadata(id, id, now + 10, now + 15, 0, id, LOGS_LUCENE9, 0); + new SnapshotMetadata(id, now + 10, now + 15, 0, id, 0); try { replicaRestoreService.queueSnapshotsForRestoration(List.of(snapshotIncluded)); Thread.sleep(300); @@ -191,8 +189,7 @@ public void shouldRemoveDuplicates() throws Exception { List duplicateSnapshots = new ArrayList<>(); for (int i = 0; i < 10; i++) { String id = "duplicate"; - duplicateSnapshots.add( - new SnapshotMetadata(id, id, now + 10, now + 15, 0, id, LOGS_LUCENE9, 0)); + duplicateSnapshots.add(new SnapshotMetadata(id, now + 10, now + 15, 0, id, 0)); } replicaRestoreService.queueSnapshotsForRestoration(duplicateSnapshots); @@ -211,7 +208,7 @@ public void shouldRemoveDuplicates() throws Exception { for (int i = 0; i < 3; i++) { now = Instant.now().toEpochMilli(); String id = "loop" + i; - snapshots.add(new SnapshotMetadata(id, id, now + 10, now + 15, 0, id, LOGS_LUCENE9, 0)); + snapshots.add(new SnapshotMetadata(id, now + 10, now + 15, 0, id, 0)); } replicaRestoreService.queueSnapshotsForRestoration(snapshots); @@ -248,7 +245,7 @@ public void shouldNotQueueIfFull() { for (int i = 0; i < MAX_QUEUE_SIZE; i++) { long now = Instant.now().toEpochMilli(); String id = "loop" + i; - snapshots.add(new SnapshotMetadata(id, id, now + 10, now + 15, 0, id, LOGS_LUCENE9, 0)); + snapshots.add(new SnapshotMetadata(id, now + 10, now + 15, 0, id, 0)); } assertThatExceptionOfType(SizeLimitExceededException.class) diff --git a/astra/src/test/java/com/slack/astra/clusterManager/SnapshotDeletionServiceTest.java b/astra/src/test/java/com/slack/astra/clusterManager/SnapshotDeletionServiceTest.java index 0fc1d401b2..d97f661e9c 100644 --- a/astra/src/test/java/com/slack/astra/clusterManager/SnapshotDeletionServiceTest.java +++ b/astra/src/test/java/com/slack/astra/clusterManager/SnapshotDeletionServiceTest.java @@ -1,6 +1,5 @@ package com.slack.astra.clusterManager; -import static com.slack.astra.proto.metadata.Metadata.IndexType.LOGS_LUCENE9; import static com.slack.astra.server.AstraConfig.DEFAULT_START_STOP_DURATION; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.AssertionsForClassTypes.assertThatExceptionOfType; @@ -162,13 +161,11 @@ public void shouldDeleteExpiredSnapshotNoReplicas() throws Exception { SnapshotMetadata snapshotMetadata = new SnapshotMetadata( chunkId, - chunkStore.getRemotePath(chunkId), Instant.now().minus(11000, ChronoUnit.MINUTES).toEpochMilli(), Instant.now().minus(10900, ChronoUnit.MINUTES).toEpochMilli(), 0, "1", - LOGS_LUCENE9, - 0); + 100); snapshotMetadataStore.createAsync(snapshotMetadata); await().until(() -> snapshotMetadataStore.listSync().size() == 1); @@ -219,12 +216,10 @@ public void shouldNotDeleteExpiredSnapshotWithReplicas() throws Exception { SnapshotMetadata snapshotMetadata = new SnapshotMetadata( chunkId, - chunkStore.getRemotePath(chunkId), Instant.now().minus(11000, ChronoUnit.MINUTES).toEpochMilli(), Instant.now().minus(10900, ChronoUnit.MINUTES).toEpochMilli(), 0, "1", - LOGS_LUCENE9, 0); snapshotMetadataStore.createAsync(snapshotMetadata); @@ -235,8 +230,7 @@ public void shouldNotDeleteExpiredSnapshotWithReplicas() throws Exception { "rep1", Instant.now().minus(10900, ChronoUnit.MINUTES).toEpochMilli(), Instant.now().minus(500, ChronoUnit.MINUTES).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataStore.createAsync(replicaMetadata); await().until(() -> snapshotMetadataStore.listSync().size() == 1); @@ -334,12 +328,10 @@ public void shouldHandleNoReplicasUnexpiredSnapshots() throws Exception { SnapshotMetadata snapshotMetadata = new SnapshotMetadata( chunkId, - chunkStore.getRemotePath(chunkId), Instant.now().minus(11000, ChronoUnit.MINUTES).toEpochMilli(), Instant.now().minus(500, ChronoUnit.MINUTES).toEpochMilli(), 0, "1", - LOGS_LUCENE9, 0); snapshotMetadataStore.createAsync(snapshotMetadata); await().until(() -> snapshotMetadataStore.listSync().size() == 1); @@ -395,12 +387,10 @@ public void shouldHandleReplicasWithLongerLifespanThanSnapshots() throws Excepti SnapshotMetadata snapshotMetadata = new SnapshotMetadata( chunkId, - chunkStore.getRemotePath(chunkId), Instant.now().minus(11000, ChronoUnit.MINUTES).toEpochMilli(), Instant.now().minus(10900, ChronoUnit.MINUTES).toEpochMilli(), 0, "1", - LOGS_LUCENE9, 0); snapshotMetadataStore.createAsync(snapshotMetadata); @@ -412,8 +402,7 @@ public void shouldHandleReplicasWithLongerLifespanThanSnapshots() throws Excepti "rep1", Instant.now().minus(11000, ChronoUnit.MINUTES).toEpochMilli(), Instant.now().minus(10900, ChronoUnit.MINUTES).toEpochMilli(), - false, - LOGS_LUCENE9); + false); replicaMetadataStore.createAsync(replicaMetadata); await().until(() -> snapshotMetadataStore.listSync().size() == 1); @@ -471,13 +460,11 @@ public void shouldHandleExceptionalObjectStorageDelete() throws Exception { SnapshotMetadata snapshotMetadata = new SnapshotMetadata( chunkId, - chunkStore.getRemotePath(chunkId), Instant.now().minus(11000, ChronoUnit.MINUTES).toEpochMilli(), Instant.now().minus(10900, ChronoUnit.MINUTES).toEpochMilli(), 0, "1", - LOGS_LUCENE9, - 0); + 100); snapshotMetadataStore.createAsync(snapshotMetadata); await().until(() -> snapshotMetadataStore.listSync().size() == 1); @@ -530,13 +517,11 @@ public void shouldHandleFailedZkDelete() throws Exception { SnapshotMetadata snapshotMetadata = new SnapshotMetadata( chunkId, - chunkStore.getRemotePath(chunkId), Instant.now().minus(11000, ChronoUnit.MINUTES).toEpochMilli(), Instant.now().minus(10900, ChronoUnit.MINUTES).toEpochMilli(), 0, "1", - LOGS_LUCENE9, - 0); + 100); snapshotMetadataStore.createAsync(snapshotMetadata); await().until(() -> snapshotMetadataStore.listSync().size() == 1); @@ -595,13 +580,11 @@ public void shouldHandleFailedObjectDelete() throws Exception { SnapshotMetadata snapshotMetadata = new SnapshotMetadata( chunkId, - chunkStore.getRemotePath(chunkId), Instant.now().minus(11000, ChronoUnit.MINUTES).toEpochMilli(), Instant.now().minus(10900, ChronoUnit.MINUTES).toEpochMilli(), 0, "1", - LOGS_LUCENE9, - 0); + 100); snapshotMetadataStore.createAsync(snapshotMetadata); await().until(() -> snapshotMetadataStore.listSync().size() == 1); @@ -653,13 +636,11 @@ public void shouldRetryTimedOutZkDeleteNextRun() throws Exception { SnapshotMetadata snapshotMetadata = new SnapshotMetadata( chunkId, - chunkStore.getRemotePath(chunkId), Instant.now().minus(11000, ChronoUnit.MINUTES).toEpochMilli(), Instant.now().minus(10900, ChronoUnit.MINUTES).toEpochMilli(), 0, "1", - LOGS_LUCENE9, - 0); + 100); snapshotMetadataStore.createAsync(snapshotMetadata); await().until(() -> snapshotMetadataStore.listSync().size() == 1); @@ -750,13 +731,11 @@ public void shouldRetryFailedObjectStorageDeleteNextRun() throws Exception { SnapshotMetadata snapshotMetadata = new SnapshotMetadata( chunkId, - chunkStore.getRemotePath(chunkId), Instant.now().minus(11000, ChronoUnit.MINUTES).toEpochMilli(), Instant.now().minus(10900, ChronoUnit.MINUTES).toEpochMilli(), 0, "1", - LOGS_LUCENE9, - 0); + 100); snapshotMetadataStore.createAsync(snapshotMetadata); await().until(() -> snapshotMetadataStore.listSync().size() == 1); @@ -824,13 +803,11 @@ public void shouldHandleSnapshotDeleteLifecycle() throws Exception { SnapshotMetadata snapshotMetadata = new SnapshotMetadata( chunkId, - chunkStore.getRemotePath(chunkId), Instant.now().minus(11000, ChronoUnit.MINUTES).toEpochMilli(), Instant.now().minus(10900, ChronoUnit.MINUTES).toEpochMilli(), 0, "1", - LOGS_LUCENE9, - 0); + 100); snapshotMetadataStore.createAsync(snapshotMetadata); await().until(() -> snapshotMetadataStore.listSync().size() == 1); diff --git a/astra/src/test/java/com/slack/astra/logstore/search/AstraDistributedQueryServiceTest.java b/astra/src/test/java/com/slack/astra/logstore/search/AstraDistributedQueryServiceTest.java index 0831b04c16..9707cae7e8 100644 --- a/astra/src/test/java/com/slack/astra/logstore/search/AstraDistributedQueryServiceTest.java +++ b/astra/src/test/java/com/slack/astra/logstore/search/AstraDistributedQueryServiceTest.java @@ -6,7 +6,6 @@ import static com.slack.astra.logstore.search.AstraDistributedQueryService.getMatchingSearchMetadata; import static com.slack.astra.logstore.search.AstraDistributedQueryService.getMatchingSnapshots; import static com.slack.astra.logstore.search.AstraDistributedQueryService.getNodesAndSnapshotsToQuery; -import static com.slack.astra.metadata.snapshot.SnapshotMetadata.LIVE_SNAPSHOT_PATH; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.fail; import static org.awaitility.Awaitility.await; @@ -31,7 +30,6 @@ import com.slack.astra.metadata.snapshot.SnapshotMetadata; import com.slack.astra.metadata.snapshot.SnapshotMetadataStore; import com.slack.astra.proto.config.AstraConfigs; -import com.slack.astra.proto.metadata.Metadata; import com.slack.astra.proto.schema.Schema; import com.slack.astra.proto.service.AstraSearch; import com.slack.astra.proto.service.AstraServiceGrpc; @@ -835,12 +833,10 @@ public void testSchema() { List.of( new SnapshotMetadata( "snapshot1", - "/1", endTime.minus(30, ChronoUnit.MINUTES).toEpochMilli(), endTime.toEpochMilli(), 10, "1", - Metadata.IndexType.LOGS_LUCENE9, 0))); DatasetMetadataStore datasetMetadataStoreMock = mock(DatasetMetadataStore.class); when(datasetMetadataStoreMock.listSync()) @@ -969,7 +965,6 @@ private SnapshotMetadata createSnapshot( chunkEndTime.toEpochMilli(), 1234, partition, - isLive ? LIVE_SNAPSHOT_PATH : "cacheSnapshotPath", 0); SnapshotMetadata snapshotMetadata = toSnapshotMetadata(chunkInfo, isLive ? LIVE_SNAPSHOT_PREFIX : ""); diff --git a/astra/src/test/java/com/slack/astra/metadata/cache/CacheSlotMetadataSerializerTest.java b/astra/src/test/java/com/slack/astra/metadata/cache/CacheSlotMetadataSerializerTest.java index e7269f585b..6a43364f5e 100644 --- a/astra/src/test/java/com/slack/astra/metadata/cache/CacheSlotMetadataSerializerTest.java +++ b/astra/src/test/java/com/slack/astra/metadata/cache/CacheSlotMetadataSerializerTest.java @@ -1,13 +1,11 @@ package com.slack.astra.metadata.cache; -import static com.slack.astra.proto.metadata.Metadata.IndexType.LOGS_LUCENE9; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.catchThrowable; import com.google.protobuf.InvalidProtocolBufferException; import com.slack.astra.proto.metadata.Metadata; import java.time.Instant; -import java.util.List; import org.junit.jupiter.api.Test; public class CacheSlotMetadataSerializerTest { @@ -22,17 +20,10 @@ public void testCacheSlotMetadataSerializer() throws InvalidProtocolBufferExcept Metadata.CacheSlotMetadata.CacheSlotState.ASSIGNED; String replicaId = "123"; long updatedTimeEpochMs = Instant.now().toEpochMilli(); - List supportedIndexTypes = List.of(LOGS_LUCENE9, LOGS_LUCENE9); CacheSlotMetadata cacheSlotMetadata = new CacheSlotMetadata( - name, - cacheSlotState, - replicaId, - updatedTimeEpochMs, - supportedIndexTypes, - hostname, - replicaSet); + name, cacheSlotState, replicaId, updatedTimeEpochMs, hostname, replicaSet); String serializedCacheSlotMetadata = serDe.toJsonStr(cacheSlotMetadata); assertThat(serializedCacheSlotMetadata).isNotEmpty(); @@ -47,8 +38,6 @@ public void testCacheSlotMetadataSerializer() throws InvalidProtocolBufferExcept assertThat(deserializedCacheSlotMetadata.cacheSlotState).isEqualTo(cacheSlotState); assertThat(deserializedCacheSlotMetadata.replicaId).isEqualTo(replicaId); assertThat(deserializedCacheSlotMetadata.updatedTimeEpochMs).isEqualTo(updatedTimeEpochMs); - assertThat(deserializedCacheSlotMetadata.supportedIndexTypes) - .containsExactlyInAnyOrderElementsOf(supportedIndexTypes); } @Test diff --git a/astra/src/test/java/com/slack/astra/metadata/cache/CacheSlotMetadataStoreTest.java b/astra/src/test/java/com/slack/astra/metadata/cache/CacheSlotMetadataStoreTest.java index dbc0051f73..dc64dd5b0c 100644 --- a/astra/src/test/java/com/slack/astra/metadata/cache/CacheSlotMetadataStoreTest.java +++ b/astra/src/test/java/com/slack/astra/metadata/cache/CacheSlotMetadataStoreTest.java @@ -1,7 +1,6 @@ package com.slack.astra.metadata.cache; import static com.slack.astra.proto.metadata.Metadata.CacheSlotMetadata.CacheSlotState; -import static com.slack.astra.proto.metadata.Metadata.IndexType.LOGS_LUCENE9; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; import static org.awaitility.Awaitility.await; @@ -14,7 +13,6 @@ import io.micrometer.core.instrument.simple.SimpleMeterRegistry; import java.io.IOException; import java.time.Instant; -import java.util.List; import java.util.concurrent.TimeUnit; import org.apache.curator.test.TestingServer; import org.apache.curator.x.async.AsyncCuratorFramework; @@ -23,8 +21,6 @@ import org.junit.jupiter.api.Test; public class CacheSlotMetadataStoreTest { - private static final List SUPPORTED_INDEX_TYPES = List.of(LOGS_LUCENE9); - private TestingServer testingServer; private AsyncCuratorFramework curatorFramework; @@ -68,18 +64,11 @@ public void testUpdateNonFreeCacheSlotStateSync() throws Exception { final CacheSlotMetadata cacheSlotMetadata = new CacheSlotMetadata( - name, - cacheSlotState, - replicaId, - updatedTimeEpochMs, - SUPPORTED_INDEX_TYPES, - hostname, - replicaSet); + name, cacheSlotState, replicaId, updatedTimeEpochMs, hostname, replicaSet); assertThat(cacheSlotMetadata.name).isEqualTo(name); assertThat(cacheSlotMetadata.cacheSlotState).isEqualTo(cacheSlotState); assertThat(cacheSlotMetadata.replicaId).isEqualTo(replicaId); assertThat(cacheSlotMetadata.updatedTimeEpochMs).isEqualTo(updatedTimeEpochMs); - assertThat(cacheSlotMetadata.supportedIndexTypes).isEqualTo(SUPPORTED_INDEX_TYPES); assertThat(cacheSlotMetadata.hostname).isEqualTo(hostname); store.createSync(cacheSlotMetadata); @@ -98,7 +87,6 @@ public void testUpdateNonFreeCacheSlotStateSync() throws Exception { assertThat(liveNode.cacheSlotState).isEqualTo(CacheSlotState.LIVE); assertThat(liveNode.replicaId).isEqualTo(replicaId); assertThat(liveNode.updatedTimeEpochMs).isGreaterThan(cacheSlotMetadata.updatedTimeEpochMs); - assertThat(liveNode.supportedIndexTypes).isEqualTo(SUPPORTED_INDEX_TYPES); store .updateNonFreeCacheSlotState(cacheSlotMetadata, CacheSlotState.EVICT) @@ -113,7 +101,6 @@ public void testUpdateNonFreeCacheSlotStateSync() throws Exception { assertThat(evictNode.cacheSlotState).isEqualTo(CacheSlotState.EVICT); assertThat(evictNode.replicaId).isEqualTo(replicaId); assertThat(evictNode.updatedTimeEpochMs).isGreaterThan(liveNode.updatedTimeEpochMs); - assertThat(evictNode.supportedIndexTypes).isEqualTo(SUPPORTED_INDEX_TYPES); store .updateNonFreeCacheSlotState(cacheSlotMetadata, CacheSlotState.FREE) @@ -128,7 +115,6 @@ public void testUpdateNonFreeCacheSlotStateSync() throws Exception { assertThat(freeNode.cacheSlotState).isEqualTo(CacheSlotState.FREE); assertThat(freeNode.replicaId).isEmpty(); assertThat(freeNode.updatedTimeEpochMs).isGreaterThan(evictNode.updatedTimeEpochMs); - assertThat(freeNode.supportedIndexTypes).isEqualTo(SUPPORTED_INDEX_TYPES); // Only non-free states can be set. assertThatIllegalArgumentException() @@ -150,18 +136,11 @@ public void testNonFreeCacheSlotState() throws Exception { final CacheSlotMetadata cacheSlotMetadata = new CacheSlotMetadata( - name, - cacheSlotState, - replicaId, - updatedTimeEpochMs, - SUPPORTED_INDEX_TYPES, - hostname, - replicaSet); + name, cacheSlotState, replicaId, updatedTimeEpochMs, hostname, replicaSet); assertThat(cacheSlotMetadata.name).isEqualTo(name); assertThat(cacheSlotMetadata.cacheSlotState).isEqualTo(cacheSlotState); assertThat(cacheSlotMetadata.replicaId).isEqualTo(replicaId); assertThat(cacheSlotMetadata.updatedTimeEpochMs).isEqualTo(updatedTimeEpochMs); - assertThat(cacheSlotMetadata.supportedIndexTypes).isEqualTo(SUPPORTED_INDEX_TYPES); assertThat(cacheSlotMetadata.hostname).isEqualTo(hostname); store.createSync(cacheSlotMetadata); @@ -180,7 +159,6 @@ public void testNonFreeCacheSlotState() throws Exception { assertThat(liveNode.cacheSlotState).isEqualTo(CacheSlotState.LIVE); assertThat(liveNode.replicaId).isEqualTo(replicaId); assertThat(liveNode.updatedTimeEpochMs).isGreaterThan(cacheSlotMetadata.updatedTimeEpochMs); - assertThat(liveNode.supportedIndexTypes).isEqualTo(SUPPORTED_INDEX_TYPES); store.updateNonFreeCacheSlotState(liveNode, CacheSlotState.EVICT).get(1, TimeUnit.SECONDS); await() @@ -193,7 +171,6 @@ public void testNonFreeCacheSlotState() throws Exception { assertThat(evictNode.cacheSlotState).isEqualTo(CacheSlotState.EVICT); assertThat(evictNode.replicaId).isEqualTo(replicaId); assertThat(evictNode.updatedTimeEpochMs).isGreaterThan(liveNode.updatedTimeEpochMs); - assertThat(evictNode.supportedIndexTypes).isEqualTo(SUPPORTED_INDEX_TYPES); store.updateNonFreeCacheSlotState(evictNode, CacheSlotState.FREE).get(1, TimeUnit.SECONDS); await() @@ -206,7 +183,6 @@ public void testNonFreeCacheSlotState() throws Exception { assertThat(freeNode.cacheSlotState).isEqualTo(CacheSlotState.FREE); assertThat(freeNode.replicaId).isEmpty(); assertThat(freeNode.updatedTimeEpochMs).isGreaterThan(evictNode.updatedTimeEpochMs); - assertThat(freeNode.supportedIndexTypes).isEqualTo(SUPPORTED_INDEX_TYPES); // Only non-free states can be set. assertThatIllegalArgumentException() @@ -229,18 +205,11 @@ public void testCacheSlotStateWithReplica() throws Exception { final CacheSlotMetadata cacheSlotMetadata = new CacheSlotMetadata( - name, - cacheSlotState, - emptyReplicaId, - updatedTimeEpochMs, - SUPPORTED_INDEX_TYPES, - hostname, - replicaSet); + name, cacheSlotState, emptyReplicaId, updatedTimeEpochMs, hostname, replicaSet); assertThat(cacheSlotMetadata.name).isEqualTo(name); assertThat(cacheSlotMetadata.cacheSlotState).isEqualTo(cacheSlotState); assertThat(cacheSlotMetadata.replicaId).isEqualTo(emptyReplicaId); assertThat(cacheSlotMetadata.updatedTimeEpochMs).isEqualTo(updatedTimeEpochMs); - assertThat(cacheSlotMetadata.supportedIndexTypes).isEqualTo(SUPPORTED_INDEX_TYPES); assertThat(cacheSlotMetadata.hostname).isEqualTo(hostname); store.createSync(cacheSlotMetadata); @@ -261,7 +230,6 @@ public void testCacheSlotStateWithReplica() throws Exception { assertThat(freeNode.cacheSlotState).isEqualTo(cacheSlotState); assertThat(freeNode.replicaId).isEqualTo(emptyReplicaId); assertThat(freeNode.updatedTimeEpochMs).isGreaterThanOrEqualTo(updatedTimeEpochMs); - assertThat(freeNode.supportedIndexTypes).isEqualTo(SUPPORTED_INDEX_TYPES); final String replicaId = "1234"; store @@ -281,7 +249,6 @@ public void testCacheSlotStateWithReplica() throws Exception { .isEqualTo(Metadata.CacheSlotMetadata.CacheSlotState.ASSIGNED); assertThat(assignedNode.replicaId).isEqualTo(replicaId); assertThat(assignedNode.updatedTimeEpochMs).isGreaterThan(freeNode.updatedTimeEpochMs); - assertThat(assignedNode.supportedIndexTypes).isEqualTo(SUPPORTED_INDEX_TYPES); store .updateCacheSlotStateStateWithReplicaId( @@ -300,6 +267,5 @@ public void testCacheSlotStateWithReplica() throws Exception { .isEqualTo(Metadata.CacheSlotMetadata.CacheSlotState.EVICT); assertThat(evictedNode.replicaId).isEqualTo(replicaId); assertThat(evictedNode.updatedTimeEpochMs).isGreaterThan(freeNode.updatedTimeEpochMs); - assertThat(evictedNode.supportedIndexTypes).isEqualTo(SUPPORTED_INDEX_TYPES); } } diff --git a/astra/src/test/java/com/slack/astra/metadata/cache/CacheSlotMetadataTest.java b/astra/src/test/java/com/slack/astra/metadata/cache/CacheSlotMetadataTest.java index 0fab4427e4..837a491df5 100644 --- a/astra/src/test/java/com/slack/astra/metadata/cache/CacheSlotMetadataTest.java +++ b/astra/src/test/java/com/slack/astra/metadata/cache/CacheSlotMetadataTest.java @@ -1,19 +1,14 @@ package com.slack.astra.metadata.cache; -import static com.slack.astra.proto.metadata.Metadata.IndexType.LOGS_LUCENE9; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; import com.slack.astra.proto.metadata.Metadata; import java.time.Instant; -import java.util.Collections; -import java.util.List; import org.junit.jupiter.api.Test; public class CacheSlotMetadataTest { - private static final List SUPPORTED_INDEX_TYPES = List.of(LOGS_LUCENE9); - @Test public void testCacheSlotMetadata() { String hostname = "hostname"; @@ -26,20 +21,13 @@ public void testCacheSlotMetadata() { CacheSlotMetadata cacheSlotMetadata = new CacheSlotMetadata( - name, - cacheSlotState, - replicaId, - updatedTimeEpochMs, - SUPPORTED_INDEX_TYPES, - hostname, - replicaSet); + name, cacheSlotState, replicaId, updatedTimeEpochMs, hostname, replicaSet); assertThat(cacheSlotMetadata.name).isEqualTo(name); assertThat(cacheSlotMetadata.hostname).isEqualTo(hostname); assertThat(cacheSlotMetadata.replicaSet).isEqualTo(replicaSet); assertThat(cacheSlotMetadata.cacheSlotState).isEqualTo(cacheSlotState); assertThat(cacheSlotMetadata.replicaId).isEqualTo(replicaId); assertThat(cacheSlotMetadata.updatedTimeEpochMs).isEqualTo(updatedTimeEpochMs); - assertThat(cacheSlotMetadata.supportedIndexTypes).isEqualTo(SUPPORTED_INDEX_TYPES); } @Test @@ -54,77 +42,34 @@ public void testCacheSlotEqualsHashcode() { CacheSlotMetadata cacheSlot = new CacheSlotMetadata( - name, - cacheSlotState, - replicaId, - updatedTimeEpochMs, - SUPPORTED_INDEX_TYPES, - hostname, - replicaSet); + name, cacheSlotState, replicaId, updatedTimeEpochMs, hostname, replicaSet); CacheSlotMetadata cacheSlotDuplicate = new CacheSlotMetadata( - name, - cacheSlotState, - replicaId, - updatedTimeEpochMs, - SUPPORTED_INDEX_TYPES, - hostname, - replicaSet); + name, cacheSlotState, replicaId, updatedTimeEpochMs, hostname, replicaSet); CacheSlotMetadata cacheSlotDifferentState = new CacheSlotMetadata( name, Metadata.CacheSlotMetadata.CacheSlotState.EVICT, replicaId, updatedTimeEpochMs, - SUPPORTED_INDEX_TYPES, hostname, replicaSet); CacheSlotMetadata cacheSlotDifferentReplicaId = new CacheSlotMetadata( - name, - cacheSlotState, - "321", - updatedTimeEpochMs, - SUPPORTED_INDEX_TYPES, - hostname, - replicaSet); + name, cacheSlotState, "321", updatedTimeEpochMs, hostname, replicaSet); CacheSlotMetadata cacheSlotDifferentUpdatedTime = new CacheSlotMetadata( - name, - cacheSlotState, - replicaId, - updatedTimeEpochMs + 1, - SUPPORTED_INDEX_TYPES, - hostname, - replicaSet); + name, cacheSlotState, replicaId, updatedTimeEpochMs + 1, hostname, replicaSet); CacheSlotMetadata cacheSlotDifferentSupportedIndexType = new CacheSlotMetadata( - name, - cacheSlotState, - replicaId, - updatedTimeEpochMs + 1, - List.of(LOGS_LUCENE9, LOGS_LUCENE9), - hostname, - replicaSet); + name, cacheSlotState, replicaId, updatedTimeEpochMs + 1, hostname, replicaSet); CacheSlotMetadata cacheSlotDifferentHostname = new CacheSlotMetadata( - name, - cacheSlotState, - replicaId, - updatedTimeEpochMs, - SUPPORTED_INDEX_TYPES, - "hostname2", - replicaSet); + name, cacheSlotState, replicaId, updatedTimeEpochMs, "hostname2", replicaSet); CacheSlotMetadata cacheSlotDifferentReplicaPartition = new CacheSlotMetadata( - name, - cacheSlotState, - replicaId, - updatedTimeEpochMs, - SUPPORTED_INDEX_TYPES, - hostname, - "rep2"); + name, cacheSlotState, replicaId, updatedTimeEpochMs, hostname, "rep2"); assertThat(cacheSlot.hashCode()).isEqualTo(cacheSlotDuplicate.hashCode()); assertThat(cacheSlot).isEqualTo(cacheSlotDuplicate); @@ -153,7 +98,6 @@ public void invalidArgumentsShouldThrow() { Metadata.CacheSlotMetadata.CacheSlotState.FREE, "123", Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, "hostname", "rep1")); assertThatIllegalArgumentException() @@ -164,7 +108,6 @@ public void invalidArgumentsShouldThrow() { Metadata.CacheSlotMetadata.CacheSlotState.ASSIGNED, "", Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, "hostname", "rep1")); assertThatIllegalArgumentException() @@ -175,7 +118,6 @@ public void invalidArgumentsShouldThrow() { Metadata.CacheSlotMetadata.CacheSlotState.EVICT, "", Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, "hostname", "rep1")); assertThatIllegalArgumentException() @@ -186,7 +128,6 @@ public void invalidArgumentsShouldThrow() { Metadata.CacheSlotMetadata.CacheSlotState.EVICTING, "", Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, "hostname", "rep1")); assertThatIllegalArgumentException() @@ -197,7 +138,6 @@ public void invalidArgumentsShouldThrow() { Metadata.CacheSlotMetadata.CacheSlotState.LOADING, "", Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, "hostname", "rep1")); assertThatIllegalArgumentException() @@ -208,7 +148,6 @@ public void invalidArgumentsShouldThrow() { Metadata.CacheSlotMetadata.CacheSlotState.LIVE, "", Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, "hostname", "rep1")); assertThatIllegalArgumentException() @@ -219,7 +158,6 @@ public void invalidArgumentsShouldThrow() { Metadata.CacheSlotMetadata.CacheSlotState.FREE, "", 0, - SUPPORTED_INDEX_TYPES, "hostname", "rep1")); assertThatIllegalArgumentException() @@ -230,7 +168,6 @@ public void invalidArgumentsShouldThrow() { Metadata.CacheSlotMetadata.CacheSlotState.FREE, null, Instant.now().toEpochMilli(), - SUPPORTED_INDEX_TYPES, "hostname", "rep1")); assertThatIllegalArgumentException() @@ -241,7 +178,6 @@ public void invalidArgumentsShouldThrow() { Metadata.CacheSlotMetadata.CacheSlotState.FREE, "123", 100000, - Collections.emptyList(), "hostname", "rep1"); }); @@ -253,7 +189,6 @@ public void invalidArgumentsShouldThrow() { Metadata.CacheSlotMetadata.CacheSlotState.FREE, "123", 100000, - SUPPORTED_INDEX_TYPES, "hostname", "rep1"); }); diff --git a/astra/src/test/java/com/slack/astra/metadata/replica/ReplicaMetadataSerializerTest.java b/astra/src/test/java/com/slack/astra/metadata/replica/ReplicaMetadataSerializerTest.java index 3b4db81d7a..16e02e1b08 100644 --- a/astra/src/test/java/com/slack/astra/metadata/replica/ReplicaMetadataSerializerTest.java +++ b/astra/src/test/java/com/slack/astra/metadata/replica/ReplicaMetadataSerializerTest.java @@ -1,6 +1,5 @@ package com.slack.astra.metadata.replica; -import static com.slack.astra.proto.metadata.Metadata.IndexType.LOGS_LUCENE9; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.catchThrowable; @@ -21,13 +20,7 @@ public void testReplicaMetadataSerializer() throws InvalidProtocolBufferExceptio ReplicaMetadata replicaMetadata = new ReplicaMetadata( - name, - snapshotId, - replicaSet, - createdTimeEpochMs, - expireAfterEpochMs, - true, - LOGS_LUCENE9); + name, snapshotId, replicaSet, createdTimeEpochMs, expireAfterEpochMs, true); String serializedReplicaMetadata = serDe.toJsonStr(replicaMetadata); assertThat(serializedReplicaMetadata).isNotEmpty(); @@ -41,7 +34,6 @@ public void testReplicaMetadataSerializer() throws InvalidProtocolBufferExceptio assertThat(deserializedReplicaMetadata.createdTimeEpochMs).isEqualTo(createdTimeEpochMs); assertThat(deserializedReplicaMetadata.expireAfterEpochMs).isEqualTo(expireAfterEpochMs); assertThat(deserializedReplicaMetadata.isRestored).isTrue(); - assertThat(deserializedReplicaMetadata.indexType).isEqualTo(LOGS_LUCENE9); } @Test @@ -63,7 +55,6 @@ public void shouldHandleEmptyExpirationAndRestore() throws InvalidProtocolBuffer assertThat(deserializedReplicaMetadata.createdTimeEpochMs).isEqualTo(1639677020380L); assertThat(deserializedReplicaMetadata.expireAfterEpochMs).isEqualTo(0L); assertThat(deserializedReplicaMetadata.isRestored).isFalse(); - assertThat(deserializedReplicaMetadata.indexType).isEqualTo(LOGS_LUCENE9); } @Test diff --git a/astra/src/test/java/com/slack/astra/metadata/replica/ReplicaMetadataTest.java b/astra/src/test/java/com/slack/astra/metadata/replica/ReplicaMetadataTest.java index 602e46926e..090637e0d5 100644 --- a/astra/src/test/java/com/slack/astra/metadata/replica/ReplicaMetadataTest.java +++ b/astra/src/test/java/com/slack/astra/metadata/replica/ReplicaMetadataTest.java @@ -1,6 +1,5 @@ package com.slack.astra.metadata.replica; -import static com.slack.astra.proto.metadata.Metadata.IndexType.LOGS_LUCENE9; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; @@ -19,13 +18,7 @@ public void testReplicaMetadata() { ReplicaMetadata replicaMetadata = new ReplicaMetadata( - name, - snapshotId, - replicaSet, - createdTimeEpochMs, - expireAfterEpochMs, - false, - LOGS_LUCENE9); + name, snapshotId, replicaSet, createdTimeEpochMs, expireAfterEpochMs, false); assertThat(replicaMetadata.name).isEqualTo(name); assertThat(replicaMetadata.snapshotId).isEqualTo(snapshotId); @@ -33,17 +26,10 @@ public void testReplicaMetadata() { assertThat(replicaMetadata.createdTimeEpochMs).isEqualTo(createdTimeEpochMs); assertThat(replicaMetadata.expireAfterEpochMs).isEqualTo(expireAfterEpochMs); assertThat(replicaMetadata.isRestored).isFalse(); - assertThat(replicaMetadata.indexType).isEqualTo(LOGS_LUCENE9); ReplicaMetadata restoredReplicaMetadata = new ReplicaMetadata( - name, - snapshotId, - replicaSet, - createdTimeEpochMs, - expireAfterEpochMs, - true, - LOGS_LUCENE9); + name, snapshotId, replicaSet, createdTimeEpochMs, expireAfterEpochMs, true); assertThat(restoredReplicaMetadata.name).isEqualTo(name); assertThat(restoredReplicaMetadata.snapshotId).isEqualTo(snapshotId); @@ -51,7 +37,6 @@ public void testReplicaMetadata() { assertThat(restoredReplicaMetadata.createdTimeEpochMs).isEqualTo(createdTimeEpochMs); assertThat(restoredReplicaMetadata.expireAfterEpochMs).isEqualTo(expireAfterEpochMs); assertThat(restoredReplicaMetadata.isRestored).isTrue(); - assertThat(restoredReplicaMetadata.indexType).isEqualTo(LOGS_LUCENE9); } @Test @@ -64,52 +49,21 @@ public void testReplicaMetadataEqualsHashcode() { ReplicaMetadata replicaMetadataA = new ReplicaMetadata( - name, - snapshotId, - replicaSet, - createdTimeEpochMs, - expireAfterEpochMs, - true, - LOGS_LUCENE9); + name, snapshotId, replicaSet, createdTimeEpochMs, expireAfterEpochMs, true); ReplicaMetadata replicaMetadataB = new ReplicaMetadata( - name, - snapshotId, - replicaSet, - createdTimeEpochMs, - expireAfterEpochMs, - true, - LOGS_LUCENE9); + name, snapshotId, replicaSet, createdTimeEpochMs, expireAfterEpochMs, true); ReplicaMetadata replicaMetadataC = new ReplicaMetadata( - "nameC", - snapshotId, - replicaSet, - createdTimeEpochMs, - expireAfterEpochMs, - true, - LOGS_LUCENE9); + "nameC", snapshotId, replicaSet, createdTimeEpochMs, expireAfterEpochMs, true); ReplicaMetadata replicaMetadataD = new ReplicaMetadata( - name, - snapshotId, - replicaSet, - createdTimeEpochMs + 1, - expireAfterEpochMs, - false, - LOGS_LUCENE9); + name, snapshotId, replicaSet, createdTimeEpochMs + 1, expireAfterEpochMs, false); ReplicaMetadata replicaMetadataE = new ReplicaMetadata( - name, - snapshotId, - replicaSet, - createdTimeEpochMs, - expireAfterEpochMs + 1, - false, - LOGS_LUCENE9); + name, snapshotId, replicaSet, createdTimeEpochMs, expireAfterEpochMs + 1, false); ReplicaMetadata replicaMetadataF = - new ReplicaMetadata( - name, snapshotId, "rep2", createdTimeEpochMs, expireAfterEpochMs, true, LOGS_LUCENE9); + new ReplicaMetadata(name, snapshotId, "rep2", createdTimeEpochMs, expireAfterEpochMs, true); assertThat(replicaMetadataA).isEqualTo(replicaMetadataB); assertThat(replicaMetadataA).isNotEqualTo(replicaMetadataC); @@ -135,8 +89,7 @@ public void invalidArgumentsShouldThrow() { "rep1", Instant.now().toEpochMilli(), Instant.now().toEpochMilli(), - false, - LOGS_LUCENE9)); + false)); assertThatIllegalArgumentException() .isThrownBy( () -> @@ -146,17 +99,14 @@ public void invalidArgumentsShouldThrow() { "rep1", Instant.now().toEpochMilli(), Instant.now().toEpochMilli(), - true, - LOGS_LUCENE9)); + true)); assertThatIllegalArgumentException() .isThrownBy( () -> - new ReplicaMetadata( - "name", "123", "rep1", 0, Instant.now().toEpochMilli(), false, LOGS_LUCENE9)); + new ReplicaMetadata("name", "123", "rep1", 0, Instant.now().toEpochMilli(), false)); assertThatIllegalArgumentException() .isThrownBy( () -> - new ReplicaMetadata( - "name", "123", "rep1", Instant.now().toEpochMilli(), -1, true, LOGS_LUCENE9)); + new ReplicaMetadata("name", "123", "rep1", Instant.now().toEpochMilli(), -1, true)); } } diff --git a/astra/src/test/java/com/slack/astra/metadata/snapshot/SnapshotMetadataSerializerTest.java b/astra/src/test/java/com/slack/astra/metadata/snapshot/SnapshotMetadataSerializerTest.java index 2d8987e124..ac0ee8ad7c 100644 --- a/astra/src/test/java/com/slack/astra/metadata/snapshot/SnapshotMetadataSerializerTest.java +++ b/astra/src/test/java/com/slack/astra/metadata/snapshot/SnapshotMetadataSerializerTest.java @@ -1,6 +1,5 @@ package com.slack.astra.metadata.snapshot; -import static com.slack.astra.proto.metadata.Metadata.IndexType.LOGS_LUCENE9; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.AssertionsForClassTypes.assertThatExceptionOfType; @@ -14,7 +13,6 @@ public class SnapshotMetadataSerializerTest { @Test public void testSnapshotMetadataSerializer() throws InvalidProtocolBufferException { final String name = "testSnapshotId"; - final String path = "/testPath_" + name; final long startTime = 1; final long endTime = 100; final long maxOffset = 123; @@ -22,8 +20,7 @@ public void testSnapshotMetadataSerializer() throws InvalidProtocolBufferExcepti final long sizeInBytes = 0; SnapshotMetadata snapshotMetadata = - new SnapshotMetadata( - name, path, startTime, endTime, maxOffset, partitionId, LOGS_LUCENE9, sizeInBytes); + new SnapshotMetadata(name, startTime, endTime, maxOffset, partitionId, sizeInBytes); String serializedSnapshot = serDe.toJsonStr(snapshotMetadata); assertThat(serializedSnapshot).isNotEmpty(); @@ -32,20 +29,17 @@ public void testSnapshotMetadataSerializer() throws InvalidProtocolBufferExcepti assertThat(deserializedSnapshotMetadata).isEqualTo(snapshotMetadata); assertThat(deserializedSnapshotMetadata.name).isEqualTo(name); - assertThat(deserializedSnapshotMetadata.snapshotPath).isEqualTo(path); assertThat(deserializedSnapshotMetadata.snapshotId).isEqualTo(name); assertThat(deserializedSnapshotMetadata.startTimeEpochMs).isEqualTo(startTime); assertThat(deserializedSnapshotMetadata.endTimeEpochMs).isEqualTo(endTime); assertThat(deserializedSnapshotMetadata.maxOffset).isEqualTo(maxOffset); assertThat(deserializedSnapshotMetadata.partitionId).isEqualTo(partitionId); - assertThat(deserializedSnapshotMetadata.indexType).isEqualTo(LOGS_LUCENE9); assertThat(deserializedSnapshotMetadata.sizeInBytesOnDisk).isEqualTo(sizeInBytes); } @Test public void testDeserializingWithoutSizeField() throws InvalidProtocolBufferException { final String name = "testSnapshotId"; - final String path = "/testPath_" + name; final long startTime = 1; final long endTime = 100; final long maxOffset = 123; @@ -54,13 +48,11 @@ public void testDeserializingWithoutSizeField() throws InvalidProtocolBufferExce Metadata.SnapshotMetadata protoSnapshotMetadata = Metadata.SnapshotMetadata.newBuilder() .setName(name) - .setSnapshotPath(path) .setSnapshotId(name) .setStartTimeEpochMs(startTime) .setEndTimeEpochMs(endTime) .setMaxOffset(maxOffset) .setPartitionId(partitionId) - .setIndexType(LOGS_LUCENE9) // leaving out the `size` field .build(); @@ -72,13 +64,11 @@ public void testDeserializingWithoutSizeField() throws InvalidProtocolBufferExce // Assert everything else is deserialized correctly assertThat(deserializedSnapshotMetadata.name).isEqualTo(name); - assertThat(deserializedSnapshotMetadata.snapshotPath).isEqualTo(path); assertThat(deserializedSnapshotMetadata.snapshotId).isEqualTo(name); assertThat(deserializedSnapshotMetadata.startTimeEpochMs).isEqualTo(startTime); assertThat(deserializedSnapshotMetadata.endTimeEpochMs).isEqualTo(endTime); assertThat(deserializedSnapshotMetadata.maxOffset).isEqualTo(maxOffset); assertThat(deserializedSnapshotMetadata.partitionId).isEqualTo(partitionId); - assertThat(deserializedSnapshotMetadata.indexType).isEqualTo(LOGS_LUCENE9); } @Test diff --git a/astra/src/test/java/com/slack/astra/metadata/snapshot/SnapshotMetadataTest.java b/astra/src/test/java/com/slack/astra/metadata/snapshot/SnapshotMetadataTest.java index 585a2f09ad..66ae0ca697 100644 --- a/astra/src/test/java/com/slack/astra/metadata/snapshot/SnapshotMetadataTest.java +++ b/astra/src/test/java/com/slack/astra/metadata/snapshot/SnapshotMetadataTest.java @@ -1,6 +1,5 @@ package com.slack.astra.metadata.snapshot; -import static com.slack.astra.proto.metadata.Metadata.IndexType.LOGS_LUCENE9; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; @@ -13,41 +12,34 @@ public class SnapshotMetadataTest { @Test public void testSnapshotMetadata() { final String name = "testSnapshotId"; - final String path = "/testPath_" + name; final long startTime = 1; final long endTime = 100; final long maxOffset = 123; final String partitionId = "1"; SnapshotMetadata snapshotMetadata = - new SnapshotMetadata( - name, path, startTime, endTime, maxOffset, partitionId, LOGS_LUCENE9, 0); + new SnapshotMetadata(name, startTime, endTime, maxOffset, partitionId, 0); assertThat(snapshotMetadata.name).isEqualTo(name); - assertThat(snapshotMetadata.snapshotPath).isEqualTo(path); assertThat(snapshotMetadata.snapshotId).isEqualTo(name); assertThat(snapshotMetadata.startTimeEpochMs).isEqualTo(startTime); assertThat(snapshotMetadata.endTimeEpochMs).isEqualTo(endTime); assertThat(snapshotMetadata.maxOffset).isEqualTo(maxOffset); assertThat(snapshotMetadata.partitionId).isEqualTo(partitionId); - assertThat(snapshotMetadata.indexType).isEqualTo(LOGS_LUCENE9); } @Test public void testEqualsAndHashCode() { final String name = "testSnapshotId"; - final String path = "/testPath_" + name; final long startTime = 1; final long endTime = 100; final long maxOffset = 0; final String partitionId = "1"; SnapshotMetadata snapshot1 = - new SnapshotMetadata( - name, path, startTime, endTime, maxOffset, partitionId, LOGS_LUCENE9, 0); + new SnapshotMetadata(name, startTime, endTime, maxOffset, partitionId, 0); SnapshotMetadata snapshot2 = - new SnapshotMetadata( - name + "2", path, startTime, endTime, maxOffset, partitionId, LOGS_LUCENE9, 0); + new SnapshotMetadata(name + "2", startTime, endTime, maxOffset, partitionId, 0); // Ensure the name field from super class is included. assertThat(snapshot1).isNotEqualTo(snapshot2); @@ -61,87 +53,52 @@ public void testEqualsAndHashCode() { @Test public void ensureValidSnapshotData() { final String name = "testSnapshotId"; - final String path = "/testPath_" + name; final long startTime = 1; final long endTime = 100; final long maxOffset = 123; final String partitionId = "1"; assertThatIllegalArgumentException() - .isThrownBy( - () -> - new SnapshotMetadata( - "", path, startTime, endTime, maxOffset, partitionId, LOGS_LUCENE9, 0)); - - assertThatIllegalArgumentException() - .isThrownBy( - () -> - new SnapshotMetadata( - name, "", startTime, endTime, maxOffset, partitionId, LOGS_LUCENE9, 0)); + .isThrownBy(() -> new SnapshotMetadata("", startTime, endTime, maxOffset, partitionId, 0)); assertThatIllegalArgumentException() - .isThrownBy( - () -> - new SnapshotMetadata( - name, path, 0, endTime, maxOffset, partitionId, LOGS_LUCENE9, 0)); + .isThrownBy(() -> new SnapshotMetadata(name, 0, endTime, maxOffset, partitionId, 0)); assertThatIllegalArgumentException() - .isThrownBy( - () -> - new SnapshotMetadata( - name, path, startTime, 0, maxOffset, partitionId, LOGS_LUCENE9, 0)); + .isThrownBy(() -> new SnapshotMetadata(name, startTime, 0, maxOffset, partitionId, 0)); // Start time < end time assertThatIllegalArgumentException() .isThrownBy( - () -> - new SnapshotMetadata( - name, path, endTime, startTime, maxOffset, partitionId, LOGS_LUCENE9, 0)); + () -> new SnapshotMetadata(name, endTime, startTime, maxOffset, partitionId, 0)); // Start time same as end time. assertThat( - new SnapshotMetadata( - name, path, startTime, startTime, maxOffset, partitionId, LOGS_LUCENE9, 0) + new SnapshotMetadata(name, startTime, startTime, maxOffset, partitionId, 0) .endTimeEpochMs) .isEqualTo(startTime); assertThatIllegalArgumentException() - .isThrownBy( - () -> - new SnapshotMetadata( - name, path, startTime, endTime, -1, partitionId, LOGS_LUCENE9, 0)); + .isThrownBy(() -> new SnapshotMetadata(name, startTime, endTime, -1, partitionId, 0)); assertThatIllegalArgumentException() - .isThrownBy( - () -> - new SnapshotMetadata( - name, path, startTime, endTime, maxOffset, "", LOGS_LUCENE9, 0)); + .isThrownBy(() -> new SnapshotMetadata(name, startTime, endTime, maxOffset, "", 0)); } @Test public void testLive() { final String name = "testSnapshotId"; - final String path = "/testPath_" + name; final long startTime = 1; final long endTime = 100; final long maxOffset = 123; final String partitionId = "1"; SnapshotMetadata nonLiveSnapshot = - new SnapshotMetadata( - name, path, startTime, endTime, maxOffset, partitionId, LOGS_LUCENE9, 0); - assertThat(SnapshotMetadata.isLive(nonLiveSnapshot)).isFalse(); + new SnapshotMetadata(name, startTime, endTime, maxOffset, partitionId, 100); + assertThat(nonLiveSnapshot.isLive()).isFalse(); SnapshotMetadata liveSnapshot = - new SnapshotMetadata( - name, - SnapshotMetadata.LIVE_SNAPSHOT_PATH, - startTime, - endTime, - maxOffset, - partitionId, - LOGS_LUCENE9, - 0); - assertThat(SnapshotMetadata.isLive(liveSnapshot)).isTrue(); + new SnapshotMetadata(name, startTime, endTime, maxOffset, partitionId, 0); + assertThat(liveSnapshot.isLive()).isTrue(); } } diff --git a/astra/src/test/java/com/slack/astra/server/AstraIndexerTest.java b/astra/src/test/java/com/slack/astra/server/AstraIndexerTest.java index 978d57378e..5a5c7d36a1 100644 --- a/astra/src/test/java/com/slack/astra/server/AstraIndexerTest.java +++ b/astra/src/test/java/com/slack/astra/server/AstraIndexerTest.java @@ -2,8 +2,6 @@ import static com.slack.astra.logstore.LuceneIndexStoreImpl.MESSAGES_FAILED_COUNTER; import static com.slack.astra.logstore.LuceneIndexStoreImpl.MESSAGES_RECEIVED_COUNTER; -import static com.slack.astra.metadata.snapshot.SnapshotMetadata.LIVE_SNAPSHOT_PATH; -import static com.slack.astra.proto.metadata.Metadata.IndexType.LOGS_LUCENE9; import static com.slack.astra.server.AstraConfig.DEFAULT_START_STOP_DURATION; import static com.slack.astra.testlib.AstraConfigUtil.makeIndexerConfig; import static com.slack.astra.testlib.AstraConfigUtil.makeKafkaConfig; @@ -199,15 +197,7 @@ public void testDeleteStaleSnapshotAndStartConsumerKafkaSearchViaGrpcSearchApi() final long endTimeMs = 100; final long maxOffset = 50; SnapshotMetadata livePartition1 = - new SnapshotMetadata( - name + "live1", - LIVE_SNAPSHOT_PATH, - startTimeMs, - endTimeMs, - maxOffset, - "0", - LOGS_LUCENE9, - 0); + new SnapshotMetadata(name + "live1", startTimeMs, endTimeMs, maxOffset, "0", 0); snapshotMetadataStore.createSync(livePartition1); assertThat(AstraMetadataTestUtils.listSyncUncached(snapshotMetadataStore)) .containsOnly(livePartition1); @@ -246,27 +236,11 @@ public void testExceptionOnIndexerStartup() throws Exception { final long endTimeMs = 100; final long maxOffset = 50; SnapshotMetadata livePartition0 = - new SnapshotMetadata( - name + "live0", - LIVE_SNAPSHOT_PATH, - startTimeMs, - endTimeMs, - maxOffset, - "0", - LOGS_LUCENE9, - 0); + new SnapshotMetadata(name + "live0", startTimeMs, endTimeMs, maxOffset, "0", 0); snapshotMetadataStore.createSync(livePartition0); SnapshotMetadata livePartition1 = - new SnapshotMetadata( - name + "live1", - LIVE_SNAPSHOT_PATH, - startTimeMs, - endTimeMs, - maxOffset, - "1", - LOGS_LUCENE9, - 0); + new SnapshotMetadata(name + "live1", startTimeMs, endTimeMs, maxOffset, "1", 0); snapshotMetadataStore.createSync(livePartition1); assertThat(AstraMetadataTestUtils.listSyncUncached(snapshotMetadataStore)) .containsOnly(livePartition1, livePartition0); @@ -300,27 +274,11 @@ public void testWithMultipleLiveSnapshotsOnIndexerStart() throws Exception { final long endTimeMs = 100; final long maxOffset = 50; SnapshotMetadata livePartition0 = - new SnapshotMetadata( - name + "live0", - LIVE_SNAPSHOT_PATH, - startTimeMs, - endTimeMs, - maxOffset, - "0", - LOGS_LUCENE9, - 0); + new SnapshotMetadata(name + "live0", startTimeMs, endTimeMs, maxOffset, "0", 0); snapshotMetadataStore.createSync(livePartition0); SnapshotMetadata livePartition1 = - new SnapshotMetadata( - name + "live1", - LIVE_SNAPSHOT_PATH, - startTimeMs, - endTimeMs, - maxOffset, - "1", - LOGS_LUCENE9, - 0); + new SnapshotMetadata(name + "live1", startTimeMs, endTimeMs, maxOffset, "1", 0); snapshotMetadataStore.createSync(livePartition1); assertThat(AstraMetadataTestUtils.listSyncUncached(snapshotMetadataStore)) .containsOnly(livePartition1, livePartition0); @@ -355,36 +313,19 @@ public void testIndexerStartsWithPreviousOffset() throws Exception { // Create a live partition for this partiton final String name = "testSnapshotId"; - final String path = "/testPath_" + name; final long startTimeMs = 1; final long endTimeMs = 100; final long maxOffset = 50; SnapshotMetadata livePartition0 = - new SnapshotMetadata( - name + "live0", - LIVE_SNAPSHOT_PATH, - startTimeMs, - endTimeMs, - maxOffset, - "0", - LOGS_LUCENE9, - 0); + new SnapshotMetadata(name + "live0", startTimeMs, endTimeMs, maxOffset, "0", 0); snapshotMetadataStore.createSync(livePartition0); SnapshotMetadata livePartition1 = - new SnapshotMetadata( - name + "live1", - LIVE_SNAPSHOT_PATH, - startTimeMs, - endTimeMs, - maxOffset, - "1", - LOGS_LUCENE9, - 0); + new SnapshotMetadata(name + "live1", startTimeMs, endTimeMs, maxOffset, "1", 0); snapshotMetadataStore.createSync(livePartition1); final SnapshotMetadata partition0 = - new SnapshotMetadata(name, path, startTimeMs, endTimeMs, maxOffset, "0", LOGS_LUCENE9, 0); + new SnapshotMetadata(name, startTimeMs, endTimeMs, maxOffset, "0", 100); snapshotMetadataStore.createSync(partition0); assertThat(AstraMetadataTestUtils.listSyncUncached(snapshotMetadataStore)) @@ -422,36 +363,19 @@ public void testIndexerCreatesRecoveryTask() throws Exception { // Create a live partition for this partiton final String name = "testSnapshotId"; - final String path = "/testPath_" + name; final long startTimeMs = 1; final long endTimeMs = 100; final long maxOffset = 30; SnapshotMetadata livePartition0 = - new SnapshotMetadata( - name + "live0", - LIVE_SNAPSHOT_PATH, - startTimeMs, - endTimeMs, - maxOffset, - "0", - LOGS_LUCENE9, - 0); + new SnapshotMetadata(name + "live0", startTimeMs, endTimeMs, maxOffset, "0", 0); snapshotMetadataStore.createSync(livePartition0); SnapshotMetadata livePartition1 = - new SnapshotMetadata( - name + "live1", - LIVE_SNAPSHOT_PATH, - startTimeMs, - endTimeMs, - maxOffset, - "1", - LOGS_LUCENE9, - 0); + new SnapshotMetadata(name + "live1", startTimeMs, endTimeMs, maxOffset, "1", 0); snapshotMetadataStore.createSync(livePartition1); final SnapshotMetadata partition0 = - new SnapshotMetadata(name, path, startTimeMs, endTimeMs, maxOffset, "0", LOGS_LUCENE9, 0); + new SnapshotMetadata(name, startTimeMs, endTimeMs, maxOffset, "0", 100); snapshotMetadataStore.createSync(partition0); assertThat(AstraMetadataTestUtils.listSyncUncached(snapshotMetadataStore)) @@ -497,36 +421,19 @@ public void testIndexerShutdownTwice() throws Exception { // Create a live partition for this partiton final String name = "testSnapshotId"; - final String path = "/testPath_" + name; final long startTimeMs = 1; final long endTimeMs = 100; final long maxOffset = 30; SnapshotMetadata livePartition0 = - new SnapshotMetadata( - name + "live0", - LIVE_SNAPSHOT_PATH, - startTimeMs, - endTimeMs, - maxOffset, - "0", - LOGS_LUCENE9, - 0); + new SnapshotMetadata(name + "live0", startTimeMs, endTimeMs, maxOffset, "0", 0); snapshotMetadataStore.createSync(livePartition0); SnapshotMetadata livePartition1 = - new SnapshotMetadata( - name + "live1", - LIVE_SNAPSHOT_PATH, - startTimeMs, - endTimeMs, - maxOffset, - "1", - LOGS_LUCENE9, - 0); + new SnapshotMetadata(name + "live1", startTimeMs, endTimeMs, maxOffset, "1", 0); snapshotMetadataStore.createSync(livePartition1); final SnapshotMetadata partition0 = - new SnapshotMetadata(name, path, startTimeMs, endTimeMs, maxOffset, "0", LOGS_LUCENE9, 0); + new SnapshotMetadata(name, startTimeMs, endTimeMs, maxOffset, "0", 100); snapshotMetadataStore.createSync(partition0); assertThat(AstraMetadataTestUtils.listSyncUncached(snapshotMetadataStore)) @@ -576,36 +483,19 @@ public void testIndexerRestart() throws Exception { // Create a live partition for this partiton final String name = "testSnapshotId"; - final String path = "/testPath_" + name; final long startTimeMs = 1; final long endTimeMs = 100; final long maxOffset = 30; SnapshotMetadata livePartition0 = - new SnapshotMetadata( - name + "live0", - LIVE_SNAPSHOT_PATH, - startTimeMs, - endTimeMs, - maxOffset, - "0", - LOGS_LUCENE9, - 0); + new SnapshotMetadata(name + "live0", startTimeMs, endTimeMs, maxOffset, "0", 0); snapshotMetadataStore.createSync(livePartition0); SnapshotMetadata livePartition1 = - new SnapshotMetadata( - name + "live1", - LIVE_SNAPSHOT_PATH, - startTimeMs, - endTimeMs, - maxOffset, - "1", - LOGS_LUCENE9, - 0); + new SnapshotMetadata(name + "live1", startTimeMs, endTimeMs, maxOffset, "1", 0); snapshotMetadataStore.createSync(livePartition1); final SnapshotMetadata partition0 = - new SnapshotMetadata(name, path, startTimeMs, endTimeMs, maxOffset, "0", LOGS_LUCENE9, 0); + new SnapshotMetadata(name, startTimeMs, endTimeMs, maxOffset, "0", 100); snapshotMetadataStore.createSync(partition0); assertThat(AstraMetadataTestUtils.listSyncUncached(snapshotMetadataStore)) diff --git a/astra/src/test/java/com/slack/astra/server/ManagerApiGrpcTest.java b/astra/src/test/java/com/slack/astra/server/ManagerApiGrpcTest.java index 848f513f00..5d9e2d90ba 100644 --- a/astra/src/test/java/com/slack/astra/server/ManagerApiGrpcTest.java +++ b/astra/src/test/java/com/slack/astra/server/ManagerApiGrpcTest.java @@ -587,39 +587,29 @@ public void shouldFetchSnapshotsWithinTimeframeAndPartition() { long end = startTime + 10; SnapshotMetadata overlapsStartTimeIncluded = - new SnapshotMetadata( - "a", "a", startTime, startTime + 6, 0, "a", Metadata.IndexType.LOGS_LUCENE9, 0); + new SnapshotMetadata("a", startTime, startTime + 6, 0, "a", 0); SnapshotMetadata overlapsStartTimeExcluded = - new SnapshotMetadata( - "b", "b", startTime, startTime + 6, 0, "b", Metadata.IndexType.LOGS_LUCENE9, 0); + new SnapshotMetadata("b", startTime, startTime + 6, 0, "b", 0); SnapshotMetadata fullyOverlapsStartEndTimeIncluded = - new SnapshotMetadata( - "c", "c", startTime + 4, startTime + 11, 0, "a", Metadata.IndexType.LOGS_LUCENE9, 0); + new SnapshotMetadata("c", startTime + 4, startTime + 11, 0, "a", 0); SnapshotMetadata fullyOverlapsStartEndTimeExcluded = - new SnapshotMetadata( - "d", "d", startTime + 4, startTime + 11, 0, "b", Metadata.IndexType.LOGS_LUCENE9, 0); + new SnapshotMetadata("d", startTime + 4, startTime + 11, 0, "b", 0); SnapshotMetadata partiallyOverlapsStartEndTimeIncluded = - new SnapshotMetadata( - "e", "e", startTime + 4, startTime + 5, 0, "a", Metadata.IndexType.LOGS_LUCENE9, 0); + new SnapshotMetadata("e", startTime + 4, startTime + 5, 0, "a", 0); SnapshotMetadata partiallyOverlapsStartEndTimeExcluded = - new SnapshotMetadata( - "f", "f", startTime + 4, startTime + 5, 0, "b", Metadata.IndexType.LOGS_LUCENE9, 0); + new SnapshotMetadata("f", startTime + 4, startTime + 5, 0, "b", 0); SnapshotMetadata overlapsEndTimeIncluded = - new SnapshotMetadata( - "g", "g", startTime + 10, startTime + 15, 0, "a", Metadata.IndexType.LOGS_LUCENE9, 0); + new SnapshotMetadata("g", startTime + 10, startTime + 15, 0, "a", 0); SnapshotMetadata overlapsEndTimeExcluded = - new SnapshotMetadata( - "h", "h", startTime + 10, startTime + 15, 0, "b", Metadata.IndexType.LOGS_LUCENE9, 0); + new SnapshotMetadata("h", startTime + 10, startTime + 15, 0, "b", 0); SnapshotMetadata notWithinStartEndTimeExcluded1 = - new SnapshotMetadata( - "i", "i", startTime, startTime + 4, 0, "a", Metadata.IndexType.LOGS_LUCENE9, 0); + new SnapshotMetadata("i", startTime, startTime + 4, 0, "a", 0); SnapshotMetadata notWithinStartEndTimeExcluded2 = - new SnapshotMetadata( - "j", "j", startTime + 11, startTime + 15, 0, "a", Metadata.IndexType.LOGS_LUCENE9, 0); + new SnapshotMetadata("j", startTime + 11, startTime + 15, 0, "a", 0); DatasetMetadata datasetWithDataInPartitionA = new DatasetMetadata( @@ -669,11 +659,9 @@ public void shouldRestoreReplicaSinglePartition() { long end = startTime + 10; SnapshotMetadata snapshotIncluded = - new SnapshotMetadata( - "g", "g", startTime + 10, startTime + 15, 0, "a", Metadata.IndexType.LOGS_LUCENE9, 0); + new SnapshotMetadata("g", startTime + 10, startTime + 15, 0, "a", 0); SnapshotMetadata snapshotExcluded = - new SnapshotMetadata( - "h", "h", startTime + 10, startTime + 15, 0, "b", Metadata.IndexType.LOGS_LUCENE9, 0); + new SnapshotMetadata("h", startTime + 10, startTime + 15, 0, "b", 0); snapshotMetadataStore.createSync(snapshotIncluded); snapshotMetadataStore.createSync(snapshotExcluded); @@ -713,14 +701,11 @@ public void shouldRestoreReplicasMultiplePartitions() { long end = startTime + 10; SnapshotMetadata snapshotIncluded = - new SnapshotMetadata( - "a", "a", startTime + 10, startTime + 15, 0, "a", Metadata.IndexType.LOGS_LUCENE9, 0); + new SnapshotMetadata("a", startTime + 10, startTime + 15, 0, "a", 0); SnapshotMetadata snapshotIncluded2 = - new SnapshotMetadata( - "b", "b", startTime + 10, startTime + 15, 0, "b", Metadata.IndexType.LOGS_LUCENE9, 0); + new SnapshotMetadata("b", startTime + 10, startTime + 15, 0, "b", 0); SnapshotMetadata snapshotExcluded = - new SnapshotMetadata( - "c", "c", startTime + 10, startTime + 15, 0, "c", Metadata.IndexType.LOGS_LUCENE9, 0); + new SnapshotMetadata("c", startTime + 10, startTime + 15, 0, "c", 0); snapshotMetadataStore.createSync(snapshotIncluded); snapshotMetadataStore.createSync(snapshotIncluded2); @@ -763,14 +748,11 @@ public void shouldRestoreGivenSnapshotIds() { long startTime = Instant.now().toEpochMilli(); SnapshotMetadata snapshotFoo = - new SnapshotMetadata( - "foo", "a", startTime + 10, startTime + 15, 0, "a", Metadata.IndexType.LOGS_LUCENE9, 0); + new SnapshotMetadata("foo", startTime + 10, startTime + 15, 0, "a", 0); SnapshotMetadata snapshotBar = - new SnapshotMetadata( - "bar", "b", startTime + 10, startTime + 15, 0, "b", Metadata.IndexType.LOGS_LUCENE9, 0); + new SnapshotMetadata("bar", startTime + 10, startTime + 15, 0, "b", 0); SnapshotMetadata snapshotBaz = - new SnapshotMetadata( - "baz", "c", startTime + 10, startTime + 15, 0, "c", Metadata.IndexType.LOGS_LUCENE9, 0); + new SnapshotMetadata("baz", startTime + 10, startTime + 15, 0, "c", 0); snapshotMetadataStore.createSync(snapshotFoo); snapshotMetadataStore.createSync(snapshotBar); diff --git a/astra/src/test/java/com/slack/astra/server/RecoveryTaskCreatorTest.java b/astra/src/test/java/com/slack/astra/server/RecoveryTaskCreatorTest.java index c356332dd6..de0d7bad41 100644 --- a/astra/src/test/java/com/slack/astra/server/RecoveryTaskCreatorTest.java +++ b/astra/src/test/java/com/slack/astra/server/RecoveryTaskCreatorTest.java @@ -1,7 +1,5 @@ package com.slack.astra.server; -import static com.slack.astra.metadata.snapshot.SnapshotMetadata.LIVE_SNAPSHOT_PATH; -import static com.slack.astra.proto.metadata.Metadata.IndexType.LOGS_LUCENE9; import static com.slack.astra.server.RecoveryTaskCreator.RECOVERY_TASKS_CREATED; import static com.slack.astra.server.RecoveryTaskCreator.STALE_SNAPSHOT_DELETE_SUCCESS; import static com.slack.astra.server.RecoveryTaskCreator.getHighestDurableOffsetForPartition; @@ -21,6 +19,7 @@ import static org.mockito.Mockito.when; import brave.Tracing; +import com.slack.astra.chunk.ChunkInfo; import com.slack.astra.metadata.core.AstraMetadataTestUtils; import com.slack.astra.metadata.core.CuratorBuilder; import com.slack.astra.metadata.core.InternalMetadataStoreException; @@ -109,39 +108,22 @@ public void shutdown() throws IOException { @Test public void testStaleSnapshotDetection() { final String name = "testSnapshotId"; - final String path = "/testPath_" + name; final long startTime = 1; final long endTime = 100; final long maxOffset = 123; SnapshotMetadata partition1 = - new SnapshotMetadata( - name, path, startTime, endTime, maxOffset, partitionId, LOGS_LUCENE9, 0); + new SnapshotMetadata(name, startTime, endTime, maxOffset, partitionId, 100); SnapshotMetadata livePartition1 = new SnapshotMetadata( - name + "1", - LIVE_SNAPSHOT_PATH, - startTime, - endTime, - maxOffset, - partitionId, - LOGS_LUCENE9, - 0); + name + "1", startTime, ChunkInfo.MAX_FUTURE_TIME, maxOffset, partitionId, 0); SnapshotMetadata livePartition11 = new SnapshotMetadata( - name + "11", - LIVE_SNAPSHOT_PATH, - startTime, - endTime, - maxOffset, - partitionId, - LOGS_LUCENE9, - 0); + name + "11", startTime, ChunkInfo.MAX_FUTURE_TIME, maxOffset, partitionId, 0); SnapshotMetadata livePartition2 = - new SnapshotMetadata( - name + "2", LIVE_SNAPSHOT_PATH, startTime, endTime, maxOffset, "2", LOGS_LUCENE9, 0); + new SnapshotMetadata(name + "2", startTime, ChunkInfo.MAX_FUTURE_TIME, maxOffset, "2", 0); SnapshotMetadata partition2 = - new SnapshotMetadata(name + "3", path, startTime, endTime, maxOffset, "2", LOGS_LUCENE9, 0); + new SnapshotMetadata(name + "3", startTime, endTime, maxOffset, "2", 100); assertThat(getStaleLiveSnapshots(List.of(partition1), partitionId)).isEmpty(); assertThat(getStaleLiveSnapshots(List.of(partition2), partitionId)).isEmpty(); @@ -171,39 +153,22 @@ public void testStaleSnapshotDetection() { @Test public void testDeleteStaleSnapshotDeletion() { final String name = "testSnapshotId"; - final String path = "/testPath_" + name; final long startTime = 1; final long endTime = 100; final long maxOffset = 123; SnapshotMetadata partition1 = - new SnapshotMetadata( - name, path, startTime, endTime, maxOffset, partitionId, LOGS_LUCENE9, 0); + new SnapshotMetadata(name, startTime, endTime, maxOffset, partitionId, 100); SnapshotMetadata livePartition1 = new SnapshotMetadata( - name + "1", - LIVE_SNAPSHOT_PATH, - startTime, - endTime, - maxOffset, - partitionId, - LOGS_LUCENE9, - 0); + name + "1", startTime, ChunkInfo.MAX_FUTURE_TIME, maxOffset, partitionId, 0); SnapshotMetadata livePartition11 = new SnapshotMetadata( - name + "11", - LIVE_SNAPSHOT_PATH, - startTime, - endTime, - maxOffset, - partitionId, - LOGS_LUCENE9, - 0); + name + "11", startTime, ChunkInfo.MAX_FUTURE_TIME, maxOffset, partitionId, 0); SnapshotMetadata livePartition2 = - new SnapshotMetadata( - name + "2", LIVE_SNAPSHOT_PATH, startTime, endTime, maxOffset, "2", LOGS_LUCENE9, 0); + new SnapshotMetadata(name + "2", startTime, ChunkInfo.MAX_FUTURE_TIME, maxOffset, "2", 0); SnapshotMetadata partition2 = - new SnapshotMetadata(name + "3", path, startTime, endTime, maxOffset, "2", LOGS_LUCENE9, 0); + new SnapshotMetadata(name + "3", startTime, endTime, maxOffset, "2", 100); testDeleteSnapshots(List.of(partition1), 0, List.of(partition1)); testDeleteSnapshots(List.of(partition2), 0, List.of(partition2)); @@ -254,39 +219,22 @@ private void testDeleteSnapshots( @Test public void shouldStaleDeletionShouldHandleTimeouts() { final String name = "testSnapshotId"; - final String path = "/testPath_" + name; final long startTime = 1; final long endTime = 100; final long maxOffset = 123; SnapshotMetadata partition1 = - new SnapshotMetadata( - name, path, startTime, endTime, maxOffset, partitionId, LOGS_LUCENE9, 0); + new SnapshotMetadata(name, startTime, endTime, maxOffset, partitionId, 100); SnapshotMetadata livePartition1 = new SnapshotMetadata( - name + "1", - LIVE_SNAPSHOT_PATH, - startTime, - endTime, - maxOffset, - partitionId, - LOGS_LUCENE9, - 0); + name + "1", startTime, ChunkInfo.MAX_FUTURE_TIME, maxOffset, partitionId, 0); SnapshotMetadata livePartition11 = new SnapshotMetadata( - name + "11", - LIVE_SNAPSHOT_PATH, - startTime, - endTime, - maxOffset, - partitionId, - LOGS_LUCENE9, - 0); + name + "11", startTime, ChunkInfo.MAX_FUTURE_TIME, maxOffset, partitionId, 0); SnapshotMetadata livePartition2 = - new SnapshotMetadata( - name + "2", LIVE_SNAPSHOT_PATH, startTime, endTime, maxOffset, "2", LOGS_LUCENE9, 0); + new SnapshotMetadata(name + "2", startTime, ChunkInfo.MAX_FUTURE_TIME, maxOffset, "2", 0); SnapshotMetadata partition2 = - new SnapshotMetadata(name + "3", path, startTime, endTime, maxOffset, "2", LOGS_LUCENE9, 0); + new SnapshotMetadata(name + "3", startTime, endTime, maxOffset, "2", 100); testDeleteSnapshotsTimeouts(List.of(partition1), List.of(partition1), false); testDeleteSnapshotsTimeouts(List.of(livePartition1), List.of(livePartition1), true); @@ -348,39 +296,22 @@ private void testDeleteSnapshotsTimeouts( @Test public void shouldStaleDeletionShouldHandleExceptions() { final String name = "testSnapshotId"; - final String path = "/testPath_" + name; final long startTime = 1; final long endTime = 100; final long maxOffset = 123; final SnapshotMetadata partition1 = - new SnapshotMetadata( - name, path, startTime, endTime, maxOffset, partitionId, LOGS_LUCENE9, 0); + new SnapshotMetadata(name, startTime, endTime, maxOffset, partitionId, 100); final SnapshotMetadata livePartition1 = new SnapshotMetadata( - name + "1", - LIVE_SNAPSHOT_PATH, - startTime, - endTime, - maxOffset, - partitionId, - LOGS_LUCENE9, - 0); + name + "1", startTime, ChunkInfo.MAX_FUTURE_TIME, maxOffset, partitionId, 0); final SnapshotMetadata livePartition11 = new SnapshotMetadata( - name + "11", - LIVE_SNAPSHOT_PATH, - startTime, - endTime, - maxOffset, - partitionId, - LOGS_LUCENE9, - 0); + name + "11", startTime, ChunkInfo.MAX_FUTURE_TIME, maxOffset, partitionId, 0); final SnapshotMetadata livePartition2 = - new SnapshotMetadata( - name + "2", LIVE_SNAPSHOT_PATH, startTime, endTime, maxOffset, "2", LOGS_LUCENE9, 0); + new SnapshotMetadata(name + "2", startTime, endTime, maxOffset, "2", 0); final SnapshotMetadata partition2 = - new SnapshotMetadata(name + "3", path, startTime, endTime, maxOffset, "2", LOGS_LUCENE9, 0); + new SnapshotMetadata(name + "3", startTime, endTime, maxOffset, "2", 100); List snapshots = List.of(partition1, livePartition1, livePartition11, partition2, livePartition2); @@ -445,60 +376,28 @@ public void shouldStaleDeletionShouldHandleExceptions() { @Test public void testMaxOffset() { final String name = "testSnapshotId"; - final String path = "/testPath_" + name; final long startTime = 1; final long endTime = 100; final long maxOffset = 100; final SnapshotMetadata partition1 = - new SnapshotMetadata( - name, path, startTime, endTime, maxOffset, partitionId, LOGS_LUCENE9, 0); + new SnapshotMetadata(name, startTime, endTime, maxOffset, partitionId, 0); final SnapshotMetadata partition11 = - new SnapshotMetadata( - name + "1", - path, - endTime + 1, - endTime * 2, - maxOffset * 2, - partitionId, - LOGS_LUCENE9, - 0); + new SnapshotMetadata(name + "1", endTime + 1, endTime * 2, maxOffset * 2, partitionId, 0); final SnapshotMetadata partition12 = new SnapshotMetadata( - name + "12", - path, - endTime * 2 + 1, - endTime * 3, - maxOffset * 3, - partitionId, - LOGS_LUCENE9, - 0); + name + "12", endTime * 2 + 1, endTime * 3, maxOffset * 3, partitionId, 0); final String partitionId2 = "2"; final long partition2Offset = maxOffset * 10; final SnapshotMetadata partition2 = - new SnapshotMetadata( - name + "2", path, startTime, endTime, partition2Offset, partitionId2, LOGS_LUCENE9, 0); + new SnapshotMetadata(name + "2", startTime, endTime, partition2Offset, partitionId2, 0); final SnapshotMetadata partition21 = new SnapshotMetadata( - name + "21", - path, - endTime + 1, - endTime * 2, - partition2Offset * 2, - partitionId2, - LOGS_LUCENE9, - 0); + name + "21", endTime + 1, endTime * 2, partition2Offset * 2, partitionId2, 0); final SnapshotMetadata partition22 = new SnapshotMetadata( - name + "22", - path, - endTime * 2 + 1, - endTime * 3, - partition2Offset * 3, - partitionId2, - LOGS_LUCENE9, - 0); + name + "22", endTime * 2 + 1, endTime * 3, partition2Offset * 3, partitionId2, 0); // empty results assertThat( @@ -768,13 +667,12 @@ public void testInit() { // Data exists for not for this partition. final String name = "testSnapshotId"; - final String path = "/testPath_" + name; final long startTime = 1; final long endTime = 100; final long maxOffset = 100; final SnapshotMetadata partition1 = - new SnapshotMetadata(name, path, startTime, endTime, maxOffset, "2", LOGS_LUCENE9, 0); + new SnapshotMetadata(name, startTime, endTime, maxOffset, "2", 0); snapshotMetadataStore.createSync(partition1); await().until(() -> snapshotMetadataStore.listSync().contains(partition1)); @@ -782,8 +680,7 @@ public void testInit() { .isEqualTo(1); final SnapshotMetadata partition11 = - new SnapshotMetadata( - name + "1", path, endTime + 1, endTime * 2, maxOffset * 2, "2", LOGS_LUCENE9, 0); + new SnapshotMetadata(name + "1", endTime + 1, endTime * 2, maxOffset * 2, "2", 0); snapshotMetadataStore.createSync(partition11); await().until(() -> snapshotMetadataStore.listSync().contains(partition11)); assertThat(AstraMetadataTestUtils.listSyncUncached(snapshotMetadataStore)) @@ -827,20 +724,18 @@ public void testDetermineStartOffsetReturnsNegativeWhenNoOffset() { // Data exists for not for this partition. final String name = "testSnapshotId"; - final String path = "/testPath_" + name; final long startTime = 1; final long endTime = 100; final long maxOffset = 100; final SnapshotMetadata partition1 = - new SnapshotMetadata(name, path, startTime, endTime, maxOffset, "2", LOGS_LUCENE9, 0); + new SnapshotMetadata(name, startTime, endTime, maxOffset, "2", 0); snapshotMetadataStore.createSync(partition1); await().until(() -> snapshotMetadataStore.listSync().contains(partition1)); assertThat(recoveryTaskCreator.determineStartingOffset(0, 0, indexerConfig)).isNegative(); final SnapshotMetadata partition11 = - new SnapshotMetadata( - name + "1", path, endTime + 1, endTime * 2, maxOffset * 2, "2", LOGS_LUCENE9, 0); + new SnapshotMetadata(name + "1", endTime + 1, endTime * 2, maxOffset * 2, "2", 0); snapshotMetadataStore.createSync(partition11); await().until(() -> snapshotMetadataStore.listSync().contains(partition11)); assertThat(AstraMetadataTestUtils.listSyncUncached(snapshotMetadataStore)) @@ -1110,14 +1005,12 @@ public void testDetermineStartingOffsetOnlySnapshotsNoDelay() { assertThat(recoveryTaskCreator.determineStartingOffset(1000, 0, indexerConfig)).isNegative(); final String name = "testSnapshotId"; - final String path = "/testPath_" + name; final long startTime = 1; final long endTime = 100; final long maxOffset = 100; final SnapshotMetadata partition1 = - new SnapshotMetadata( - name, path, startTime, endTime, maxOffset, partitionId, LOGS_LUCENE9, 0); + new SnapshotMetadata(name, startTime, endTime, maxOffset, partitionId, 100); snapshotMetadataStore.createSync(partition1); await().until(() -> snapshotMetadataStore.listSync().contains(partition1)); assertThat( @@ -1133,14 +1026,7 @@ public void testDetermineStartingOffsetOnlySnapshotsNoDelay() { final SnapshotMetadata partition11 = new SnapshotMetadata( - name + "11", - path, - endTime + 1, - endTime * 2, - maxOffset * 2, - partitionId, - LOGS_LUCENE9, - 0); + name + "11", endTime + 1, endTime * 2, maxOffset * 2, partitionId, 100); snapshotMetadataStore.createSync(partition11); await().until(() -> snapshotMetadataStore.listSync().contains(partition11)); @@ -1157,14 +1043,7 @@ public void testDetermineStartingOffsetOnlySnapshotsNoDelay() { // Live partition is cleaned up, no delay. SnapshotMetadata livePartition1 = new SnapshotMetadata( - name + "live1", - LIVE_SNAPSHOT_PATH, - startTime, - endTime, - maxOffset, - partitionId, - LOGS_LUCENE9, - 0); + name + "live1", startTime, ChunkInfo.MAX_FUTURE_TIME, maxOffset, partitionId, 0); snapshotMetadataStore.createSync(livePartition1); await().until(() -> snapshotMetadataStore.listSync().contains(livePartition1)); assertThat(AstraMetadataTestUtils.listSyncUncached(snapshotMetadataStore)) @@ -1180,14 +1059,7 @@ public void testDetermineStartingOffsetOnlySnapshotsNoDelay() { snapshotMetadataStore.createSync(livePartition1); SnapshotMetadata livePartition11 = new SnapshotMetadata( - name + "live11", - LIVE_SNAPSHOT_PATH, - startTime, - endTime, - maxOffset, - partitionId, - LOGS_LUCENE9, - 0); + name + "live11", startTime, ChunkInfo.MAX_FUTURE_TIME, maxOffset, partitionId, 0); snapshotMetadataStore.createSync(livePartition11); await().until(() -> snapshotMetadataStore.listSync().contains(livePartition11)); assertThat(AstraMetadataTestUtils.listSyncUncached(snapshotMetadataStore)) @@ -1204,14 +1076,7 @@ public void testDetermineStartingOffsetOnlySnapshotsNoDelay() { snapshotMetadataStore.createSync(livePartition11); SnapshotMetadata livePartition2 = new SnapshotMetadata( - name + "2", - LIVE_SNAPSHOT_PATH, - startTime, - endTime, - maxOffset * 5, - "2", - LOGS_LUCENE9, - 0); + name + "2", startTime, ChunkInfo.MAX_FUTURE_TIME, maxOffset * 5, "2", 0); snapshotMetadataStore.createSync(livePartition2); await() .until( @@ -1232,8 +1097,7 @@ public void testDetermineStartingOffsetOnlySnapshotsNoDelay() { snapshotMetadataStore.createSync(livePartition1); snapshotMetadataStore.createSync(livePartition11); SnapshotMetadata partition2 = - new SnapshotMetadata( - name + "3", path, startTime, endTime, maxOffset * 3, "2", LOGS_LUCENE9, 0); + new SnapshotMetadata(name + "3", startTime, endTime, maxOffset * 3, "2", 0); snapshotMetadataStore.createSync(partition2); await() .until( @@ -1269,14 +1133,12 @@ public void testDetermineStartingOffsetOnlySnapshotsWithDelay() { assertThat(recoveryTaskCreator.determineStartingOffset(1000, 0, indexerConfig)).isNegative(); final String name = "testSnapshotId"; - final String path = "/testPath_" + name; final long startTime = 1; final long endTime = 100; final long maxOffset = 100; final SnapshotMetadata partition1 = - new SnapshotMetadata( - name, path, startTime, endTime, maxOffset, partitionId, LOGS_LUCENE9, 0); + new SnapshotMetadata(name, startTime, endTime, maxOffset, partitionId, 100); snapshotMetadataStore.createSync(partition1); await().until(() -> snapshotMetadataStore.listSync().contains(partition1)); assertThat( @@ -1302,14 +1164,7 @@ public void testDetermineStartingOffsetOnlySnapshotsWithDelay() { final SnapshotMetadata partition11 = new SnapshotMetadata( - name + "11", - path, - endTime + 1, - endTime * 2, - maxOffset * 2, - partitionId, - LOGS_LUCENE9, - 0); + name + "11", endTime + 1, endTime * 2, maxOffset * 2, partitionId, 100); snapshotMetadataStore.createSync(partition11); await().until(() -> snapshotMetadataStore.listSync().contains(partition11)); @@ -1338,14 +1193,7 @@ public void testDetermineStartingOffsetOnlySnapshotsWithDelay() { // Live partition is cleaned up, new recovery task is created. SnapshotMetadata livePartition1 = new SnapshotMetadata( - name + "live1", - LIVE_SNAPSHOT_PATH, - startTime, - endTime, - maxOffset, - partitionId, - LOGS_LUCENE9, - 0); + name + "live1", startTime, ChunkInfo.MAX_FUTURE_TIME, maxOffset, partitionId, 0); snapshotMetadataStore.createSync(livePartition1); await() .until( @@ -1379,14 +1227,7 @@ public void testDetermineStartingOffsetOnlySnapshotsWithDelay() { snapshotMetadataStore.createSync(livePartition1); SnapshotMetadata livePartition11 = new SnapshotMetadata( - name + "live11", - LIVE_SNAPSHOT_PATH, - startTime, - endTime, - maxOffset, - partitionId, - LOGS_LUCENE9, - 0); + name + "live11", startTime, ChunkInfo.MAX_FUTURE_TIME, maxOffset, partitionId, 0); snapshotMetadataStore.createSync(livePartition11); await() .until( @@ -1425,14 +1266,7 @@ public void testDetermineStartingOffsetOnlySnapshotsWithDelay() { snapshotMetadataStore.createSync(livePartition11); SnapshotMetadata livePartition2 = new SnapshotMetadata( - name + "2", - LIVE_SNAPSHOT_PATH, - startTime, - endTime, - maxOffset * 5, - "2", - LOGS_LUCENE9, - 0); + name + "2", startTime, ChunkInfo.MAX_FUTURE_TIME, maxOffset * 5, "2", 0); snapshotMetadataStore.createSync(livePartition2); await() .until( @@ -1470,8 +1304,7 @@ public void testDetermineStartingOffsetOnlySnapshotsWithDelay() { snapshotMetadataStore.createSync(livePartition1); snapshotMetadataStore.createSync(livePartition11); SnapshotMetadata partition2 = - new SnapshotMetadata( - name + "3", path, startTime, endTime, maxOffset * 3, "2", LOGS_LUCENE9, 0); + new SnapshotMetadata(name + "3", startTime, endTime, maxOffset * 3, "2", 100); snapshotMetadataStore.createSync(partition2); await() .until( @@ -1531,14 +1364,12 @@ public void testRecoveryTaskCreationFailureFailsDetermineStartOffset() { assertThat(recoveryTaskCreator.determineStartingOffset(1000, 0, indexerConfig)).isNegative(); final String name = "testSnapshotId"; - final String path = "/testPath_" + name; final long startTime = 1; final long endTime = 100; final long maxOffset = 100; final SnapshotMetadata partition1 = - new SnapshotMetadata( - name, path, startTime, endTime, maxOffset, partitionId, LOGS_LUCENE9, 0); + new SnapshotMetadata(name, startTime, endTime, maxOffset, partitionId, 100); snapshotMetadataStore.createSync(partition1); await().until(() -> snapshotMetadataStore.listSync().contains(partition1)); assertThat( @@ -1584,14 +1415,12 @@ public void testSnapshotListFailureFailsDetermineStartOffset() { assertThat(recoveryTaskCreator.determineStartingOffset(1000, 0, indexerConfig)).isNegative(); final String name = "testSnapshotId"; - final String path = "/testPath_" + name; final long startTime = 1; final long endTime = 100; final long maxOffset = 100; final SnapshotMetadata partition1 = - new SnapshotMetadata( - name, path, startTime, endTime, maxOffset, partitionId, LOGS_LUCENE9, 0); + new SnapshotMetadata(name, startTime, endTime, maxOffset, partitionId, 100); snapshotMetadataStore.createSync(partition1); await().until(() -> snapshotMetadataStore.listSync().contains(partition1)); assertThat( @@ -1641,14 +1470,12 @@ public void testRecoveryListFailureFailsDetermineStartOffset() { assertThat(recoveryTaskCreator.determineStartingOffset(1000, 0, indexerConfig)).isNegative(); final String name = "testSnapshotId"; - final String path = "/testPath_" + name; final long startTime = 1; final long endTime = 100; final long maxOffset = 100; final SnapshotMetadata partition1 = - new SnapshotMetadata( - name, path, startTime, endTime, maxOffset, partitionId, LOGS_LUCENE9, 0); + new SnapshotMetadata(name, startTime, endTime, maxOffset, partitionId, 100); snapshotMetadataStore.createSync(partition1); await().until(() -> snapshotMetadataStore.listSync().contains(partition1)); assertThat( @@ -1699,14 +1526,12 @@ public void testFailureToDeleteStaleSnapshotsFailsDetermineStartOffset() assertThat(recoveryTaskCreator.determineStartingOffset(1000, 0, indexerConfig)).isNegative(); final String name = "testSnapshotId"; - final String path = "/testPath_" + name; final long startTime = 1; final long endTime = 100; final long maxOffset = 100; final SnapshotMetadata partition1 = - new SnapshotMetadata( - name, path, startTime, endTime, maxOffset, partitionId, LOGS_LUCENE9, 0); + new SnapshotMetadata(name, startTime, endTime, maxOffset, partitionId, 100); snapshotMetadataStore.createSync(partition1); await().until(() -> snapshotMetadataStore.listSync().contains(partition1)); assertThat( @@ -1728,14 +1553,7 @@ public void testFailureToDeleteStaleSnapshotsFailsDetermineStartOffset() // Add a live partition to be deleted. SnapshotMetadata livePartition1 = new SnapshotMetadata( - name + "1", - LIVE_SNAPSHOT_PATH, - startTime, - endTime, - maxOffset, - partitionId, - LOGS_LUCENE9, - 0); + name + "1", startTime, ChunkInfo.MAX_FUTURE_TIME, maxOffset, partitionId, 0); snapshotMetadataStore.createSync(livePartition1); await().until(() -> snapshotMetadataStore.listSync().contains(livePartition1)); assertThat(AstraMetadataTestUtils.listSyncUncached(snapshotMetadataStore)) @@ -1867,14 +1685,12 @@ public void testMultipleRecoveryTaskCreationWithSnapshotDelay() { assertThat(AstraMetadataTestUtils.listSyncUncached(recoveryTaskStore)).isEmpty(); final String name = "testSnapshotId"; - final String path = "/testPath_" + name; final long startTime = 1; final long endTime = 100; final long maxOffset = 100; final SnapshotMetadata partition1 = - new SnapshotMetadata( - name, path, startTime, endTime, maxOffset, partitionId, LOGS_LUCENE9, 0); + new SnapshotMetadata(name, startTime, endTime, maxOffset, partitionId, 100); snapshotMetadataStore.createSync(partition1); await().until(() -> snapshotMetadataStore.listSync().contains(partition1)); assertThat( diff --git a/astra/src/test/java/com/slack/astra/testlib/ChunkManagerUtil.java b/astra/src/test/java/com/slack/astra/testlib/ChunkManagerUtil.java index 789e3bad85..d4ae4b3a4f 100644 --- a/astra/src/test/java/com/slack/astra/testlib/ChunkManagerUtil.java +++ b/astra/src/test/java/com/slack/astra/testlib/ChunkManagerUtil.java @@ -124,7 +124,7 @@ public void close() throws IOException, TimeoutException { } public static List fetchNonLiveSnapshot(List snapshots) { - Predicate nonLiveSnapshotPredicate = s -> !SnapshotMetadata.isLive(s); + Predicate nonLiveSnapshotPredicate = s -> !s.isLive(); return fetchSnapshotMatching(snapshots, nonLiveSnapshotPredicate); } diff --git a/astra/src/test/java/com/slack/astra/util/SnapshotUtil.java b/astra/src/test/java/com/slack/astra/util/SnapshotUtil.java deleted file mode 100644 index d74a373348..0000000000 --- a/astra/src/test/java/com/slack/astra/util/SnapshotUtil.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.slack.astra.util; - -import com.slack.astra.metadata.snapshot.SnapshotMetadata; -import com.slack.astra.proto.metadata.Metadata; - -public class SnapshotUtil { - public static SnapshotMetadata makeSnapshot(String name) { - return new SnapshotMetadata( - name + "snapshotId", - "/testPath_" + name, - 1, - 100, - 1, - "1", - Metadata.IndexType.LOGS_LUCENE9, - 0); - } -}