Skip to content

Commit

Permalink
fix translog path and UTs
Browse files Browse the repository at this point in the history
  • Loading branch information
@linuxpi
linuxpi committed Jun 7, 2023
1 parent 8bdcdda commit 670b02f
Show file tree
Hide file tree
Showing 7 changed files with 36 additions and 13 deletions.
3 changes: 3 additions & 0 deletions .idea/inspectionProfiles/Project_Default.xml
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 5 additions & 1 deletion .idea/runConfigurations/Debug_OpenSearch.xml
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 5 additions & 0 deletions gradle/run.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,11 @@ testClusters {
testDistribution = 'archive'
if (numZones > 1) numberOfZones = numZones
if (numNodes > 1) numberOfNodes = numNodes
systemProperty 'opensearch.experimental.feature.replication_type.enabled', 'true'
systemProperty 'opensearch.experimental.feature.remote_store.enabled', 'true'
systemProperty 'opensearch.experimental.feature.segment_replication_experimental.enabled', 'true'
setting 'remote_store.segment.pressure.enabled', 'true'

}
}

Expand Down
2 changes: 1 addition & 1 deletion server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ public class RemoteFsTranslog extends Translog {
private final SetOnce<Boolean> olderPrimaryCleaned = new SetOnce<>();

private static final int REMOTE_DELETION_PERMITS = 2;
private static final String TRANSLOG = "translog";
public static final String TRANSLOG = "translog";

// Semaphore used to allow only single remote generation to happen at a time
private final Semaphore remoteGenerationDeletionPermits = new Semaphore(REMOTE_DELETION_PERMITS);
Expand Down
4 changes: 2 additions & 2 deletions server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ public TranslogTransferManager(
) {
this.shardId = shardId;
this.transferService = transferService;
this.remoteBaseTransferPath = remoteBaseTransferPath;
this.remoteBaseTransferPath = remoteBaseTransferPath.add("data");
this.remoteMetadataTransferPath = remoteBaseTransferPath.add(METADATA_DIR);
this.fileTransferTracker = fileTransferTracker;
}
Expand Down Expand Up @@ -110,7 +110,7 @@ public boolean transferSnapshot(TransferSnapshot transferSnapshot, TranslogTrans
fileSnapshot -> transferService.uploadBlobAsync(
ThreadPool.Names.TRANSLOG_TRANSFER,
fileSnapshot,
remoteBaseTransferPath.add("data").add(String.valueOf(fileSnapshot.getPrimaryTerm())),
remoteBaseTransferPath.add(String.valueOf(fileSnapshot.getPrimaryTerm())),
latchedActionListener
)
);
Expand Down
25 changes: 18 additions & 7 deletions server/src/test/java/org/opensearch/index/translog/RemoteFSTranslogTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.opensearch.common.util.BigArrays.NON_RECYCLING_INSTANCE;
import static org.opensearch.index.translog.RemoteFsTranslog.TRANSLOG;
import static org.opensearch.index.translog.SnapshotMatchers.containsOperationsInAnyOrder;
import static org.opensearch.index.translog.TranslogDeletionPolicies.createTranslogDeletionPolicy;

Expand Down Expand Up @@ -484,20 +485,22 @@ public void testSimpleOperationsUpload() throws Exception {
assertEquals(6, translog.allUploaded().size());

Set<String> mdFiles = blobStoreTransferService.listAll(
repository.basePath().add(shardId.getIndex().getUUID()).add(String.valueOf(shardId.id())).add("metadata")
getTranslogDirectory().add("metadata")
);
assertEquals(2, mdFiles.size());
logger.info("All md files {}", mdFiles);

Set<String> tlogFiles = blobStoreTransferService.listAll(
repository.basePath().add(shardId.getIndex().getUUID()).add(String.valueOf(shardId.id())).add(String.valueOf(primaryTerm.get()))
getTranslogDirectory().add("data").add(String.valueOf(primaryTerm.get()))
);
logger.info("All data files {}", tlogFiles);

// assert content of ckp and tlog files
BlobPath path = repository.basePath()
.add(shardId.getIndex().getUUID())
.add(String.valueOf(shardId.id()))
.add(TRANSLOG)
.add("data")
.add(String.valueOf(primaryTerm.get()));
for (TranslogReader reader : translog.readers) {
final long readerGeneration = reader.getGeneration();
Expand Down Expand Up @@ -537,6 +540,8 @@ public void testSimpleOperationsUpload() throws Exception {
repository.basePath()
.add(shardId.getIndex().getUUID())
.add(String.valueOf(shardId.id()))
.add(TRANSLOG)
.add("data")
.add(String.valueOf(primaryTerm.get()))
).size()
);
Expand All @@ -555,6 +560,8 @@ public void testSimpleOperationsUpload() throws Exception {
repository.basePath()
.add(shardId.getIndex().getUUID())
.add(String.valueOf(shardId.id()))
.add(TRANSLOG)
.add("data")
.add(String.valueOf(primaryTerm.get()))
).size()
);
Expand Down Expand Up @@ -587,7 +594,7 @@ public void testMetadataFileDeletion() throws Exception {
() -> assertEquals(
2,
blobStoreTransferService.listAll(
repository.basePath().add(shardId.getIndex().getUUID()).add(String.valueOf(shardId.id())).add(METADATA_DIR)
getTranslogDirectory().add(METADATA_DIR)
).size()
)
);
Expand All @@ -603,7 +610,7 @@ public void testMetadataFileDeletion() throws Exception {
() -> assertEquals(
1 + moreDocs,
blobStoreTransferService.listAll(
repository.basePath().add(shardId.getIndex().getUUID()).add(String.valueOf(shardId.id())).add(METADATA_DIR)
getTranslogDirectory().add(METADATA_DIR)
).size()
)
);
Expand All @@ -623,7 +630,7 @@ public void testMetadataFileDeletion() throws Exception {
() -> assertEquals(
2,
blobStoreTransferService.listAll(
repository.basePath().add(shardId.getIndex().getUUID()).add(String.valueOf(shardId.id())).add(METADATA_DIR)
getTranslogDirectory().add(METADATA_DIR)
).size()
)
);
Expand All @@ -643,7 +650,7 @@ public void testMetadataFileDeletion() throws Exception {

// Check all metadata files corresponds to old primary term
Set<String> mdFileNames = blobStoreTransferService.listAll(
repository.basePath().add(shardId.getIndex().getUUID()).add(String.valueOf(shardId.id())).add(METADATA_DIR)
getTranslogDirectory().add(METADATA_DIR)
);
assertTrue(mdFileNames.stream().allMatch(name -> name.startsWith(String.valueOf(oldPrimaryTerm).concat("__"))));

Expand All @@ -659,7 +666,7 @@ public void testMetadataFileDeletion() throws Exception {

// Check that all metadata files are belonging now to the new primary
mdFileNames = blobStoreTransferService.listAll(
repository.basePath().add(shardId.getIndex().getUUID()).add(String.valueOf(shardId.id())).add(METADATA_DIR)
getTranslogDirectory().add(METADATA_DIR)
);
assertTrue(mdFileNames.stream().allMatch(name -> name.startsWith(String.valueOf(newPrimaryTerm).concat("__"))));

Expand All @@ -671,6 +678,10 @@ public void testMetadataFileDeletion() throws Exception {
}
}

private BlobPath getTranslogDirectory() {
return repository.basePath().add(shardId.getIndex().getUUID()).add(String.valueOf(shardId.id())).add(TRANSLOG);
}

private Long populateTranslogOps(boolean withMissingOps) throws IOException {
long minSeqNo = SequenceNumbers.NO_OPS_PERFORMED;
long maxSeqNo = SequenceNumbers.NO_OPS_PERFORMED;
Expand Down
4 changes: 2 additions & 2 deletions test/framework/src/main/java/org/opensearch/bootstrap/BootstrapForTesting.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -170,8 +170,8 @@ public boolean implies(ProtectionDomain domain, Permission permission) {
});
// Create access control context for mocking
PriviledgedMockMaker.createAccessControlContext();
System.setSecurityManager(SecureSM.createTestSecureSM(getTrustedHosts()));
Security.selfTest();
// System.setSecurityManager(SecureSM.createTestSecureSM(getTrustedHosts()));
// Security.selfTest();

// guarantee plugin classes are initialized first, in case they have one-time hacks.
// this just makes unit testing more realistic
Expand Down

0 comments on commit 670b02f

Please sign in to comment.