Skip to content

Commit

Permalink
Simplify BlobStoreRepository
Browse files Browse the repository at this point in the history 
Based on elastic/elasticsearch#42578
and elastic/elasticsearch#42833
  • Loading branch information
@mkleen
mkleen committed Jun 19, 2020
1 parent f2fd96c commit f743c87
Show file tree
Hide file tree
Showing 6 changed files with 205 additions and 293 deletions.
36 changes: 17 additions & 19 deletions ...-repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,6 @@ public static List<Setting<?>> mandatorySettings() {
return List.of(Repository.ACCOUNT_SETTING, Repository.KEY_SETTING);
}

private final BlobPath basePath;
private final ByteSizeValue chunkSize;
private final AzureStorageService storageService;
private final boolean readonly;
Expand All @@ -171,32 +170,36 @@ public AzureRepository(RepositoryMetaData metadata,
NamedXContentRegistry namedXContentRegistry,
AzureStorageService storageService,
ThreadPool threadPool) {
super(metadata, environment.settings(), namedXContentRegistry, threadPool);
super(metadata, environment.settings(), namedXContentRegistry, threadPool, buildBasePath(metadata));
this.chunkSize = Repository.CHUNK_SIZE_SETTING.get(metadata.settings());
this.storageService = storageService;

// If the user explicitly did not define a readonly value, we set it by ourselves depending on the location mode setting.
// For secondary_only setting, the repository should be read only
final LocationMode locationMode = Repository.LOCATION_MODE_SETTING.get(metadata.settings());
if (Repository.READONLY_SETTING.exists(metadata.settings())) {
this.readonly = Repository.READONLY_SETTING.get(metadata.settings());
} else {
this.readonly = locationMode == LocationMode.SECONDARY_ONLY;
}
}

private static BlobPath buildBasePath(RepositoryMetaData metadata) {
final String basePath = Strings.trimLeadingCharacter(Repository.BASE_PATH_SETTING.get(metadata.settings()), '/');
if (Strings.hasLength(basePath)) {
// Remove starting / if any
BlobPath path = new BlobPath();
for (final String elem : basePath.split("/")) {
path = path.add(elem);
}
this.basePath = path;
} else {
this.basePath = BlobPath.cleanPath();
}

// If the user explicitly did not define a readonly value, we set it by ourselves depending on the location mode setting.
// For secondary_only setting, the repository should be read only
final LocationMode locationMode = Repository.LOCATION_MODE_SETTING.get(metadata.settings());
if (Repository.READONLY_SETTING.exists(metadata.settings())) {
this.readonly = Repository.READONLY_SETTING.get(metadata.settings());
return path;
} else {
this.readonly = locationMode == LocationMode.SECONDARY_ONLY;
return BlobPath.cleanPath();
}
}



@VisibleForTesting
@Override
protected BlobStore getBlobStore() {
Expand All @@ -212,15 +215,10 @@ protected AzureBlobStore createBlobStore() {

LOGGER.debug((org.apache.logging.log4j.util.Supplier<?>) () -> new ParameterizedMessage(
"using container [{}], chunk_size [{}], compress [{}], base_path [{}]",
blobStore, chunkSize, isCompress(), basePath));
blobStore, chunkSize, isCompress(), basePath()));
return blobStore;
}

@Override
public BlobPath basePath() {
return basePath;
}

/**
* {@inheritDoc}
*/
Expand Down
8 changes: 1 addition & 7 deletions .../es-repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,6 @@ public final class HdfsRepository extends BlobStoreRepository {

private final Environment environment;
private final ByteSizeValue chunkSize;
private final BlobPath basePath = BlobPath.cleanPath();
private final URI uri;
private final String pathSetting;

Expand All @@ -69,7 +68,7 @@ public final class HdfsRepository extends BlobStoreRepository {

public HdfsRepository(RepositoryMetaData metadata, Environment environment,
NamedXContentRegistry namedXContentRegistry, ThreadPool threadPool) {
super(metadata, environment.settings(), namedXContentRegistry, threadPool);
super(metadata, environment.settings(), namedXContentRegistry, threadPool, BlobPath.cleanPath());

this.environment = environment;
this.chunkSize = metadata.settings().getAsBytesSize("chunk_size", null);
Expand Down Expand Up @@ -229,11 +228,6 @@ protected HdfsBlobStore createBlobStore() {
return blobStore;
}

@Override
public BlobPath basePath() {
return basePath;
}

@Override
protected ByteSizeValue chunkSize() {
return chunkSize;
Expand Down
25 changes: 10 additions & 15 deletions plugins/es-repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,8 +95,6 @@ public static List<Setting<?>> optionalSettings() {

private final ByteSizeValue chunkSize;

private final BlobPath basePath;

private final boolean serverSideEncryption;

private final String storageClass;
Expand All @@ -111,7 +109,7 @@ public static List<Setting<?>> optionalSettings() {
final NamedXContentRegistry namedXContentRegistry,
final S3Service service,
final ThreadPool threadPool) {
super(metadata, settings, namedXContentRegistry, threadPool);
super(metadata, settings, namedXContentRegistry, threadPool, buildBasePath(metadata));
this.service = service;

// Parse and validate the user's S3 Storage Class setting
Expand All @@ -129,13 +127,6 @@ public static List<Setting<?>> optionalSettings() {
") can't be lower than " + BUFFER_SIZE_SETTING.getKey() + " (" + bufferSize + ").");
}

final String basePath = BASE_PATH_SETTING.get(metadata.settings());
if (Strings.hasLength(basePath)) {
this.basePath = new BlobPath().add(basePath);
} else {
this.basePath = BlobPath.cleanPath();
}

this.serverSideEncryption = SERVER_SIDE_ENCRYPTION_SETTING.get(metadata.settings());

this.storageClass = STORAGE_CLASS_SETTING.get(metadata.settings());
Expand All @@ -151,6 +142,15 @@ public static List<Setting<?>> optionalSettings() {
storageClass);
}

private static BlobPath buildBasePath(RepositoryMetaData metadata) {
final String basePath = BASE_PATH_SETTING.get(metadata.settings());
if (Strings.hasLength(basePath)) {
return new BlobPath().add(basePath);
} else {
return BlobPath.cleanPath();
}
}

@Override
protected S3BlobStore createBlobStore() {
return new S3BlobStore(service, bucket, serverSideEncryption, bufferSize, cannedACL, storageClass, metadata);
Expand All @@ -162,11 +162,6 @@ protected BlobStore getBlobStore() {
return super.getBlobStore();
}

@Override
public BlobPath basePath() {
return basePath;
}

@Override
protected ByteSizeValue chunkSize() {
return chunkSize;
Expand Down
12 changes: 6 additions & 6 deletions ...ins/es-repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,23 +89,23 @@ public static List<Setting<?>> mandatorySettings() {
*/
public URLRepository(RepositoryMetaData metadata, Environment environment,
NamedXContentRegistry namedXContentRegistry, ThreadPool threadPool) {
super(metadata, environment.settings(), namedXContentRegistry, threadPool);
super(metadata, environment.settings(), namedXContentRegistry, threadPool, BlobPath.cleanPath());

if (URL_SETTING.exists(metadata.settings()) == false && REPOSITORIES_URL_SETTING.exists(settings) == false) {
if (URL_SETTING.exists(metadata.settings()) == false && REPOSITORIES_URL_SETTING.exists(environment.settings()) == false) {
throw new RepositoryException(metadata.name(), "missing url");
}
this.environment = environment;
supportedProtocols = SUPPORTED_PROTOCOLS_SETTING.get(settings);
urlWhiteList = ALLOWED_URLS_SETTING.get(settings).toArray(new URIPattern[]{});
supportedProtocols = SUPPORTED_PROTOCOLS_SETTING.get(environment.settings());
urlWhiteList = ALLOWED_URLS_SETTING.get(environment.settings()).toArray(new URIPattern[]{});
basePath = BlobPath.cleanPath();
url = URL_SETTING.exists(metadata.settings())
? URL_SETTING.get(metadata.settings()) : REPOSITORIES_URL_SETTING.get(settings);
? URL_SETTING.get(metadata.settings()) : REPOSITORIES_URL_SETTING.get(environment.settings());
}

@Override
protected BlobStore createBlobStore() {
URL normalizedURL = checkURL(url);
return new URLBlobStore(settings, normalizedURL);
return new URLBlobStore(environment.settings(), normalizedURL);
}

// only use for testing
Expand Down
Loading

0 comments on commit f743c87

Please sign in to comment.