Skip to content

Commit

Permalink
Remove BlobContainer.move() method (#31100)
Browse files Browse the repository at this point in the history 
closes #30680
  • Loading branch information
@tlrx
tlrx committed Jun 7, 2018
1 parent f1256fa commit 6e0a6d1
Show file tree
Hide file tree
Showing 22 changed files with 56 additions and 447 deletions.
5 changes: 0 additions & 5 deletions ...repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,11 +82,6 @@ public Map<String, BlobMetaData> listBlobsByPrefix(String blobNamePrefix) throws
throw new UnsupportedOperationException("URL repository doesn't support this operation");
}

@Override
public void move(String from, String to) throws IOException {
throw new UnsupportedOperationException("URL repository doesn't support this operation");
}

/**
* This operation is not supported by URLBlobContainer
*/
Expand Down
36 changes: 2 additions & 34 deletions ...re-storage/src/test/java/org/elasticsearch/repositories/azure/AzureStorageTestServer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@
import java.util.concurrent.atomic.AtomicLong;

import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Collections.emptyList;
import static java.util.Collections.emptyMap;
import static java.util.Collections.singletonMap;

Expand Down Expand Up @@ -159,44 +158,13 @@ private static PathTrie<RequestHandler> defaultHandlers(final String endpoint, f
objectsPaths("PUT " + endpoint + "/{container}").forEach(path ->
handlers.insert(path, (params, headers, body, requestId) -> {
final String destContainerName = params.get("container");
final String destBlobName = objectName(params);

final Container destContainer =containers.get(destContainerName);
if (destContainer == null) {
return newContainerNotFoundError(requestId);
}

final String destBlobName = objectName(params);

// Request is a copy request
List<String> headerCopySource = headers.getOrDefault("x-ms-copy-source", emptyList());
if (headerCopySource.isEmpty() == false) {
String srcBlobName = headerCopySource.get(0);

Container srcContainer = null;
for (Container container : containers.values()) {
String prefix = endpoint + "/" + container.name + "/";
if (srcBlobName.startsWith(prefix)) {
srcBlobName = srcBlobName.replaceFirst(prefix, "");
srcContainer = container;
break;
}
}

if (srcContainer == null || srcContainer.objects.containsKey(srcBlobName) == false) {
return newBlobNotFoundError(requestId);
}

byte[] bytes = srcContainer.objects.get(srcBlobName);
if (bytes != null) {
destContainer.objects.put(destBlobName, bytes);
return new Response(RestStatus.ACCEPTED, singletonMap("x-ms-copy-status", "success"), "text/plain", EMPTY_BYTE);
} else {
return newBlobNotFoundError(requestId);
}
} else {
destContainer.objects.put(destBlobName, body);
}

destContainer.objects.put(destBlobName, body);
return new Response(RestStatus.CREATED, emptyMap(), "text/plain", EMPTY_BYTE);
})
);
Expand Down
17 changes: 0 additions & 17 deletions ...itory-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,23 +127,6 @@ public Map<String, BlobMetaData> listBlobsByPrefix(@Nullable String prefix) thro
}
}

@Override
public void move(String sourceBlobName, String targetBlobName) throws IOException {
logger.trace("move({}, {})", sourceBlobName, targetBlobName);
try {
String source = keyPath + sourceBlobName;
String target = keyPath + targetBlobName;

logger.debug("moving blob [{}] to [{}] in container {{}}", source, target, blobStore);

blobStore.moveBlob(source, target);
} catch (URISyntaxException | StorageException e) {
logger.warn("can not move blob [{}] to [{}] in container {{}}: {}", sourceBlobName, targetBlobName, blobStore,
e.getMessage());
throw new IOException(e);
}
}

@Override
public Map<String, BlobMetaData> listBlobs() throws IOException {
logger.trace("listBlobs()");
Expand Down
18 changes: 5 additions & 13 deletions ...epository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,31 +98,23 @@ public boolean doesContainerExist()
return this.client.doesContainerExist(this.clientName, this.locMode, container);
}

public boolean blobExists(String blob) throws URISyntaxException, StorageException
{
public boolean blobExists(String blob) throws URISyntaxException, StorageException {
return this.client.blobExists(this.clientName, this.locMode, container, blob);
}

public void deleteBlob(String blob) throws URISyntaxException, StorageException
{
public void deleteBlob(String blob) throws URISyntaxException, StorageException {
this.client.deleteBlob(this.clientName, this.locMode, container, blob);
}

public InputStream getInputStream(String blob) throws URISyntaxException, StorageException, IOException
{
public InputStream getInputStream(String blob) throws URISyntaxException, StorageException, IOException {
return this.client.getInputStream(this.clientName, this.locMode, container, blob);
}

public Map<String,BlobMetaData> listBlobsByPrefix(String keyPath, String prefix) throws URISyntaxException, StorageException
{
public Map<String, BlobMetaData> listBlobsByPrefix(String keyPath, String prefix)
throws URISyntaxException, StorageException {
return this.client.listBlobsByPrefix(this.clientName, this.locMode, container, keyPath, prefix);
}

public void moveBlob(String sourceBlob, String targetBlob) throws URISyntaxException, StorageException
{
this.client.moveBlob(this.clientName, this.locMode, container, sourceBlob, targetBlob);
}

public void writeBlob(String blobName, InputStream inputStream, long blobSize) throws URISyntaxException, StorageException {
this.client.writeBlob(this.clientName, this.locMode, container, blobName, inputStream, blobSize);
}
Expand Down
3 changes: 0 additions & 3 deletions ...sitory-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,9 +78,6 @@ InputStream getInputStream(String account, LocationMode mode, String container,
Map<String,BlobMetaData> listBlobsByPrefix(String account, LocationMode mode, String container, String keyPath, String prefix)
throws URISyntaxException, StorageException;

void moveBlob(String account, LocationMode mode, String container, String sourceBlob, String targetBlob)
throws URISyntaxException, StorageException;

void writeBlob(String account, LocationMode mode, String container, String blobName, InputStream inputStream, long blobSize) throws
URISyntaxException, StorageException;

Expand Down
17 changes: 0 additions & 17 deletions ...ry-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -325,23 +325,6 @@ enumBlobListingDetails, null, generateOperationContext(account))) {
return blobsBuilder.immutableMap();
}

@Override
public void moveBlob(String account, LocationMode mode, String container, String sourceBlob, String targetBlob) throws URISyntaxException, StorageException {
logger.debug("moveBlob container [{}], sourceBlob [{}], targetBlob [{}]", container, sourceBlob, targetBlob);

CloudBlobClient client = this.getSelectedClient(account, mode);
CloudBlobContainer blobContainer = client.getContainerReference(container);
CloudBlockBlob blobSource = blobContainer.getBlockBlobReference(sourceBlob);
if (SocketAccess.doPrivilegedException(() -> blobSource.exists(null, null, generateOperationContext(account)))) {
CloudBlockBlob blobTarget = blobContainer.getBlockBlobReference(targetBlob);
SocketAccess.doPrivilegedVoidException(() -> {
blobTarget.startCopy(blobSource, null, null, null, generateOperationContext(account));
blobSource.delete(DeleteSnapshotsOption.NONE, null, null, generateOperationContext(account));
});
logger.debug("moveBlob container [{}], sourceBlob [{}], targetBlob [{}] -> done", container, sourceBlob, targetBlob);
}
}

@Override
public void writeBlob(String account, LocationMode mode, String container, String blobName, InputStream inputStream, long blobSize)
throws URISyntaxException, StorageException {
Expand Down
36 changes: 1 addition & 35 deletions ...ry-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,17 +104,6 @@ public Map<String, BlobMetaData> listBlobsByPrefix(String account, LocationMode
return blobsBuilder.immutableMap();
}

@Override
public void moveBlob(String account, LocationMode mode, String container, String sourceBlob, String targetBlob) throws URISyntaxException, StorageException {
for (String blobName : blobs.keySet()) {
if (endsWithIgnoreCase(blobName, sourceBlob)) {
ByteArrayOutputStream outputStream = blobs.get(blobName);
blobs.put(blobName.replace(sourceBlob, targetBlob), outputStream);
blobs.remove(blobName);
}
}
}

@Override
public void writeBlob(String account, LocationMode mode, String container, String blobName, InputStream inputStream, long blobSize)
throws URISyntaxException, StorageException {
Expand All @@ -134,7 +123,7 @@ public void writeBlob(String account, LocationMode mode, String container, Strin
* @param prefix the prefix to look for
* @see java.lang.String#startsWith
*/
public static boolean startsWithIgnoreCase(String str, String prefix) {
private static boolean startsWithIgnoreCase(String str, String prefix) {
if (str == null || prefix == null) {
return false;
}
Expand All @@ -149,29 +138,6 @@ public static boolean startsWithIgnoreCase(String str, String prefix) {
return lcStr.equals(lcPrefix);
}

/**
* Test if the given String ends with the specified suffix,
* ignoring upper/lower case.
*
* @param str the String to check
* @param suffix the suffix to look for
* @see java.lang.String#startsWith
*/
public static boolean endsWithIgnoreCase(String str, String suffix) {
if (str == null || suffix == null) {
return false;
}
if (str.endsWith(suffix)) {
return true;
}
if (str.length() < suffix.length()) {
return false;
}
String lcStr = str.substring(0, suffix.length()).toLowerCase(Locale.ROOT);
String lcPrefix = suffix.toLowerCase(Locale.ROOT);
return lcStr.equals(lcPrefix);
}

private static class PermissionRequiringInputStream extends ByteArrayInputStream {

private PermissionRequiringInputStream(byte[] buf) {
Expand Down
65 changes: 0 additions & 65 deletions ...torage/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageTestServer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -367,47 +367,6 @@ private static PathTrie<RequestHandler> defaultHandlers(final String endpoint, f
return newResponse(RestStatus.OK, emptyMap(), buildObjectResource(bucket.name, objectId, body));
});

// Rewrite or Copy Object
//
// https://cloud.google.com/storage/docs/json_api/v1/objects/rewrite
// https://cloud.google.com/storage/docs/json_api/v1/objects/copy
handlers.insert("POST " + endpoint + "/storage/v1/b/{srcBucket}/o/{src}/{action}/b/{destBucket}/o/{dest}",
(params, headers, body) -> {
final String action = params.get("action");
if ((action.equals("rewriteTo") == false) && (action.equals("copyTo") == false)) {
return newError(RestStatus.INTERNAL_SERVER_ERROR, "Action not implemented. None of \"rewriteTo\" or \"copyTo\".");
}
final String source = params.get("src");
if (Strings.hasText(source) == false) {
return newError(RestStatus.INTERNAL_SERVER_ERROR, "source object name is missing");
}
final Bucket srcBucket = buckets.get(params.get("srcBucket"));
if (srcBucket == null) {
return newError(RestStatus.NOT_FOUND, "source bucket not found");
}
final String dest = params.get("dest");
if (Strings.hasText(dest) == false) {
return newError(RestStatus.INTERNAL_SERVER_ERROR, "destination object name is missing");
}
final Bucket destBucket = buckets.get(params.get("destBucket"));
if (destBucket == null) {
return newError(RestStatus.NOT_FOUND, "destination bucket not found");
}
final byte[] sourceBytes = srcBucket.objects.get(source);
if (sourceBytes == null) {
return newError(RestStatus.NOT_FOUND, "source object not found");
}
destBucket.objects.put(dest, sourceBytes);
if (action.equals("rewriteTo")) {
final XContentBuilder respBuilder = jsonBuilder();
buildRewriteResponse(respBuilder, destBucket.name, dest, sourceBytes.length);
return newResponse(RestStatus.OK, emptyMap(), respBuilder);
} else {
assert action.equals("copyTo");
return newResponse(RestStatus.OK, emptyMap(), buildObjectResource(destBucket.name, dest, sourceBytes));
}
});

// List Objects
//
// https://cloud.google.com/storage/docs/json_api/v1/objects/list
Expand Down Expand Up @@ -701,28 +660,4 @@ private static XContentBuilder buildObjectResource(final XContentBuilder builder
.field("size", String.valueOf(bytes.length))
.endObject();
}

/**
* Builds the rewrite response as defined by
* https://cloud.google.com/storage/docs/json_api/v1/objects/rewrite
*/
private static XContentBuilder buildRewriteResponse(final XContentBuilder builder,
final String destBucket,
final String dest,
final int byteSize) throws IOException {
builder.startObject()
.field("kind", "storage#rewriteResponse")
.field("totalBytesRewritten", String.valueOf(byteSize))
.field("objectSize", String.valueOf(byteSize))
.field("done", true)
.startObject("resource")
.field("kind", "storage#object")
.field("id", String.join("/", destBucket, dest))
.field("name", dest)
.field("bucket", destBucket)
.field("size", String.valueOf(byteSize))
.endObject()
.endObject();
return builder;
}
}
6 changes: 0 additions & 6 deletions ...gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@

import java.io.IOException;
import java.io.InputStream;
import java.nio.file.FileAlreadyExistsException;
import java.util.Map;

class GoogleCloudStorageBlobContainer extends AbstractBlobContainer {
Expand Down Expand Up @@ -74,11 +73,6 @@ public void deleteBlob(String blobName) throws IOException {
blobStore.deleteBlob(buildKey(blobName));
}

@Override
public void move(String sourceBlobName, String targetBlobName) throws IOException {
blobStore.moveBlob(buildKey(sourceBlobName), buildKey(targetBlobName));
}

protected String buildKey(String blobName) {
assert blobName != null;
return path + blobName;
Expand Down
24 changes: 0 additions & 24 deletions ...ory-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
import com.google.cloud.storage.Bucket;
import com.google.cloud.storage.Storage;
import com.google.cloud.storage.Storage.BlobListOption;
import com.google.cloud.storage.Storage.CopyRequest;
import com.google.cloud.storage.StorageException;
import org.elasticsearch.common.SuppressForbidden;
import org.elasticsearch.common.blobstore.BlobContainer;
Expand Down Expand Up @@ -314,29 +313,6 @@ void deleteBlobs(Collection<String> blobNames) throws IOException {
}
}

/**
* Moves a blob within the same bucket
*
* @param sourceBlobName name of the blob to move
* @param targetBlobName new name of the blob in the same bucket
*/
void moveBlob(String sourceBlobName, String targetBlobName) throws IOException {
final BlobId sourceBlobId = BlobId.of(bucket, sourceBlobName);
final BlobId targetBlobId = BlobId.of(bucket, targetBlobName);
final CopyRequest request = CopyRequest.newBuilder()
.setSource(sourceBlobId)
.setTarget(targetBlobId)
.build();
SocketAccess.doPrivilegedVoidIOException(() -> {
// There's no atomic "move" in GCS so we need to copy and delete
storage.copy(request).getResult();
final boolean deleted = storage.delete(sourceBlobId);
if (deleted == false) {
throw new IOException("Failed to move source [" + sourceBlobName + "] to target [" + targetBlobName + "]");
}
});
}

private static String buildKey(String keyPath, String s) {
assert s != null;
return keyPath + s;
Expand Down
11 changes: 0 additions & 11 deletions plugins/repository-gcs/src/test/java/com/google/cloud/storage/StorageRpcOptionUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,6 @@

import com.google.cloud.storage.spi.v1.StorageRpc;

import static org.mockito.Mockito.mock;

/**
* Utility class that exposed Google SDK package protected methods to
* create specific StorageRpc objects in unit tests.
Expand All @@ -42,13 +40,4 @@ public static String getPrefix(final Storage.BlobListOption... options) {
}
return null;
}

public static CopyWriter createCopyWriter(final Blob result) {
return new CopyWriter(mock(StorageOptions.class), mock(StorageRpc.RewriteResponse.class)) {
@Override
public Blob getResult() {
return result;
}
};
}
}
Loading

0 comments on commit 6e0a6d1

Please sign in to comment.