From cd9e502215b413eb61a44a0fb29a896009405c48 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Tue, 3 Sep 2019 11:00:24 +0200 Subject: [PATCH 1/4] Use fixture in GoogleCloudStorageBlobStoreRepositoryTests --- ...eCloudStorageBlobStoreRepositoryTests.java | 298 ++++++++++++++++-- .../ESBlobStoreRepositoryIntegTestCase.java | 4 +- 2 files changed, 273 insertions(+), 29 deletions(-) diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java index fa9631d1a0010..9be14f3f89582 100644 --- a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java +++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java @@ -19,27 +19,99 @@ package org.elasticsearch.repositories.gcs; -import com.google.cloud.storage.Storage; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; +import com.sun.net.httpserver.HttpServer; +import org.apache.http.HttpStatus; import org.elasticsearch.cluster.metadata.RepositoryMetaData; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.mocksocket.MockHttpServer; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.repositories.blobstore.ESBlobStoreRepositoryIntegTestCase; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.rest.RestUtils; import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import java.io.BufferedInputStream; +import java.io.BufferedReader; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.URLDecoder; +import java.security.KeyPairGenerator; +import java.util.Arrays; +import java.util.Base64; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.zip.GZIPInputStream; +import static java.nio.charset.StandardCharsets.ISO_8859_1; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.CREDENTIALS_FILE_SETTING; +import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.ENDPOINT_SETTING; +import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.TOKEN_URI_SETTING; +import static org.elasticsearch.repositories.gcs.GoogleCloudStorageRepository.BUCKET; +import static org.elasticsearch.repositories.gcs.GoogleCloudStorageRepository.CLIENT_NAME; + +@SuppressForbidden(reason = "this test uses a HttpServer to emulate a Google Cloud Storage endpoint") public class GoogleCloudStorageBlobStoreRepositoryTests extends ESBlobStoreRepositoryIntegTestCase { - private static final String BUCKET = "gcs-repository-test"; + private static HttpServer httpServer; + private static byte[] serviceAccount; + + @BeforeClass + public static void startHttpServer() throws Exception { + httpServer = MockHttpServer.createHttp(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0); + httpServer.start(); + serviceAccount = createServiceAccount(); + } + + @Before + public void setUpHttpServer() { + httpServer.createContext("/", new InternalHttpHandler()); + httpServer.createContext("/token", new FakeOAuth2HttpHandler()); + } - // Static list of blobs shared among all nodes in order to act like a remote repository service: - // all nodes must see the same content - private static final ConcurrentMap blobs = new ConcurrentHashMap<>(); + @AfterClass + public static void stopHttpServer() { + httpServer.stop(0); + httpServer = null; + } + + @After + public void tearDownHttpServer() { + httpServer.removeContext("/"); + httpServer.removeContext("/token"); + } @Override protected String repositoryType() { @@ -50,38 +122,31 @@ protected String repositoryType() { protected Settings repositorySettings() { return Settings.builder() .put(super.repositorySettings()) - .put("bucket", BUCKET) - .put("base_path", GoogleCloudStorageBlobStoreRepositoryTests.class.getSimpleName()) + .put(BUCKET.getKey(), "bucket") + .put(CLIENT_NAME.getKey(), "test") .build(); } @Override protected Collection> nodePlugins() { - return Collections.singletonList(MockGoogleCloudStoragePlugin.class); - } - - @After - public void wipeRepository() { - blobs.clear(); + return Collections.singletonList(GoogleCloudStoragePlugin.class); } - public static class MockGoogleCloudStoragePlugin extends GoogleCloudStoragePlugin { + @Override + protected Settings nodeSettings(int nodeOrdinal) { + final Settings.Builder settings = Settings.builder(); + settings.put(super.nodeSettings(nodeOrdinal)); - public MockGoogleCloudStoragePlugin(final Settings settings) { - super(settings); - } + final InetSocketAddress address = httpServer.getAddress(); + final String endpoint = "http://" + InetAddresses.toUriString(address.getAddress()) + ":" + address.getPort(); + settings.put(ENDPOINT_SETTING.getConcreteSettingForNamespace("test").getKey(), endpoint); + settings.put(TOKEN_URI_SETTING.getConcreteSettingForNamespace("test").getKey(), endpoint + "/token"); - @Override - protected GoogleCloudStorageService createStorageService() { - return new MockGoogleCloudStorageService(); - } - } + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setFile(CREDENTIALS_FILE_SETTING.getConcreteSettingForNamespace("test").getKey(), serviceAccount); + settings.setSecureSettings(secureSettings); - public static class MockGoogleCloudStorageService extends GoogleCloudStorageService { - @Override - public Storage client(String clientName) { - return new MockStorage(BUCKET, blobs); - } + return settings.build(); } public void testChunkSize() { @@ -121,4 +186,183 @@ public void testChunkSize() { }); assertEquals("failed to parse value [101mb] for setting [chunk_size], must be <= [100mb]", e.getMessage()); } + + private static byte[] createServiceAccount() throws Exception { + final KeyPairGenerator keyPairGenerator = KeyPairGenerator.getInstance("RSA"); + keyPairGenerator.initialize(1024); + final String privateKey = Base64.getEncoder().encodeToString(keyPairGenerator.generateKeyPair().getPrivate().getEncoded()); + + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (XContentBuilder builder = new XContentBuilder(XContentType.JSON.xContent(), out)) { + builder.startObject(); + { + builder.field("type", "service_account"); + builder.field("project_id", getTestClass().getName().toLowerCase(Locale.ROOT)); + builder.field("private_key_id", UUID.randomUUID().toString()); + builder.field("private_key", "-----BEGIN PRIVATE KEY-----\n" + privateKey + "\n-----END PRIVATE KEY-----\n"); + builder.field("client_email", "elastic@appspot.gserviceaccount.com"); + builder.field("client_id", String.valueOf(randomNonNegativeLong())); + } + builder.endObject(); + } + return out.toByteArray(); + } + + /** + * Minimal HTTP handler that acts as a Google Cloud Storage compliant server + * + * Note: it does not support resumable uploads + */ + @SuppressForbidden(reason = "this test uses a HttpServer to emulate a Google Cloud Storage endpoint") + private static class InternalHttpHandler implements HttpHandler { + + private final ConcurrentMap blobs = new ConcurrentHashMap<>(); + + @Override + public void handle(final HttpExchange exchange) throws IOException { + final String request = exchange.getRequestMethod() + " " + exchange.getRequestURI().toString(); + try { + if (Regex.simpleMatch("GET /storage/v1/b/bucket/o*", request)) { + final Map params = new HashMap<>(); + RestUtils.decodeQueryString(exchange.getRequestURI().getQuery(), 0, params); + final String prefix = params.get("prefix"); + + final List> listOfBlobs = blobs.entrySet().stream() + .filter(blob -> prefix == null || blob.getKey().startsWith(prefix)).collect(Collectors.toList()); + + final StringBuilder list = new StringBuilder(); + list.append("{\"kind\":\"storage#objects\",\"items\":["); + for (Iterator> it = listOfBlobs.iterator(); it.hasNext(); ) { + Map.Entry blob = it.next(); + list.append("{\"kind\":\"storage#object\","); + list.append("\"bucket\":\"bucket\","); + list.append("\"name\":\"").append(blob.getKey()).append("\","); + list.append("\"id\":\"").append(blob.getKey()).append("\","); + list.append("\"size\":\"").append(blob.getValue().length()).append("\""); + list.append('}'); + + if (it.hasNext()) { + list.append(','); + } + } + list.append("]}"); + + byte[] response = list.toString().getBytes(UTF_8); + exchange.getResponseHeaders().add("Content-Type", "application/json; charset=utf-8"); + exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length); + exchange.getResponseBody().write(response); + + } else if (Regex.simpleMatch("GET /storage/v1/b/bucket*", request)) { + byte[] response = ("{\"kind\":\"storage#bucket\",\"name\":\"bucket\",\"id\":\"0\"}").getBytes(UTF_8); + exchange.getResponseHeaders().add("Content-Type", "application/json; charset=utf-8"); + exchange.sendResponseHeaders(HttpStatus.SC_OK, response.length); + exchange.getResponseBody().write(response); + + } else if (Regex.simpleMatch("GET /download/storage/v1/b/bucket/o/*", request)) { + BytesReference blob = blobs.get(exchange.getRequestURI().getPath().replace("/download/storage/v1/b/bucket/o/", "")); + if (blob != null) { + exchange.getResponseHeaders().add("Content-Type", "application/octet-stream"); + exchange.sendResponseHeaders(RestStatus.OK.getStatus(), blob.length()); + exchange.getResponseBody().write(blob.toBytesRef().bytes); + } else { + exchange.sendResponseHeaders(RestStatus.NOT_FOUND.getStatus(), -1); + } + + } else if (Regex.simpleMatch("DELETE /storage/v1/b/bucket/o/*", request)) { + int deletions = 0; + for (Iterator> iterator = blobs.entrySet().iterator(); iterator.hasNext(); ) { + Map.Entry blob = iterator.next(); + if (blob.getKey().startsWith(exchange.getRequestURI().toString())) { + iterator.remove(); + deletions++; + } + } + exchange.sendResponseHeaders((deletions > 0 ? RestStatus.OK : RestStatus.NO_CONTENT).getStatus(), -1); + + } else if (Regex.simpleMatch("POST /batch/storage/v1", request)) { + final String uri = "/storage/v1/b/bucket/o/"; + final StringBuilder batch = new StringBuilder(); + for (String line : Streams.readAllLines(new BufferedInputStream(exchange.getRequestBody()))) { + if (line.length() == 0 || line.startsWith("--") || line.toLowerCase(Locale.ROOT).startsWith("content")) { + batch.append(line).append('\n'); + } else if (line.startsWith("DELETE")) { + final String name = line.substring(line.indexOf(uri) + uri.length(), line.lastIndexOf(" HTTP")); + if (Strings.hasText(name)) { + if (blobs.entrySet().removeIf(blob -> blob.getKey().equals(URLDecoder.decode(name, UTF_8)))) { + batch.append("HTTP/1.1 204 NO_CONTENT").append('\n'); + batch.append('\n'); + } + } + } + } + byte[] response = batch.toString().getBytes(UTF_8); + exchange.getResponseHeaders().add("Content-Type", exchange.getRequestHeaders().getFirst("Content-Type")); + exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length); + exchange.getResponseBody().write(response); + + } else if (Regex.simpleMatch("POST /upload/storage/v1/b/bucket/*uploadType=multipart*", request)) { + byte[] response = new byte[0]; + try ( + InputStream input = new GZIPInputStream(exchange.getRequestBody()); + BufferedReader reader = new BufferedReader(new InputStreamReader(input, ISO_8859_1)) + ) { + String blob = null, line; + while ((line = reader.readLine()) != null) { + boolean markAndContinue = false; + if (line.length() == 0 || line.equals("\r\n") || line.startsWith("--") + || line.toLowerCase(Locale.ROOT).startsWith("content")) { + markAndContinue = true; + } else if (line.startsWith("{\"bucket\":\"bucket\"")) { + markAndContinue = true; + Matcher matcher = Pattern.compile("\"name\":\"([^\"]*)\"").matcher(line); + if (matcher.find()) { + blob = matcher.group(1); + response = line.getBytes(UTF_8); + } + } + if (markAndContinue) { + reader.mark(1); + continue; + } + if (blob != null) { + reader.reset(); + try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { + int c; + while ((c = reader.read()) != -1) { + out.write(c); // one char to one byte, because of the ISO_8859_1 encoding + } + byte[] trailing = ("\r\n--__END_OF_PART__--\r\n").getBytes(ISO_8859_1); + byte[] content = Arrays.copyOf(out.toByteArray(), out.toByteArray().length - trailing.length); + blobs.put(blob, new BytesArray(content)); + } finally { + //blob = null; + reader.mark(0); + } + } + } + } + exchange.getResponseHeaders().add("Content-Type", "application/json"); + exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length); + exchange.getResponseBody().write(response); + + } else { + exchange.sendResponseHeaders(RestStatus.INTERNAL_SERVER_ERROR.getStatus(), -1); + } + } finally { + exchange.close(); + } + } + } + + @SuppressForbidden(reason = "this test uses a HttpServer to emulate a fake OAuth2 authentication service") + private static class FakeOAuth2HttpHandler implements HttpHandler { + @Override + public void handle(final HttpExchange exchange) throws IOException { + byte[] response = ("{\"access_token\":\"foo\",\"token_type\":\"Bearer\",\"expires_in\":3600}").getBytes(UTF_8); + exchange.getResponseHeaders().add("Content-Type", "application/json"); + exchange.sendResponseHeaders(HttpStatus.SC_OK, response.length); + exchange.getResponseBody().write(response); + exchange.close(); + } + } } diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java index e78975fdab515..09814c33105d4 100644 --- a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java @@ -66,8 +66,8 @@ protected Settings repositorySettings() { final Settings.Builder settings = Settings.builder(); settings.put("compress", randomBoolean()); if (randomBoolean()) { - long size = 1 << randomIntBetween(7, 10); - settings.put("chunk_size", new ByteSizeValue(size, randomFrom(ByteSizeUnit.BYTES, ByteSizeUnit.KB))); + long size = 1 << randomInt(10); + settings.put("chunk_size", new ByteSizeValue(size, ByteSizeUnit.KB)); } return settings.build(); } From 3f57667279756b8ad4ddf40a3a8b91b8236ef7c9 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Wed, 4 Sep 2019 12:08:16 +0200 Subject: [PATCH 2/4] Change multipart upload --- ...eCloudStorageBlobStoreRepositoryTests.java | 71 ++++++++++--------- 1 file changed, 38 insertions(+), 33 deletions(-) diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java index 9be14f3f89582..07be6fd32ed3f 100644 --- a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java +++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java @@ -48,11 +48,8 @@ import org.junit.BeforeClass; import java.io.BufferedInputStream; -import java.io.BufferedReader; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.URLDecoder; @@ -74,7 +71,6 @@ import java.util.stream.Collectors; import java.util.zip.GZIPInputStream; -import static java.nio.charset.StandardCharsets.ISO_8859_1; import static java.nio.charset.StandardCharsets.UTF_8; import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.CREDENTIALS_FILE_SETTING; import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.ENDPOINT_SETTING; @@ -302,41 +298,50 @@ public void handle(final HttpExchange exchange) throws IOException { } else if (Regex.simpleMatch("POST /upload/storage/v1/b/bucket/*uploadType=multipart*", request)) { byte[] response = new byte[0]; - try ( - InputStream input = new GZIPInputStream(exchange.getRequestBody()); - BufferedReader reader = new BufferedReader(new InputStreamReader(input, ISO_8859_1)) - ) { - String blob = null, line; - while ((line = reader.readLine()) != null) { + try (BufferedInputStream in = new BufferedInputStream(new GZIPInputStream(exchange.getRequestBody()))) { + String blob = null; + int read; + while ((read = in.read()) != -1) { boolean markAndContinue = false; - if (line.length() == 0 || line.equals("\r\n") || line.startsWith("--") - || line.toLowerCase(Locale.ROOT).startsWith("content")) { - markAndContinue = true; - } else if (line.startsWith("{\"bucket\":\"bucket\"")) { - markAndContinue = true; - Matcher matcher = Pattern.compile("\"name\":\"([^\"]*)\"").matcher(line); - if (matcher.find()) { - blob = matcher.group(1); - response = line.getBytes(UTF_8); + try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { + do { // search next new line char and stops + char current = (char) read; + if (current == '\n') { + break; + } else if (current != '\r') { + out.write(read); + } + } while ((read = in.read()) != -1); + + final String line = new String(out.toByteArray(), UTF_8); + if (line.length() == 0 || line.equals("\r\n") || line.startsWith("--") + || line.toLowerCase(Locale.ROOT).startsWith("content")) { + markAndContinue = true; + } else if (line.startsWith("{\"bucket\":\"bucket\"")) { + markAndContinue = true; + Matcher matcher = Pattern.compile("\"name\":\"([^\"]*)\"").matcher(line); + if (matcher.find()) { + blob = matcher.group(1); + response = line.getBytes(UTF_8); + } + } + if (markAndContinue) { + in.mark(Integer.MAX_VALUE); + continue; } - } - if (markAndContinue) { - reader.mark(1); - continue; } if (blob != null) { - reader.reset(); - try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { - int c; - while ((c = reader.read()) != -1) { - out.write(c); // one char to one byte, because of the ISO_8859_1 encoding + in.reset(); + try (ByteArrayOutputStream binary = new ByteArrayOutputStream()) { + while ((read = in.read()) != -1) { + binary.write(read); } - byte[] trailing = ("\r\n--__END_OF_PART__--\r\n").getBytes(ISO_8859_1); - byte[] content = Arrays.copyOf(out.toByteArray(), out.toByteArray().length - trailing.length); - blobs.put(blob, new BytesArray(content)); + binary.flush(); + byte[] tmp = binary.toByteArray(); + // removes the trailing end "\r\n--__END_OF_PART__--\r\n" which is 23 bytes long + blobs.put(blob, new BytesArray(Arrays.copyOf(tmp, tmp.length - 23))); } finally { - //blob = null; - reader.mark(0); + blob = null; } } } From 69218392c136daa46668fc5b961223b9671ddc4b Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Wed, 4 Sep 2019 16:28:40 +0200 Subject: [PATCH 3/4] Fix \r\n handling --- ...leCloudStorageBlobStoreRepositoryTests.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java index 07be6fd32ed3f..b6ae48077263f 100644 --- a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java +++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java @@ -304,13 +304,19 @@ public void handle(final HttpExchange exchange) throws IOException { while ((read = in.read()) != -1) { boolean markAndContinue = false; try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { - do { // search next new line char and stops - char current = (char) read; - if (current == '\n') { - break; - } else if (current != '\r') { - out.write(read); + do { // search next consecutive {carriage return, new line} chars and stop + if ((char) read == '\r') { + int next = in.read(); + if (next != -1) { + if (next == '\n') { + break; + } + out.write(read); + out.write(next); + continue; + } } + out.write(read); } while ((read = in.read()) != -1); final String line = new String(out.toByteArray(), UTF_8); From aef77214e456d0217a1ef7ff95ed11cefcb0b552 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Wed, 4 Sep 2019 16:59:17 +0200 Subject: [PATCH 4/4] Feedback --- .../gcs/GoogleCloudStorageBlobStoreRepositoryTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java index b6ae48077263f..b267e686e6531 100644 --- a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java +++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java @@ -268,7 +268,7 @@ public void handle(final HttpExchange exchange) throws IOException { int deletions = 0; for (Iterator> iterator = blobs.entrySet().iterator(); iterator.hasNext(); ) { Map.Entry blob = iterator.next(); - if (blob.getKey().startsWith(exchange.getRequestURI().toString())) { + if (blob.getKey().equals(exchange.getRequestURI().toString())) { iterator.remove(); deletions++; }