From a106249d11a8e643a61685191d334fb9223574fb Mon Sep 17 00:00:00 2001 From: sramazzina Date: Thu, 14 Mar 2024 23:09:50 +0100 Subject: [PATCH 1/8] Changed Blob type from PageBlobs to AppendBlobs --- .../apache/hop/vfs/azure/AzureFileObject.java | 105 +++++------------- 1 file changed, 28 insertions(+), 77 deletions(-) diff --git a/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/AzureFileObject.java b/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/AzureFileObject.java index c1ab71971de..e927e484c32 100644 --- a/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/AzureFileObject.java +++ b/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/AzureFileObject.java @@ -20,13 +20,7 @@ import com.microsoft.azure.storage.Constants; import com.microsoft.azure.storage.StorageException; -import com.microsoft.azure.storage.blob.CloudBlob; -import com.microsoft.azure.storage.blob.CloudBlobClient; -import com.microsoft.azure.storage.blob.CloudBlobContainer; -import com.microsoft.azure.storage.blob.CloudBlobDirectory; -import com.microsoft.azure.storage.blob.CloudBlockBlob; -import com.microsoft.azure.storage.blob.CloudPageBlob; -import com.microsoft.azure.storage.blob.ListBlobItem; +import com.microsoft.azure.storage.blob.*; import org.apache.commons.vfs2.FileObject; import org.apache.commons.vfs2.FileSystemException; import org.apache.commons.vfs2.FileType; @@ -54,28 +48,19 @@ public class AzureFileObject extends AbstractFileObject { public static final int DEFAULT_BLOB_SIZE = 1024; - private class PageBlobOutputStream extends OutputStream { + public class BlockBlobOutputStream extends OutputStream { + + private CloudAppendBlob ab; private final OutputStream outputStream; long written = 0; - private CloudPageBlob pb; - private int blockIncrement; - private long blobSize; - - private PageBlobOutputStream( - CloudPageBlob pb, OutputStream outputStream, int blockIncrement, long blobSize) { - if (blockIncrement % Constants.PAGE_SIZE != 0) - throw new IllegalArgumentException("Block increment must be in multiple of 512."); - if (blobSize % Constants.PAGE_SIZE != 0) - throw new IllegalArgumentException("Block increment must be in multiple of 512."); - this.blockIncrement = blockIncrement; - this.outputStream = new BufferedOutputStream(outputStream, blockIncrement); - this.pb = pb; - this.blobSize = blobSize; + + public BlockBlobOutputStream(CloudAppendBlob ab, OutputStream outputStream) { + this.ab = ab; + this.outputStream = outputStream; } @Override public void write(int b) throws IOException { - checkBlobSize(1); outputStream.write(b); written(1); } @@ -85,26 +70,6 @@ public void write(byte[] b) throws IOException { write(b, 0, b.length); } - @Override - public void write(byte[] b, int off, int len) throws IOException { - checkBlobSize(len); - outputStream.write(b, off, len); - written(len); - } - - private void checkBlobSize(int length) throws IOException { - if (written + length > blobSize) { - while (written + length > blobSize) { - blobSize += blockIncrement; - } - try { - pb.resize(blobSize); - } catch (StorageException e) { - throw new IOException("Failed to resize blob.", e); - } - } - } - protected void written(int len) { written += len; lastModified = System.currentTimeMillis(); @@ -112,40 +77,32 @@ protected void written(int len) { } @Override - public void flush() throws IOException {} + public void write(byte[] b, int off, int len) throws IOException { + outputStream.write(b, off, len); + written(len); + } + + @Override + public void flush() throws IOException { + super.flush(); + } @Override public void close() throws IOException { - HashMap md = new HashMap<>(pb.getMetadata()); + + HashMap md = new HashMap<>(ab.getMetadata()); md.put("ActualLength", String.valueOf(written)); - pb.getProperties().setContentDisposition("vfs ; length=\"" + written + "\""); - pb.setMetadata(md); + ab.getProperties().setContentDisposition("vfs ; length=\"" + written + "\""); + ab.setMetadata(md); try { - pb.uploadProperties(); + ab.uploadProperties(); } catch (StorageException e) { throw new IOException("Failed to update meta-data.", e); } - checkPageBoundary(); + outputStream.close(); - URI uri = pb.getUri(); - try { - uri = pb.getServiceClient().getCredentials().transformUri(uri); - } catch (URISyntaxException e) { - throw new IOException(e); - } catch (StorageException e) { - throw new IOException(e); - } closeBlob(); } - - private void checkPageBoundary() throws IOException { - long spare = written % Constants.PAGE_SIZE; - if (spare != 0) { - byte[] b = new byte[Constants.PAGE_SIZE - (int) spare]; - write(b); - outputStream.flush(); - } - } } private final CloudBlobClient service; @@ -436,21 +393,15 @@ protected long doGetLastModifiedTime() throws Exception { protected OutputStream doGetOutputStream(boolean bAppend) throws Exception { if (container != null && !containerPath.equals("")) { if (bAppend) throw new UnsupportedOperationException(); - final CloudPageBlob pb = container.getPageBlobReference(removeLeadingSlash(containerPath)); - - // Get the block increment... - // - IVariables variables = Variables.getADefaultVariableSpace(); - String configBlockIncrement = AzureConfigSingleton.getConfig().getBlockIncrement(); - int blockIncrement = Const.toInt(variables.resolve(configBlockIncrement), 4096); + final CloudAppendBlob cab = container.getAppendBlobReference(removeLeadingSlash(containerPath)); if (type == FileType.IMAGINARY) { type = FileType.FILE; - return new PageBlobOutputStream( - pb, pb.openWriteNew(DEFAULT_BLOB_SIZE), blockIncrement, DEFAULT_BLOB_SIZE); + return new BlockBlobOutputStream( + cab, cab.openWriteNew()); } else { - return new PageBlobOutputStream( - pb, pb.openWriteExisting(), blockIncrement, pb.getProperties().getLength()); + return new BlockBlobOutputStream( + cab, cab.openWriteExisting()); } } else { throw new UnsupportedOperationException(); From 34a7648c64c66b74d1914d568fa01160d45284f3 Mon Sep 17 00:00:00 2001 From: sramazzina Date: Mon, 18 Mar 2024 22:53:02 +0100 Subject: [PATCH 2/8] Refactoring --- .../org/apache/hop/vfs/azure/AzureFileObject.java | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/AzureFileObject.java b/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/AzureFileObject.java index e927e484c32..528d50ff6ab 100644 --- a/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/AzureFileObject.java +++ b/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/AzureFileObject.java @@ -18,7 +18,6 @@ package org.apache.hop.vfs.azure; -import com.microsoft.azure.storage.Constants; import com.microsoft.azure.storage.StorageException; import com.microsoft.azure.storage.blob.*; import org.apache.commons.vfs2.FileObject; @@ -27,12 +26,7 @@ import org.apache.commons.vfs2.provider.AbstractFileName; import org.apache.commons.vfs2.provider.AbstractFileObject; import org.apache.commons.vfs2.provider.UriParser; -import org.apache.hop.core.Const; -import org.apache.hop.core.variables.IVariables; -import org.apache.hop.core.variables.Variables; -import org.apache.hop.vfs.azure.config.AzureConfigSingleton; -import java.io.BufferedOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -46,15 +40,14 @@ import java.util.Map; public class AzureFileObject extends AbstractFileObject { - public static final int DEFAULT_BLOB_SIZE = 1024; - public class BlockBlobOutputStream extends OutputStream { + public class AppendBlobOutputStream extends OutputStream { private CloudAppendBlob ab; private final OutputStream outputStream; long written = 0; - public BlockBlobOutputStream(CloudAppendBlob ab, OutputStream outputStream) { + public AppendBlobOutputStream(CloudAppendBlob ab, OutputStream outputStream) { this.ab = ab; this.outputStream = outputStream; } @@ -397,10 +390,10 @@ protected OutputStream doGetOutputStream(boolean bAppend) throws Exception { if (type == FileType.IMAGINARY) { type = FileType.FILE; - return new BlockBlobOutputStream( + return new AppendBlobOutputStream( cab, cab.openWriteNew()); } else { - return new BlockBlobOutputStream( + return new AppendBlobOutputStream( cab, cab.openWriteExisting()); } } else { From 5c94299683f84e9d6a128d2944b9c12f784098e2 Mon Sep 17 00:00:00 2001 From: sramazzina Date: Thu, 14 Mar 2024 23:09:50 +0100 Subject: [PATCH 3/8] fix #3732 - Changed Blob type from PageBlobs to AppendBlobs --- .../apache/hop/vfs/azure/AzureFileObject.java | 105 +++++------------- 1 file changed, 28 insertions(+), 77 deletions(-) diff --git a/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/AzureFileObject.java b/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/AzureFileObject.java index c1ab71971de..e927e484c32 100644 --- a/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/AzureFileObject.java +++ b/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/AzureFileObject.java @@ -20,13 +20,7 @@ import com.microsoft.azure.storage.Constants; import com.microsoft.azure.storage.StorageException; -import com.microsoft.azure.storage.blob.CloudBlob; -import com.microsoft.azure.storage.blob.CloudBlobClient; -import com.microsoft.azure.storage.blob.CloudBlobContainer; -import com.microsoft.azure.storage.blob.CloudBlobDirectory; -import com.microsoft.azure.storage.blob.CloudBlockBlob; -import com.microsoft.azure.storage.blob.CloudPageBlob; -import com.microsoft.azure.storage.blob.ListBlobItem; +import com.microsoft.azure.storage.blob.*; import org.apache.commons.vfs2.FileObject; import org.apache.commons.vfs2.FileSystemException; import org.apache.commons.vfs2.FileType; @@ -54,28 +48,19 @@ public class AzureFileObject extends AbstractFileObject { public static final int DEFAULT_BLOB_SIZE = 1024; - private class PageBlobOutputStream extends OutputStream { + public class BlockBlobOutputStream extends OutputStream { + + private CloudAppendBlob ab; private final OutputStream outputStream; long written = 0; - private CloudPageBlob pb; - private int blockIncrement; - private long blobSize; - - private PageBlobOutputStream( - CloudPageBlob pb, OutputStream outputStream, int blockIncrement, long blobSize) { - if (blockIncrement % Constants.PAGE_SIZE != 0) - throw new IllegalArgumentException("Block increment must be in multiple of 512."); - if (blobSize % Constants.PAGE_SIZE != 0) - throw new IllegalArgumentException("Block increment must be in multiple of 512."); - this.blockIncrement = blockIncrement; - this.outputStream = new BufferedOutputStream(outputStream, blockIncrement); - this.pb = pb; - this.blobSize = blobSize; + + public BlockBlobOutputStream(CloudAppendBlob ab, OutputStream outputStream) { + this.ab = ab; + this.outputStream = outputStream; } @Override public void write(int b) throws IOException { - checkBlobSize(1); outputStream.write(b); written(1); } @@ -85,26 +70,6 @@ public void write(byte[] b) throws IOException { write(b, 0, b.length); } - @Override - public void write(byte[] b, int off, int len) throws IOException { - checkBlobSize(len); - outputStream.write(b, off, len); - written(len); - } - - private void checkBlobSize(int length) throws IOException { - if (written + length > blobSize) { - while (written + length > blobSize) { - blobSize += blockIncrement; - } - try { - pb.resize(blobSize); - } catch (StorageException e) { - throw new IOException("Failed to resize blob.", e); - } - } - } - protected void written(int len) { written += len; lastModified = System.currentTimeMillis(); @@ -112,40 +77,32 @@ protected void written(int len) { } @Override - public void flush() throws IOException {} + public void write(byte[] b, int off, int len) throws IOException { + outputStream.write(b, off, len); + written(len); + } + + @Override + public void flush() throws IOException { + super.flush(); + } @Override public void close() throws IOException { - HashMap md = new HashMap<>(pb.getMetadata()); + + HashMap md = new HashMap<>(ab.getMetadata()); md.put("ActualLength", String.valueOf(written)); - pb.getProperties().setContentDisposition("vfs ; length=\"" + written + "\""); - pb.setMetadata(md); + ab.getProperties().setContentDisposition("vfs ; length=\"" + written + "\""); + ab.setMetadata(md); try { - pb.uploadProperties(); + ab.uploadProperties(); } catch (StorageException e) { throw new IOException("Failed to update meta-data.", e); } - checkPageBoundary(); + outputStream.close(); - URI uri = pb.getUri(); - try { - uri = pb.getServiceClient().getCredentials().transformUri(uri); - } catch (URISyntaxException e) { - throw new IOException(e); - } catch (StorageException e) { - throw new IOException(e); - } closeBlob(); } - - private void checkPageBoundary() throws IOException { - long spare = written % Constants.PAGE_SIZE; - if (spare != 0) { - byte[] b = new byte[Constants.PAGE_SIZE - (int) spare]; - write(b); - outputStream.flush(); - } - } } private final CloudBlobClient service; @@ -436,21 +393,15 @@ protected long doGetLastModifiedTime() throws Exception { protected OutputStream doGetOutputStream(boolean bAppend) throws Exception { if (container != null && !containerPath.equals("")) { if (bAppend) throw new UnsupportedOperationException(); - final CloudPageBlob pb = container.getPageBlobReference(removeLeadingSlash(containerPath)); - - // Get the block increment... - // - IVariables variables = Variables.getADefaultVariableSpace(); - String configBlockIncrement = AzureConfigSingleton.getConfig().getBlockIncrement(); - int blockIncrement = Const.toInt(variables.resolve(configBlockIncrement), 4096); + final CloudAppendBlob cab = container.getAppendBlobReference(removeLeadingSlash(containerPath)); if (type == FileType.IMAGINARY) { type = FileType.FILE; - return new PageBlobOutputStream( - pb, pb.openWriteNew(DEFAULT_BLOB_SIZE), blockIncrement, DEFAULT_BLOB_SIZE); + return new BlockBlobOutputStream( + cab, cab.openWriteNew()); } else { - return new PageBlobOutputStream( - pb, pb.openWriteExisting(), blockIncrement, pb.getProperties().getLength()); + return new BlockBlobOutputStream( + cab, cab.openWriteExisting()); } } else { throw new UnsupportedOperationException(); From 4db65470be7f2be9d080124b935a6600142a9c5e Mon Sep 17 00:00:00 2001 From: sramazzina Date: Mon, 18 Mar 2024 22:53:02 +0100 Subject: [PATCH 4/8] Refactoring --- .../org/apache/hop/vfs/azure/AzureFileObject.java | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/AzureFileObject.java b/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/AzureFileObject.java index e927e484c32..528d50ff6ab 100644 --- a/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/AzureFileObject.java +++ b/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/AzureFileObject.java @@ -18,7 +18,6 @@ package org.apache.hop.vfs.azure; -import com.microsoft.azure.storage.Constants; import com.microsoft.azure.storage.StorageException; import com.microsoft.azure.storage.blob.*; import org.apache.commons.vfs2.FileObject; @@ -27,12 +26,7 @@ import org.apache.commons.vfs2.provider.AbstractFileName; import org.apache.commons.vfs2.provider.AbstractFileObject; import org.apache.commons.vfs2.provider.UriParser; -import org.apache.hop.core.Const; -import org.apache.hop.core.variables.IVariables; -import org.apache.hop.core.variables.Variables; -import org.apache.hop.vfs.azure.config.AzureConfigSingleton; -import java.io.BufferedOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -46,15 +40,14 @@ import java.util.Map; public class AzureFileObject extends AbstractFileObject { - public static final int DEFAULT_BLOB_SIZE = 1024; - public class BlockBlobOutputStream extends OutputStream { + public class AppendBlobOutputStream extends OutputStream { private CloudAppendBlob ab; private final OutputStream outputStream; long written = 0; - public BlockBlobOutputStream(CloudAppendBlob ab, OutputStream outputStream) { + public AppendBlobOutputStream(CloudAppendBlob ab, OutputStream outputStream) { this.ab = ab; this.outputStream = outputStream; } @@ -397,10 +390,10 @@ protected OutputStream doGetOutputStream(boolean bAppend) throws Exception { if (type == FileType.IMAGINARY) { type = FileType.FILE; - return new BlockBlobOutputStream( + return new AppendBlobOutputStream( cab, cab.openWriteNew()); } else { - return new BlockBlobOutputStream( + return new AppendBlobOutputStream( cab, cab.openWriteExisting()); } } else { From d492606417f10205db86b0433c9033ef53ff65e8 Mon Sep 17 00:00:00 2001 From: sramazzina Date: Tue, 19 Mar 2024 14:41:51 +0100 Subject: [PATCH 5/8] fix #3732 - Little refactoring to existing classes. Added new schema definition --- .../{PageBlobInputStream.java => AppendBlobInputStream.java} | 4 ++-- .../main/java/org/apache/hop/vfs/azure/AzureFileObject.java | 2 +- .../main/java/org/apache/hop/vfs/azure/AzureVfsPlugin.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) rename plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/{PageBlobInputStream.java => AppendBlobInputStream.java} (95%) diff --git a/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/PageBlobInputStream.java b/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/AppendBlobInputStream.java similarity index 95% rename from plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/PageBlobInputStream.java rename to plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/AppendBlobInputStream.java index 2d38fac4413..8641b202a2f 100644 --- a/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/PageBlobInputStream.java +++ b/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/AppendBlobInputStream.java @@ -23,13 +23,13 @@ import java.io.IOException; import java.io.InputStream; -public class PageBlobInputStream extends InputStream { +public class AppendBlobInputStream extends InputStream { private BlobInputStream inputStream; private long fileSize; private long totalRead = 0; - public PageBlobInputStream(BlobInputStream inputStream, long fileSize) { + public AppendBlobInputStream(BlobInputStream inputStream, long fileSize) { this.inputStream = inputStream; this.fileSize = fileSize; } diff --git a/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/AzureFileObject.java b/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/AzureFileObject.java index 528d50ff6ab..7b0eded6f0f 100644 --- a/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/AzureFileObject.java +++ b/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/AzureFileObject.java @@ -404,7 +404,7 @@ protected OutputStream doGetOutputStream(boolean bAppend) throws Exception { @Override protected InputStream doGetInputStream() throws Exception { if (container != null && !containerPath.equals("") && type == FileType.FILE) { - return new PageBlobInputStream(cloudBlob.openInputStream(), size); + return new AppendBlobInputStream(cloudBlob.openInputStream(), size); } else { throw new UnsupportedOperationException(); } diff --git a/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/AzureVfsPlugin.java b/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/AzureVfsPlugin.java index 03632232acf..3fcde51f0f2 100644 --- a/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/AzureVfsPlugin.java +++ b/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/AzureVfsPlugin.java @@ -26,7 +26,7 @@ public class AzureVfsPlugin implements IVfs { @Override public String[] getUrlSchemes() { - return new String[] {"azure"}; + return new String[] {"azure", "azfs"}; } @Override From 678d8f13303d85d7db34d476d76627ddb4ff1112 Mon Sep 17 00:00:00 2001 From: Sergio De Lorenzis Date: Tue, 19 Mar 2024 15:51:15 +0100 Subject: [PATCH 6/8] Removed File Block size for Azure Plugin --- .../vfs/azure/config/AzureConfigPlugin.java | 38 +------------------ 1 file changed, 1 insertion(+), 37 deletions(-) diff --git a/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/config/AzureConfigPlugin.java b/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/config/AzureConfigPlugin.java index 51c771ec934..1108f1033ff 100644 --- a/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/config/AzureConfigPlugin.java +++ b/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/config/AzureConfigPlugin.java @@ -74,17 +74,7 @@ public class AzureConfigPlugin implements IConfigOptions, IGuiPluginCompositeWid description = "The key to use for the Azure VFS") private String key; - @GuiWidgetElement( - id = WIDGET_ID_AZURE_BLOCK_INCREMENT, - parentId = ConfigPluginOptionsTab.GUI_WIDGETS_PARENT_ID, - type = GuiElementType.TEXT, - variables = true, - label = "i18n::AzureVFS.FileBlockSize.Label", - toolTip = "i18n::AzureVFS.FileBlockSize.Description") - @CommandLine.Option( - names = {"-azi", "--azure-block-increment"}, - description = "The block increment size for new files on Azure, multiples of 512 only.") - private String blockIncrement; + /** * Gets instance * @@ -96,7 +86,6 @@ public static AzureConfigPlugin getInstance() { AzureConfig config = AzureConfigSingleton.getConfig(); instance.account = config.getAccount(); instance.key = config.getKey(); - instance.blockIncrement = config.getBlockIncrement(); return instance; } @@ -121,12 +110,6 @@ public boolean handleOption( changed = true; } - if (blockIncrement != null) { - config.setBlockIncrement(blockIncrement); - log.logBasic("The Azure file block increment is set to '" + blockIncrement + "'"); - changed = true; - } - // Save to file if anything changed // if (changed) { @@ -163,10 +146,6 @@ public void persistContents(GuiCompositeWidgets compositeWidgets) { key = ((TextVar) control).getText(); AzureConfigSingleton.getConfig().setKey(key); break; - case WIDGET_ID_AZURE_BLOCK_INCREMENT: - blockIncrement = ((TextVar) control).getText(); - AzureConfigSingleton.getConfig().setKey(blockIncrement); - break; } } // Save the project... @@ -210,19 +189,4 @@ public void setKey(String key) { this.key = key; } - /** - * Gets blockIncrement - * - * @return value of blockIncrement - */ - public String getBlockIncrement() { - return blockIncrement; - } - - /** - * @param blockIncrement The blockIncrement to set - */ - public void setBlockIncrement(String blockIncrement) { - this.blockIncrement = blockIncrement; - } } From 22924931addb58ad436617af64f03dfc537aa685 Mon Sep 17 00:00:00 2001 From: sramazzina Date: Tue, 19 Mar 2024 17:04:40 +0100 Subject: [PATCH 7/8] fix #3732 - Managed multiple schema definitions on same Azure provider (azfs vs azure) --- .../hop/vfs/azure/AzureFileNameParser.java | 81 +++++++++++-------- 1 file changed, 49 insertions(+), 32 deletions(-) diff --git a/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/AzureFileNameParser.java b/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/AzureFileNameParser.java index 3b1bdf8095d..ff000274def 100644 --- a/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/AzureFileNameParser.java +++ b/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/AzureFileNameParser.java @@ -38,40 +38,57 @@ public AzureFileNameParser() { } @Override - public FileName parseUri(final VfsComponentContext context, FileName base, String filename) + public FileName parseUri(final VfsComponentContext context, FileName base, String uri) throws FileSystemException { - final StringBuilder name = new StringBuilder(); - Authority auth = null; - String path = null; - FileType fileType; - - int eidx = filename.indexOf("@/"); - if (eidx != -1) - filename = - filename.substring(0, eidx + 1) + "windowsazure.com" + filename.substring(eidx + 1); - - String scheme; - try { - auth = extractToPath(filename, name); - if (auth.getUserName() == null) { - scheme = UriParser.extractScheme(filename, name); - UriParser.canonicalizePath(name, 0, name.length(), this); - UriParser.fixSeparators(name); - } else { - scheme = auth.getScheme(); - } - fileType = UriParser.normalisePath(name); - path = name.toString(); - if (path.equals("")) { - path = "/"; + + StringBuilder sb = new StringBuilder(uri); + + UriParser.normalisePath(sb); + + String normalizedUri = sb.toString(); + String scheme = normalizedUri.substring(0, normalizedUri.indexOf(':')); + String absPath = "/"; + FileType fileType = FileType.IMAGINARY; + String[] s = normalizedUri.split("/"); + + if (s.length > 1) { + if (scheme.equals("azure")) { + + String container = s[1]; + for (int i = 1; i < s.length; i++) { + absPath += s[i]; + + if (s.length > 1 && i != s.length - 1) { + absPath += "/"; + } + } + + if (uri.endsWith("/")) { + fileType = FileType.FOLDER; + } else if (!absPath.endsWith("/")) { + fileType = FileType.FILE; + } + + } else if (scheme.equals("azfs")) { + + String account = s[1]; + String container = s[2]; + + for (int i = 2; i < s.length; i++) { + absPath += s[i]; + + if (s.length > 1 && i != s.length - 1) { + absPath += "/"; + } + } + + if (uri.endsWith("/")) { + fileType = FileType.FOLDER; + } else if (!absPath.endsWith("/")) { + fileType = FileType.FILE; + } } - } catch (FileSystemException fse) { - scheme = UriParser.extractScheme(filename, name); - UriParser.canonicalizePath(name, 0, name.length(), this); - UriParser.fixSeparators(name); - fileType = UriParser.normalisePath(name); - path = name.toString(); } - return new AzureFileName(scheme, path, fileType); + return new AzureFileName(scheme, absPath, fileType); } } From d889d345a248d46a8d60e32c6537165540a74593 Mon Sep 17 00:00:00 2001 From: Sergio De Lorenzis Date: Wed, 20 Mar 2024 12:33:52 +0100 Subject: [PATCH 8/8] Unit test v1 for AzureFileNameParser class --- .../vfs/azure/AzureFileNameParserTest.java | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 plugins/tech/azure/src/test/java/org/apache/hop/vfs/azure/AzureFileNameParserTest.java diff --git a/plugins/tech/azure/src/test/java/org/apache/hop/vfs/azure/AzureFileNameParserTest.java b/plugins/tech/azure/src/test/java/org/apache/hop/vfs/azure/AzureFileNameParserTest.java new file mode 100644 index 00000000000..87b50bb81fa --- /dev/null +++ b/plugins/tech/azure/src/test/java/org/apache/hop/vfs/azure/AzureFileNameParserTest.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.vfs.azure; + +import org.apache.commons.vfs2.FileSystemException; +import org.apache.commons.vfs2.FileType; +import org.apache.commons.vfs2.provider.VfsComponentContext; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.mockito.Mockito; + +import java.util.Arrays; +import java.util.Collection; + +import static org.junit.Assert.*; +@RunWith(Parameterized.class) +public class AzureFileNameParserTest { + + private AzureFileNameParser parser; + + private final String inputUri; + private final String expectedScheme; + private final String expectedContainer; + private final String expectedPathAfterContainer; + + private final FileType expectedType; + + @Before + public void setup(){ + parser = new AzureFileNameParser(); + } + + public AzureFileNameParserTest(String inputUri, String expectedScheme, String expectedContainer, String expectedPathAfterContainer, FileType expectedType) { + this.inputUri = inputUri; + this.expectedScheme = expectedScheme; + this.expectedContainer = expectedContainer; + this.expectedPathAfterContainer = expectedPathAfterContainer; + this.expectedType = expectedType; + } + + @Parameterized.Parameters + public static Collection azureUris() { + return Arrays.asList(new Object[][] { + { "azfs:/hopsa/container/folder1/parquet-test-delo2-azfs-00-0001.parquet","azfs", "container","/folder1/parquet-test-delo2-azfs-00-0001.parquet", FileType.FILE }, + { "azfs:/hopsa/container/folder1/", "azfs", "container","/folder1", FileType.FOLDER }, + { "azure://test/folder1/", "azure", "test", "/folder1", FileType.FOLDER }, + { "azure://mycontainer/folder1/parquet-test-delo2-azfs-00-0001.parquet","azure", "mycontainer","/folder1/parquet-test-delo2-azfs-00-0001.parquet", FileType.FILE }, + }); + } + + @Test + public void parseUri() throws FileSystemException { + VfsComponentContext context = Mockito.mock(VfsComponentContext.class); + + AzureFileName actual = (AzureFileName) parser.parseUri(context, null, inputUri); + + assertEquals(expectedScheme, actual.getScheme()); + assertEquals(expectedContainer, actual.getContainer()); + assertEquals(expectedPathAfterContainer, actual.getPathAfterContainer()); + assertEquals(expectedType, actual.getType()); + } +}