Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use the storage-blob library instead of deprecated azure-storage #223

Merged
merged 5 commits into from
Oct 26, 2023
Merged

Use the storage-blob library instead of deprecated azure-storage #223

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
2776214
Use the storage-blob library instead of deprecated azure-storage
aleksandrlevochkin Sep 26, 2023
7ecb7fa
Merge branch 'main' into users/levochkin/migrate-blobstorage-provider…
aleksandrlevochkin Sep 27, 2023
1e0f30a
Merge branch 'main' into users/levochkin/migrate-blobstorage-provider…
kirill-ivlev Oct 11, 2023
7f7c644
Merge branch 'main' into users/levochkin/migrate-blobstorage-provider…
kirill-ivlev Oct 17, 2023
4a95365
Merge branch 'main' into users/levochkin/migrate-blobstorage-provider…
kirill-ivlev Oct 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
187 changes: 81 additions & 106 deletions common-npm-packages/az-blobstorage-provider/azureBlobStorageProvider.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import path = require('path');
import azureStorage = require('azure-storage');
import fs = require('fs');
import models = require('artifact-engine/Models');
import store = require('artifact-engine/Store');
import tl = require('azure-pipelines-task-lib/task');
import { Readable } from 'stream';
import { BlobItem, BlobServiceClient, ContainerClient, StorageSharedKeyCredential } from '@azure/storage-blob';

const resourcePath: string = path.join(__dirname, 'module.json');
tl.setResourcePath(resourcePath);
Expand All @@ -12,158 +12,133 @@ export class AzureBlobProvider implements models.IArtifactProvider {

public artifactItemStore: store.ArtifactItemStore;

constructor(storageAccount: string, container: string, accessKey: string, prefixFolderPath?: string, host?: string, addPrefixToDownloadedItems?: boolean) {
constructor(storageAccount: string, containerName: string, accessKey: string, prefixFolderPath?: string, addPrefixToDownloadedItems?: boolean) {
this._storageAccount = storageAccount;
this._accessKey = accessKey;
this._container = container;
this._containerName = containerName;

if (!!prefixFolderPath) {
this._prefixFolderPath = prefixFolderPath.endsWith("/") ? prefixFolderPath : prefixFolderPath + "/";
} else {
this._prefixFolderPath = "";
}
this._blobSvc = azureStorage.createBlobService(this._storageAccount, this._accessKey, host);

const sharedKeyCredential = new StorageSharedKeyCredential(this._storageAccount, this._accessKey);

this._blobServiceClient = new BlobServiceClient(
`https://${this._storageAccount}.blob.core.windows.net`,
sharedKeyCredential
);

this._containerClient = this._blobServiceClient.getContainerClient(this._containerName);
this._addPrefixToDownloadedItems = !!addPrefixToDownloadedItems;
}

public putArtifactItem(item: models.ArtifactItem, readStream: NodeJS.ReadableStream): Promise<models.ArtifactItem> {
return new Promise(async (resolve, reject) => {
await this._ensureContainerExistence();

var self = this;
console.log(tl.loc("UploadingItem", item.path));
var blobPath = this._prefixFolderPath ? this._prefixFolderPath + "/" + item.path : item.path;

var writeStream = this._blobSvc.createWriteStreamToBlockBlob(this._container, blobPath, null, function (error, result, response) {
if (error) {
console.log(tl.loc("FailedToUploadBlob", blobPath, error.message));
reject(error);
} else {
var blobUrl = self._blobSvc.getUrl(self._container, blobPath);
console.log(tl.loc("CreatedBlobForItem", item.path, blobUrl));
item.metadata["destinationUrl"] = blobUrl;
resolve(item);
}
});

readStream.pipe(writeStream);
writeStream.on("error",
(error) => {
console.log("ErrorInWriteStream", error);
reject(error);
});
readStream.on("error",
(error) => {
console.log(tl.loc("ErrorInReadStream", error));
reject(error);
});
});
public async putArtifactItem(item: models.ArtifactItem, readStream: Readable): Promise<models.ArtifactItem> {
await this._containerClient.createIfNotExists();
console.log(tl.loc("UploadingItem", item.path));

const blobPath = this._prefixFolderPath ? this._prefixFolderPath + "/" + item.path : item.path;
const blockBlobClient = this._containerClient.getBlockBlobClient(blobPath);

try {
await blockBlobClient.uploadStream(readStream);
const blobUrl = blockBlobClient.url + "/" + this._containerName + "/" + blobPath;

console.log(tl.loc("CreatedBlobForItem", item.path, blobUrl));
item.metadata["destinationUrl"] = blobUrl;

return item;
} catch (error) {
console.log(tl.loc("FailedToUploadBlob", blobPath, error.message));
throw error;
}
}

public getRootItems(): Promise<models.ArtifactItem[]> {
return this._getItems(this._container, this._prefixFolderPath);
return this._getItems(this._containerName, this._prefixFolderPath);
}

public getArtifactItems(artifactItem: models.ArtifactItem): Promise<models.ArtifactItem[]> {
throw new Error(tl.loc("GetArtifactItemsNotSupported"));
}

public getArtifactItem(artifactItem: models.ArtifactItem): Promise<NodeJS.ReadableStream> {
return new Promise((resolve, reject) => {
var readStream: NodeJS.ReadableStream;
if (!this._addPrefixToDownloadedItems && !!this._prefixFolderPath) {
// Adding prefix path to get the absolute path
readStream = this._blobSvc.createReadStream(this._container, this._prefixFolderPath + artifactItem.path, null);
} else {
readStream = this._blobSvc.createReadStream(this._container, artifactItem.path, null);
}
public async getArtifactItem(artifactItem: models.ArtifactItem): Promise<NodeJS.ReadableStream> {
let blobPath = artifactItem.path;

if (!this._addPrefixToDownloadedItems && !!this._prefixFolderPath) {
blobPath += this._prefixFolderPath;
}

const blockBlobClient = this._containerClient.getBlockBlobClient(blobPath);

try {
const downloadResponse = await blockBlobClient.download();

// Replace full path by filename in order to save the file directly to destination folder
artifactItem.path = path.basename(artifactItem.path);
resolve(readStream);
});
return downloadResponse.readableStreamBody;
} catch (error) {
console.log(tl.loc("FailedToDownloadBlob", blobPath, error.message));
throw error;
}
}

public dispose() {
}

private _ensureContainerExistence(): Promise<void> {
return new Promise<void>((resolve, reject) => {
if (!this._isContainerExists) {
var self = this;
this._blobSvc.createContainerIfNotExists(this._container, function (error, result, response) {
if (!!error) {
console.log(tl.loc("FailedToCreateContainer", self._container, error.message));
reject(error);
} else {
self._isContainerExists = true;
console.log(tl.loc("CreatedContainer", self._container));
resolve();
}
});
} else {
resolve();
}
});
}

private _getItems(container: string, parentRelativePath?: string): Promise<models.ArtifactItem[]> {
var promise = new Promise<models.ArtifactItem[]>(async (resolve, reject) => {
var items: models.ArtifactItem[] = [];
var continuationToken = null;
var result;
do {
result = await this._getListOfItemsInsideContainer(container, parentRelativePath, continuationToken);
items = items.concat(this._convertBlobResultToArtifactItem(result.entries));
continuationToken = result.continuationToken;
if (!!continuationToken) {
console.log(tl.loc("ContinuationTokenExistsFetchingRemainingFiles"));
}
} while (continuationToken);
private async _getItems(container: string, parentRelativePath?: string): Promise<models.ArtifactItem[]> {
try {
const result = await this._getListOfItemsInsideContainer(container, parentRelativePath);
const artifactItems: models.ArtifactItem[] = this._convertBlobResultToArtifactItem(result);

console.log(tl.loc("SuccessFullyFetchedItemList"));
resolve(items);
});

return promise;
return artifactItems;
} catch (error) {
console.log(tl.loc("FailedToListItemInsideContainer", container, error.message));
throw error;
}
}

private async _getListOfItemsInsideContainer(container, parentRelativePath, continuationToken): Promise<azureStorage.BlobService.ListBlobsResult> {
var promise = new Promise<azureStorage.BlobService.ListBlobsResult>((resolve, reject) => {
this._blobSvc.listBlobsSegmentedWithPrefix(container, parentRelativePath, continuationToken, async (error, result) => {
if (!!error) {
console.log(tl.loc("FailedToListItemInsideContainer", container, error.message));
reject(error);
} else {
resolve(result);
}
});
});
private async _getListOfItemsInsideContainer(containerName: string, parentRelativePath: string): Promise<BlobItem[]> {
const options = { prefix: parentRelativePath };
const items: BlobItem[] = [];
const maxPageSize = 100;

for await (const page of this._containerClient.listBlobsFlat(options).byPage({ maxPageSize })) {
for (const blob of page.segment.blobItems) {
items.push(blob);
}
}

return promise;
return items;
}

private _convertBlobResultToArtifactItem(blobResult: azureStorage.BlobService.BlobResult[]): models.ArtifactItem[] {
private _convertBlobResultToArtifactItem(blobItems: BlobItem[]): models.ArtifactItem[] {
var artifactItems: models.ArtifactItem[] = new Array<models.ArtifactItem>();
blobResult.forEach(element => {
blobItems.forEach(item => {
var artifactitem: models.ArtifactItem = new models.ArtifactItem();
artifactitem.itemType = models.ItemType.File;
artifactitem.fileLength = parseInt(element.contentLength);
artifactitem.lastModified = new Date(element.lastModified + 'Z');
artifactitem.fileLength = item.properties.contentLength;
artifactitem.lastModified = new Date(item.properties.lastModified + 'Z');
if (!this._addPrefixToDownloadedItems && !!this._prefixFolderPath) {
// Supplying relative path without prefix; removing the first occurence
artifactitem.path = element.name.replace(this._prefixFolderPath, "").trim();
artifactitem.path = item.name.replace(this._prefixFolderPath, "").trim();
} else {
artifactitem.path = element.name;
artifactitem.path = item.name;
}
artifactItems.push(artifactitem);
});

return artifactItems;
}

private _storageAccount: string;
private _accessKey: string;
private _container: string;
private _containerName: string;
private _containerClient: ContainerClient;
private _blobServiceClient: BlobServiceClient;
private _prefixFolderPath: string;
private _isContainerExists: boolean = false;
private _blobSvc: azureStorage.BlobService;
private _addPrefixToDownloadedItems: boolean = false;
}
10 changes: 3 additions & 7 deletions common-npm-packages/az-blobstorage-provider/blobservice.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,23 +2,19 @@ import artifactProviders = require('artifact-engine/Providers');
import azureBlobProvider = require('./azureBlobStorageProvider');
import artifactProcessor = require('artifact-engine/Engine');
import models = require('artifact-engine/Models');
import path = require('path');
import util = require('util');

export class BlobService {
private _storageAccountName: string;
private _storageAccessKey: string;
private _host: string;

public constructor(storageAccountName: string, storageAccessKey: string, host?: string) {
public constructor(storageAccountName: string, storageAccessKey: string) {
this._storageAccountName = storageAccountName;
this._storageAccessKey = storageAccessKey;
this._host = host;
}

public async uploadBlobs(source: string, container: string, prefixFolderPath?: string, itemPattern?: string): Promise<string[]> {
var fileProvider = new artifactProviders.FilesystemProvider(source);
var azureProvider = new azureBlobProvider.AzureBlobProvider(this._storageAccountName, container, this._storageAccessKey, prefixFolderPath, this._host);
var azureProvider = new azureBlobProvider.AzureBlobProvider(this._storageAccountName, container, this._storageAccessKey, prefixFolderPath);
var processor = new artifactProcessor.ArtifactEngine();
var processorOptions = new artifactProcessor.ArtifactEngineOptions();
if (itemPattern) {
Expand All @@ -39,7 +35,7 @@ export class BlobService {

public async downloadBlobs(destination: string, container: string, prefixFolderPath?: string, itemPattern?: string, addPrefixToDownloadedItems?: boolean): Promise<void> {
var fileProvider = new artifactProviders.FilesystemProvider(destination);
var azureProvider = new azureBlobProvider.AzureBlobProvider(this._storageAccountName, container, this._storageAccessKey, prefixFolderPath, this._host, !!addPrefixToDownloadedItems);
var azureProvider = new azureBlobProvider.AzureBlobProvider(this._storageAccountName, container, this._storageAccessKey, prefixFolderPath, !!addPrefixToDownloadedItems);
var processor = new artifactProcessor.ArtifactEngine();
var processorOptions = new artifactProcessor.ArtifactEngineOptions();
if (itemPattern) {
Expand Down
2 changes: 0 additions & 2 deletions common-npm-packages/az-blobstorage-provider/manualTest.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
import path = require("path");
import util = require('util');
import * as BlobService from './blobservice';

var config = require("./config.json");
Expand Down
Loading