feat(verdaccio-aws-s3-storage): separate s3 subfolders (key prefix fo…

…r different packages) (#313)

* test: createPackage with s3 mock

* feat: Add Custom Storage

Remove packagename as key for s3 objects
Add package path from custom storage of packages

* test: Add number of assertions to tests

* refactor: trailing slash function

remove the duplications in trailing slash on paths

* chore: add data to gitignore

this folder is used per convention for minio or localstack for database

* fix: replace s3config with whole config

this is to access the packages and their store key

* Add readme section

* fix: jest registry in yarn.lock

* Upgrade: aws-sdk

aws-s3-storage plugin

* fix: set minio endpoint in config.yaml

* fix: Remove registry.env file from repo

Update the README

* Fix: aws secret access key example in README

* Add npmignore

plugins/aws-s3-storage/.gitignore

@@ -1,2 +1,3 @@
 lib/
 node_modules/
+data

plugins/aws-s3-storage/.npmignore

@@ -0,0 +1,3 @@
+./conf
+Dockerfile
+docker-compose.yaml

plugins/aws-s3-storage/Dockerfile

@@ -0,0 +1,17 @@
+FROM verdaccio/verdaccio:4
+
+USER root
+
+ENV NODE_ENV=production
+
+RUN apk --no-cache add openssl ca-certificates wget && \
+    apk --no-cache add g++ gcc libgcc libstdc++ linux-headers make python && \
+    wget -q -O /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub && \
+    wget -q https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.25-r0/glibc-2.25-r0.apk && \
+    apk add glibc-2.25-r0.apk
+
+RUN npm i
+
+COPY . ./build/plugins/aws-s3-storage/
+
+USER verdaccio

plugins/aws-s3-storage/README.md

@@ -50,3 +50,38 @@ store:
     accessKeyId: your-access-key-id # optional, aws accessKeyId for private S3 bucket
     secretAccessKey: your-secret-access-key # optional, aws secretAccessKey for private S3 bucket
 ```
+
+store properties can be defined for packages. The storage location corresponds to the folder in s3 bucket.
+
+```
+packages:
+  '@scope/*':
+    access: all
+    publish: $all
+    storage: 'scoped'
+  '**':
+    access: $all
+    publish: $all
+    proxy: npmjs
+    storage: 'public'
+```
+
+# Developer Testing #
+
+In case of local testing, this project can be used self-efficiently. Four main ingredients are as follows:
+
+* config.yaml, see [verdaccio documentation](https://verdaccio.org/docs/en/configuration.html)
+* The provided docker file allows to test the plugin, with no need for main verdaccio application
+* The provided docker-compose also provides minio in orchestration as a local substitute for S3 backend
+* registry.envs set as follows. This file does not exist on the repo and should be generated after cloning the project.  
+
+```    
+AWS_ACCESS_KEY_ID=foobar
+AWS_SECRET_ACCESS_KEY=1234567e
+AWS_DEFAULT_REGION=eu-central-1
+AWS_S3_ENDPOINT=https://localhost:9000/
+AWS_S3_PATH_STYLE=true
+```
+
+The default values should work out of the box. If you change anything, make sure the corresponding variables are set in
+other parts of the ingredient as well.

plugins/aws-s3-storage/conf/config.yaml

@@ -0,0 +1,43 @@
+storage: /verdaccio/storage
+
+plugins: plugins
+
+store:
+  aws-s3-storage:
+    bucket: rise
+    keyPrefix: verdaccio
+    region: eu-central-1
+    endpoint: http://minio:9000
+    accessKeyId: foobar
+    secretAccessKey: 1234567e
+    s3ForcePathStyle: true
+
+uplinks:
+  npmjs:
+    url: https://registry.npmjs.org/
+    cache: false
+
+packages:
+  '@rise/*':
+    access: all
+    publish: $all
+    storage: 'private'
+  '@*/*':
+    access: $all
+    publish: $all
+    proxy: npmjs
+    storage: 'public'
+  '**':
+    access: $all
+    publish: $all
+    proxy: npmjs
+    storage: 'public'
+logs:
+  - {type: stdout, format: pretty, level: trace}
+
+listen:
+  - 0.0.0.0:4873
+
+auth:
+  htpasswd:
+    file: ./htpasswd

plugins/aws-s3-storage/docker-compose.yaml

@@ -0,0 +1,24 @@
+version: '3.7'
+
+services:
+  minio:
+    image: minio/minio
+    restart: always
+    command: server /data
+    environment:
+      MINIO_ACCESS_KEY: 'foobar'
+      MINIO_SECRET_KEY: '1234567e'
+    ports:
+      - 9000:9000
+    volumes:
+      - ./data/minio:/data
+
+  verdaccio:
+    container_name: verdaccio
+    build: .
+    env_file:
+      - registry.env
+    volumes:
+      - ./conf:/verdaccio/conf
+    ports:
+      - 4873:4873

plugins/aws-s3-storage/package.json

@@ -30,12 +30,13 @@
   "dependencies": {
     "@verdaccio/commons-api": "^0.1.2",
     "@verdaccio/streams": "^2.0.0",
-    "aws-sdk": "2.596.0"
+    "aws-sdk": "^2.607.0"
   },
   "devDependencies": {
     "@verdaccio/babel-preset": "^9.0.0",
     "@verdaccio/eslint-config": "^9.0.0",
     "@verdaccio/types": "^9.0.0",
+    "jest": "^24.9.0",
     "recursive-readdir": "2.2.2"
   },
   "scripts": {

plugins/aws-s3-storage/src/addTrailingSlash.ts

@@ -0,0 +1,3 @@
+export default (path?: string): string => {
+  return path != null ? (path.endsWith('/') ? path : `${path}/`) : '';
+};

plugins/aws-s3-storage/src/index.ts

@@ -14,6 +14,7 @@ import { S3 } from 'aws-sdk';
 import { S3Config } from './config';
 import S3PackageManager from './s3PackageManager';
 import { convertS3Error, is404Error } from './s3Errors';
+import addTrailingSlash from './addTrailingSlash';
 
 export default class S3Database implements IPluginStorage<S3Config> {
   public logger: Logger;
@@ -27,14 +28,14 @@ export default class S3Database implements IPluginStorage<S3Config> {
     if (!config) {
       throw new Error('s3 storage missing config. Add `store.s3-storage` to your config file');
     }
-    this.config = Object.assign({}, config.store['aws-s3-storage']);
+    this.config = Object.assign(config, config.store['aws-s3-storage']);
+
     if (!this.config.bucket) {
       throw new Error('s3 storage requires a bucket');
     }
     const configKeyPrefix = this.config.keyPrefix;
     this._localData = null;
-    this.config.keyPrefix =
-      configKeyPrefix != null ? (configKeyPrefix.endsWith('/') ? configKeyPrefix : `${configKeyPrefix}/`) : '';
+    this.config.keyPrefix = addTrailingSlash(configKeyPrefix);
 
     this.logger.debug({ config: JSON.stringify(this.config, null, 4) }, 's3: configuration: @{config}');
 

plugins/aws-s3-storage/src/s3PackageManager.ts

@@ -7,14 +7,16 @@ import { HttpError } from 'http-errors';
 import { is404Error, convertS3Error, create409Error } from './s3Errors';
 import { deleteKeyPrefix } from './deleteKeyPrefix';
 import { S3Config } from './config';
+import addTrailingSlash from './addTrailingSlash';
 
 const pkgFileName = 'package.json';
 
 export default class S3PackageManager implements ILocalPackageManager {
   public config: S3Config;
   public logger: Logger;
-  private packageName: string;
-  private s3: S3;
+  private readonly packageName: string;
+  private readonly s3: S3;
+  private readonly packagePath: string;
 
   public constructor(config: S3Config, packageName: string, logger: Logger) {
     this.config = config;
@@ -29,6 +31,15 @@ export default class S3PackageManager implements ILocalPackageManager {
     this.logger.trace({ s3ForcePathStyle }, 's3: [S3PackageManager constructor] s3ForcePathStyle @{s3ForcePathStyle}');
     this.logger.trace({ accessKeyId }, 's3: [S3PackageManager constructor] accessKeyId @{accessKeyId}');
     this.logger.trace({ secretAccessKey }, 's3: [S3PackageManager constructor] secretAccessKey @{secretAccessKey}');
+
+    const packageAccess = this.config.getMatchedPackagesSpec(packageName);
+    if (packageAccess) {
+      const storage = packageAccess.storage;
+      const packageCustomFolder = addTrailingSlash(storage);
+      this.packagePath = `${this.config.keyPrefix}${packageCustomFolder}${this.packageName}`;
+    } else {
+      this.packagePath = `${this.config.keyPrefix}${this.packageName}`;
+    }
   }
 
   public updatePackage(
@@ -69,7 +80,7 @@ export default class S3PackageManager implements ILocalPackageManager {
       this.s3.getObject(
         {
           Bucket: this.config.bucket,
-          Key: `${this.config.keyPrefix}${this.packageName}/${pkgFileName}`,
+          Key: `${this.packagePath}/${pkgFileName}`,
         },
         (err, response) => {
           if (err) {
@@ -101,7 +112,7 @@ export default class S3PackageManager implements ILocalPackageManager {
     this.s3.deleteObject(
       {
         Bucket: this.config.bucket,
-        Key: `${this.config.keyPrefix}${this.packageName}/${fileName}`,
+        Key: `${this.packagePath}/${fileName}`,
       },
       err => {
         if (err) {
@@ -118,7 +129,7 @@ export default class S3PackageManager implements ILocalPackageManager {
       this.s3,
       {
         Bucket: this.config.bucket,
-        Prefix: `${this.config.keyPrefix}${this.packageName}`,
+        Prefix: `${this.packagePath}`,
       },
       function(err) {
         if (err && is404Error(err as VerdaccioError)) {
@@ -139,7 +150,7 @@ export default class S3PackageManager implements ILocalPackageManager {
     this.s3.headObject(
       {
         Bucket: this.config.bucket,
-        Key: `${this.config.keyPrefix}${this.packageName}/${pkgFileName}`,
+        Key: `${this.packagePath}/${pkgFileName}`,
       },
       (err, data) => {
         if (err) {
@@ -172,7 +183,7 @@ export default class S3PackageManager implements ILocalPackageManager {
         // TODO: not sure whether save the object with spaces will increase storage size
         Body: JSON.stringify(value, null, '  '),
         Bucket: this.config.bucket,
-        Key: `${this.config.keyPrefix}${this.packageName}/${pkgFileName}`,
+        Key: `${this.packagePath}/${pkgFileName}`,
       },
       callback
     );
@@ -217,7 +228,7 @@ export default class S3PackageManager implements ILocalPackageManager {
 
     const baseS3Params = {
       Bucket: this.config.bucket,
-      Key: `${this.config.keyPrefix}${this.packageName}/${name}`,
+      Key: `${this.packagePath}/${name}`,
     };
 
     // NOTE: I'm using listObjectVersions so I don't have to download the full object with getObject.
@@ -226,7 +237,7 @@ export default class S3PackageManager implements ILocalPackageManager {
     this.s3.headObject(
       {
         Bucket: this.config.bucket,
-        Key: `${this.config.keyPrefix}${this.packageName}/${name}`,
+        Key: `${this.packagePath}/${name}`,
       },
       err => {
         if (err) {
@@ -334,7 +345,7 @@ export default class S3PackageManager implements ILocalPackageManager {
 
     const request = this.s3.getObject({
       Bucket: this.config.bucket,
-      Key: `${this.config.keyPrefix}${this.packageName}/${name}`,
+      Key: `${this.packagePath}/${name}`,
     });
 
     let headersSent = false;