Merge branch 'main' into add-minio-tests

.eslintrc

@@ -1,4 +1,5 @@
 {
+    "root": true,
     "env": {
         "browser": true,
         "commonjs": true,

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/project-automation.yml

@@ -5,6 +5,6 @@ on:
       - opened
 jobs:
   add_to_product_board:
-    uses: flowforge/.github/.github/workflows/project-automation.yml@main
+    uses: flowfuse/.github/.github/workflows/project-automation.yml@main
     secrets:
       token: ${{ secrets.PROJECT_ACCESS_TOKEN }}

.github/workflows/publish.yml

@@ -0,0 +1,46 @@
+name: Build and push packages
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  build:
+    uses: 'flowfuse/github-actions-workflows/.github/workflows/[email protected]'
+    with:
+      node: '[
+              {"version": "18", "tests": false, "lint": true},
+            ]'
+
+  publish:
+    needs: build
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    uses: 'flowfuse/github-actions-workflows/.github/workflows/[email protected]'
+    with:
+      package_name: file-server
+      publish_package: true
+    secrets:
+      npm_registry_token: ${{ secrets.NPM_PUBLISH_TOKEN }}
+
+  dispatch_container_build:
+    needs: publish
+    runs-on: ubuntu-latest
+    steps:
+      - name: Generate a token
+        id: generate_token
+        uses: tibdex/github-app-token@v2
+        with:
+          app_id: ${{ secrets.GH_BOT_APP_ID }}
+          private_key: ${{ secrets.GH_BOT_APP_KEY }}
+      - name: Trigger flie-server container image build
+        uses: benc-uk/workflow-dispatch@v1
+        with:
+          workflow: fileserver-container.yml
+          repo: flowfuse/helm
+          ref: main
+          token: ${{ steps.generate_token.outputs.token }}
+          inputs: '{"fileserver_ref": "${{ github.ref }}", "fileserver_release_name": "${{ needs.publish.outputs.release_name }}"}'

.github/workflows/release-build.yml

@@ -8,12 +8,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
-      - uses: actions/setup-node@v1
+        uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
         with:
           node-version: 16
-      - uses: JS-DevTools/npm-publish@v1
+      - uses: JS-DevTools/npm-publish@v3
         with:
           token: ${{ secrets.NPM_PUBLISH_TOKEN }}
           access: public
-

.gitignore

@@ -1,7 +1,6 @@
 var/
 /node_modules/
 docs/jsdoc
-package-lock.json
 etc/flowforge-storage.local.yml
 
 *.env

CHANGELOG.md

@@ -1,3 +1,117 @@
+#### 2.8.0: Release
+
+
+#### 2.7.1: Release
+
+ - Add locking in the app to avoid deadlock (#123) @knolleary
+ - Bump fast-xml-parser and @aws-sdk/client-s3 (#121) @app/dependabot
+
+#### 2.7.0: Release
+
+
+#### 2.6.1: Release
+
+ - Repackaging of 2.6.0
+
+#### 2.6.0: Release
+
+ - Bump braces from 3.0.2 to 3.0.3 (#113) @app/dependabot
+ - ci: Bump build_node_package workflow to 0.14.0 (#112) @ppawlowski
+
+#### 2.5.0: Release
+
+ - Bump actions/checkout from 3 to 4 (#107) @app/dependabot
+ - Bump actions/setup-node from 1 to 4 (#106) @app/dependabot
+ - Enable dependabot for github actions (#105) @ppawlowski
+
+#### 2.4.0: Release
+
+ - Bump tar from 6.1.15 to 6.2.1 (#103) @app/dependabot
+
+#### 2.3.0: Release
+
+
+#### 2.2.0: Release
+
+ - Bump jsonata and @node-red/util (#100) @app/dependabot
+ - Bump ip from 2.0.0 to 2.0.1 (#99) @app/dependabot
+
+#### 2.1.0: Release
+
+ - Feat: Add health check enpoint (#97) @ppawlowski
+
+#### 2.0.0: Release
+
+
+#### 1.15.0: Release
+
+ - Update npm-publish action version to v2 (#94) @ppawlowski
+ - Tidy up some more ff references (#93) @knolleary
+ - Update npm org name (#86) @knolleary
+
+#### 1.14.0: Release
+
+
+#### 1.13.0: Release
+
+ - Bump workflows to v0.1.1 version (#90) @ppawlowski
+ - Pin reusable workflows to v0.1.0 (#89) @ppawlowski
+ - Allow config to be passed via env for testing (#88) @knolleary
+ - Add /metrics endpoint (#87) @hardillb
+ - Fix run serve (#84) @hardillb
+ - Fix doule export (#83) @hardillb
+ - Allow JSON HTTP Request logging from the FileServer (#82) @hardillb
+ - Update ff references in package.json (#80) @knolleary
+ - Change repo references in workflows after github org rename (#78) @ppawlowski
+
+#### 1.12.0: Release
+
+ - Publish nightly package to npmjs (#77) @ppawlowski
+ - Bump the semver dep (#76) @hardillb
+ - Update dependencies to clear CVEs (#75) @hardillb
+ - Pin reusable workflow to commit SHA (#74) @ppawlowski
+ - Disable scheduled package build (#73) @ppawlowski
+
+#### 1.11.0: Release
+
+ - Bump nodemon to allow other deps to be updated (#71) @hardillb
+ - First pass at variable quotas (#70) @hardillb
+ - Move to SQLite from Memory context (#68) @hardillb
+ - Add file-server container build pipeline dispatcher (#69) @ppawlowski
+ - FIX: Publish package on schedule (#65) @ppawlowski
+ - FIX: Allow publish only when changes are pushed to `main` branch (#64) @ppawlowski
+ - Allow publish only when PR is merged (#63) @ppawlowski
+ - Update dependencies (#62) @knolleary
+ - Remove defunct docs tasks (#61) @knolleary
+ - Introduce build and publish workflow (#59) @ppawlowski
+
+#### 1.10.0: Release
+
+ - Chore: Set root flag in eslint (#57) @Pezmc
+
+#### 1.9.0: Release
+
+ - Add package-lock.json (#55) @Pezmc
+
+#### 1.8.0: Release
+
+
+#### 1.7.0: Release
+
+
+#### 1.6.0: Release
+
+
+#### 1.5.0: Release
+
+
+#### 1.4.0: Release
+
+ - Add APIs to support synchronous context (#48) @Steve-Mcl
+
+#### 1.3.0: Release
+
+
 #### 1.2.0: Release
 
  - S3 quota check (#42) @hardillb

README.md

@@ -1,8 +1,8 @@
-# A Basic Object Store for use with FlowForge
+# A Basic Object Store for use with FlowFuse
 
 ## Authorisation
 
-All requests should include a `Authorization` header with a Bearer token assigned by the FlowForge platform to identify
+All requests should include a `Authorization` header with a Bearer token assigned by the FlowFuse platform to identify
 ## End Points
 
 ### File Storage
@@ -109,12 +109,20 @@ driver:
   type: localfs
   options:
     root: var/root
+telemetry:
+  backend:
+    prometheus:
+      enabled: true
 ```
 
 - base_url - Where to reach the core FlowForge platform
 - driver
     - type - can be `s3`, `localfs` or `memory` (for testing)
     - options - will vary by driver
+- telemetry
+    - backend
+         - prometheus
+             - enabled - turns on the `/metrics` endpoint to track resource usage
 
 ### File Storage
 #### S3
@@ -157,17 +165,6 @@ options.
 
 ### Context Storage
 
-#### Redis
-
-This driver requires an instance of Redis with the RedisJSON enabled e.g. the `redislabs/rejson` docker container
-
-```yaml
-context:
-  type: redis
-  options:
-    urls: redis://localhost:6379
-```
-
 #### Sequelize
 
 This driver can use either PostgreSQL or SQLite to hold context values.
@@ -199,10 +196,6 @@ context:
 Where `context.options.storage` is the filename of the SQLite database, by default it will be written to 
 the `var` directory if a fully qualified path is not provided.
 
-#### Memory
-
-This driver is purely to make testing easier, it has no configuration options.
-
 ### Environment variables
 
 - FLOWFORGE_HOME default `/opt/flowforge-file-storage`

etc/flowforge-storage.yml

@@ -16,4 +16,11 @@ driver:
   #   forcePathStyle: true
   #   region: eu-west-1
 context:
-  type: memory
+  type: sequelize
+  options:
+    type: sqlite
+    storage: ff-context.db
+# telemetry:
+#   backend:
+#     prometheus:
+#       enabled: true

forge/auth.js

@@ -1,4 +1,3 @@
-
 const fp = require('fastify-plugin')
 const got = require('got')
 
@@ -9,7 +8,8 @@ module.exports = fp(async function (app, opts, done) {
     const client = got.extend({
         prefixUrl: `${app.config.base_url}/account/check/project`,
         headers: {
-            'user-agent': 'FlowForge Storage Server'
+            'user-agent': 'FlowForge Storage Server',
+            'ff-quota': true
         },
         timeout: {
             request: 500
@@ -22,22 +22,30 @@ module.exports = fp(async function (app, opts, done) {
                 authorization: `Bearer ${token}`
             }
         })
-        return !!project
+        return JSON.parse(project.body)
     }
 
     async function checkAuth (request, reply) {
+        if (request.url === '/metrics' || request.url === '/health') {
+            return
+        }
         try {
             const token = getAuthToken(request)
             const cacheOk = checkCache(token, request.params.projectId)
             if (!cacheOk) {
-                if (!await checkToken(request.params.projectId, token)) {
+                const tokenResponse = await checkToken(request.params.projectId, token)
+                if (!tokenResponse) {
                     throw new Error('Invalid token')
                 }
+                request.quota = tokenResponse.quota
                 // update cache
                 authCache[token] = {
                     ttl: Date.now(),
-                    projectId: request.params.projectId
+                    projectId: request.params.projectId,
+                    quota: tokenResponse.quota
                 }
+            } else {
+                request.quota = authCache[token].quota
             }
         } catch (error) {
             // always send 401 for security reasons