Fix JS tests and add them to GA (#217)

* wip

* remove plugin transform

* change package type to module

* remove test transforms - test files have correct format now.

* test files compile; but errors in the source files

* [tests] add jsom test-environments to make tests work

* [tests] skipp substratee tests

* [tests] skip cantillon tests as they are unsupported, but they compile know

* [tests] fix all tests except for createProofOfAttendance

* [README] add note on how to run the tests from the IDE.

* remove babel-jest dependency.

* make build work; successfully launch api in the app.

* remove obsolete bs58 dependency

* [encointer.test] fix last test

* [encointer.test] fix incorrect handling of cid: use cid_hex instead of cid_bs58

* [tests] extract testSetup function

* cleanup

* [GA] add js_ci

* fix gesell connection

* [tests] most gesell tests pass

* [account] fix global settings access

* [tests] gesell: updated estimated tx-fee

* [test] fix all gesell tests.

* [consts] remove obsolete pallet overrides

* [tests] run gesell e2e tests seperatly

* remove @polkadot/dev dep. It is no longer used.

* [tests] also add separate tests to run Cantillon e2e tests

* minor cleanup

* cleanup

* Revert "[consts] remove obsolete pallet overrides"

This reverts commit f7817cb

.github/workflows/js_ci.yml

@@ -0,0 +1,76 @@
+name: JS-CI
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  integration-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      # Setup Environment
+      - uses: actions/checkout@v2
+
+      - uses: monorepo-actions/config-for-actions@main
+        id: config
+        with:
+          config_files: ./.github/action-config.json
+
+      # JS Stuff
+      - uses: actions/setup-node@v2
+        with:
+          node-version: ${{ steps.config.outputs.node-version }}
+
+      - name: Download encointer-node
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          github_token: ${{secrets.GITHUB_TOKEN}}
+          workflow: build-and-test.yml
+          name: executables
+          # in fact this action should download the latest artifact, but sometimes fails. Then we need to
+          # set the `run_id` to force a download of an updated binary.
+          run_id: 1282695776
+          path: node
+          repo: encointer/encointer-node
+
+      - name: Run encointer-node
+        run: |
+          ls -alt node
+          chmod +x node/encointer-node-notee
+          ./node/encointer-node-notee --tmp --dev --enable-offchain-indexing true --rpc-methods unsafe &
+
+      - name: Test JS
+        # We don't really have unit tests in the JS part. Most tests need a local node running.
+        run: |
+          cd lib/js_service_encointer
+          yarn install
+          yarn test
+
+  e2e-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    strategy:
+      matrix:
+        network: [gesell] # Todo: add cantillon once we support it again.
+    steps:
+      # Setup Environment
+      - uses: actions/checkout@v2
+
+      - uses: monorepo-actions/config-for-actions@main
+        id: config
+        with:
+          config_files: ./.github/action-config.json
+
+      # JS Stuff
+      - uses: actions/setup-node@v2
+        with:
+          node-version: ${{ steps.config.outputs.node-version }}
+
+      - name: Run e2e tests against  ${{ matrix.network }}
+        run: |
+          cd lib/js_service_encointer
+          yarn install
+          yarn test:${{ matrix.network }}

lib/js_service_encointer/README.md

@@ -22,3 +22,10 @@ graph of all modules and files (also `.css` and images) and bundles that all tog
 ready to be served to the browser. It ensures that the necessary polyfills are included and only compiles code
 from the modules that is in fact imported somewhere - in other words, it builds the bare minimum of the code.
 
+## Jest
+To be able to run the tests from Webstorm, we must add the following line the to the run/debug config in the node 
+options.
+
+```
+--experimental-vm-modules
+```

lib/js_service_encointer/babel.config.js

@@ -1,8 +1,13 @@
 module.exports = {
-  presets: [['@babel/env', { modules: false }]],
-  plugins: [
-    ['@babel/plugin-transform-runtime', {
-      regenerator: true
-    }]
-  ]
+  env: {
+    production: {
+      presets: [['@babel/env', { modules: false }]],
+      plugins: [
+        ['@babel/plugin-transform-runtime', {
+          regenerator: true
+        }]
+      ]
+    },
+    test: {}
+  },
 };

lib/js_service_encointer/jest.config-cantillon-e2e.js

@@ -0,0 +1,6 @@
+import config from './jest.config.js';
+
+export default {
+  ...config,
+  testMatch: ['**/?(*.)+(cantillon.test-e2e).[jt]s?(x)'],
+};

lib/js_service_encointer/jest.config-gesell-e2e.js

@@ -0,0 +1,6 @@
+import config from './jest.config.js';
+
+export default {
+  ...config,
+  testMatch: ['**/?(*.)+(gesell.test-e2e).[jt]s?(x)'],
+};

lib/js_service_encointer/jest.config.js

@@ -1,21 +1,13 @@
-const { defaults } = require('jest-config');
+import { defaults } from 'jest-config';
 
-module.exports = {
+export default {
   moduleFileExtensions: [...defaults.moduleFileExtensions, 'ts', 'tsx'],
   testMatch: ['**/__tests__/**/*.[jt]s?(x)', '**/?(*.)+(spec|test).[jt]s?(x)'],
-  moduleNameMapper: {
-    '@encointer/node-api(.*)$': '<rootDir>/../../../encointer-js/packages/node-api/src/$1',
-    '@encointer/util(.*)$': '<rootDir>../../../encointer-js/packages/util/src/$1',
-    '@encointer/types(.*)$': '<rootDir>../../../encointer-js/packages/types/src/$1',
-    '@encointer/worker-api(.*)$': '<rootDir>/../../../encointer-js/packages/worker-api/src/$1'
-  },
+  moduleNameMapper: {},
   modulePathIgnorePatterns: [
     '<rootDir>/dist'
   ],
-  transform: {
-    '^.+\\.(js|jsx|ts|tsx)$': require.resolve('babel-jest')
-  },
-  // transform esm `@polkadot`, `@encointer` and `@babels` esm modules such that jest understands them.
-  transformIgnorePatterns: ['/node_modules/(?!@polkadot|@encointer|@babel/runtime/helpers/esm/)'],
+  transform: {},
+  transformIgnorePatterns: [],
   verbose: true,
 };

lib/js_service_encointer/package.json

@@ -3,13 +3,16 @@
   "version": "1.0.0",
   "main": "index.js",
   "license": "MIT",
+  "type": "module",
   "scripts": {
     "clean": "rm -rf dist/*",
     "lint": "eslint src/**.js",
     "build": "cross-env NODE_ENV=production ./scripts/buildWebpack.cjs",
     "encointer:link": "./scripts/encointer-link.cjs",
     "encointer:unlink": "./scripts/encointer-unlink.cjs",
-    "test": "polkadot-dev-run-test"
+    "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js",
+    "test:gesell": "node --experimental-vm-modules node_modules/jest/bin/jest.js --config jest.config-gesell-e2e.js",
+    "test:cantillon": "node --experimental-vm-modules node_modules/jest/bin/jest.js --config jest.config-cantillon-e2e.js"
   },
   "exports": {
     ".": "./src/index.js",
@@ -34,7 +37,6 @@
     "@polkadot/util-crypto": "^7.4.1",
     "Buffer": "^0.0.0",
     "bn.js": "^4.12.0",
-    "bs58": "^4.0.1",
     "buffer": "^6.0.3",
     "core-js": "^3.12.1",
     "node-rsa": "^1.1.1",
@@ -49,10 +51,8 @@
     "@babel/register": "^7.15.3",
     "@babel/runtime": "^7.15.4",
     "@jest/globals": "^27.2.3",
-    "@polkadot/dev": "^0.62.11",
     "@webpack-cli/info": "^1.3.0",
     "@webpack-cli/init": "^1.1.3",
-    "babel-jest": "^27.2.3",
     "babel-loader": "^8.2.2",
     "browserslist": "^4.17.3",
     "constants-browserify": "^1.0.0",

lib/js_service_encointer/src/index.js

@@ -1,10 +1,10 @@
-import 'core-js/stable';
-import 'regenerator-runtime/runtime';
+import 'core-js/stable/index.js';
+import 'regenerator-runtime/runtime.js';
 
-import account from './service/account';
-import encointer from './service/encointer';
-import settings from './service/settings';
-import chain from './service/chain';
+import account from './service/account.js';
+import encointer from './service/encointer.js';
+import settings from './service/settings.js';
+import chain from './service/chain.js';
 
 // send message to JSChannel: PolkaWallet
 function send (path, data) {

lib/js_service_encointer/src/service/account.js

@@ -1,22 +1,22 @@
 import { cryptoWaitReady, keyExtractSuri, mnemonicGenerate } from '@polkadot/util-crypto';
 import { hexToU8a, u8aToHex, hexToString, assert, u8aToBuffer, bufferToU8a, compactAddLength } from '@polkadot/util';
-import { ss58Decode } from 'oo7-substrate/src/ss58';
+import { ss58Decode } from 'oo7-substrate/src/ss58.js';
 import { polkadotIcon } from '@polkadot/ui-shared';
 import BN from 'bn.js';
 import {
   parseQrCode,
   getSigner,
   makeTx,
   getSubmittable
-} from '../utils/QrSigner';
-
+} from '../utils/QrSigner.js';
 import { Keyring } from '@polkadot/keyring';
 import { createType, i128 } from '@polkadot/types';
 import { encodeFloatToI64F64, parseI64F64, toI64F64 } from '@encointer/util';
-import { TrustedCallMap } from '../config/trustedCall';
+import { TrustedCallMap } from '../config/trustedCall.js';
 import { bs58 } from '@polkadot/util-crypto/base58/bs58';
-import { callWorker, encointerBalances, substrateeRegistry, transfer } from '../config/consts';
-import { unsubscribe } from '../utils/unsubscribe';
+import { callWorker, encointerBalances, substrateeRegistry, transfer } from '../config/consts.js';
+import { unsubscribe } from '../utils/unsubscribe.js';
+import settings from './settings.js';
 
 export const keyring = new Keyring({ ss58Format: 0, type: 'sr25519' });
 
@@ -271,7 +271,7 @@ export async function txFeeEstimate (txInfo, paramList) {
   }
 
   let dispatchInfo;
-  if (window.settings.connectedToTeeProxy() && TrustedCallMap[txInfo.module][txInfo.call] !== null) {
+  if (settings.connectedToTeeProxy() && TrustedCallMap[txInfo.module][txInfo.call] !== null) {
     if (isTcIsRegisterAttestations(txInfo)) {
       // todo: in a meetup with more than 3 people this is longer. calculate actual size based on message length.
       dispatchInfo = dispatchInfo = api.tx[substrateeRegistry][callWorker](new Array(768))
@@ -345,7 +345,7 @@ export function sendTx (txInfo, paramList) {
     });
   }
 
-  if (window.settings.connectedToTeeProxy() && TrustedCallMap[txInfo.module][txInfo.call] !== null) {
+  if (settings.connectedToTeeProxy() && TrustedCallMap[txInfo.module][txInfo.call] !== null) {
     return _sendTrustedTx(keyPair, txInfo, paramList);
   }
 

lib/js_service_encointer/src/service/chain.js

@@ -1,4 +1,4 @@
-import { unsubscribe } from '../utils/unsubscribe';
+import { unsubscribe } from '../utils/unsubscribe.js';
 
 /**
  * Mainly debug method introduced to test subscriptions. Subscribes to the timestamp of the last block

lib/js_service_encointer/src/service/encointer.js

@@ -4,9 +4,9 @@ import { cryptoWaitReady, encodeAddress } from '@polkadot/util-crypto';
 import { createType } from '@polkadot/types';
 
 import { parseI64F64 } from '@encointer/util';
-import { keyring } from './account';
-import { pallets } from '../config/consts';
-import { unsubscribe } from '../utils/unsubscribe';
+import { keyring } from './account.js';
+import { pallets } from '../config/consts.js';
+import { unsubscribe } from '../utils/unsubscribe.js';
 
 const divisor = new BN('1'.padEnd(18 + 1, '0'));
 

lib/js_service_encointer/src/service/settings.js

@@ -3,14 +3,14 @@ import { ApiPromise } from '@polkadot/api';
 import { EncointerWorker } from '@encointer/worker-api';
 import { options } from '@encointer/node-api';
 import WS from 'websocket';
-import { pallets } from '../config/consts';
-import { keyring } from './account';
+import { pallets } from '../config/consts.js';
+import { keyring } from './account.js';
 
 const { w3cwebsocket: WebSocket } = WS;
 
 async function connect (endpoint, configOverride) {
   return new Promise(async (resolve, reject) => {
-    console.log(`configOverride: ${configOverride.toJSON()}`);
+    console.log(`configOverride: ${JSON.stringify(configOverride)}`);
     const provider = new WsProvider(endpoint);
     try {
       const api = await ApiPromise.create({

lib/js_service_encointer/test/cantillon.test.js → lib/js_service_encointer/test/cantillon.test-e2e.js

@@ -1,52 +1,42 @@
+/**
+ * @jest-environment jsdom
+ */
+
 'use strict';
 
 import { getCommunityIdentifiers } from '../src/service/encointer';
 import account, { _sendTrustedTx } from '../src/service/account';
 import { bufferToU8a, compactAddLength, hexToU8a, u8aToBuffer, u8aToHex } from '@polkadot/util';
-import { ApiPromise, Keyring } from '@polkadot/api';
 import { cryptoWaitReady } from '@polkadot/util-crypto';
 import { EncointerWorker } from '@encointer/worker-api';
-import * as bs58 from 'bs58';
+import { bs58 } from '@polkadot/util-crypto/base58/bs58';
 import { createType } from '@polkadot/types';
 import { getTrustedCall } from './testUtils/helpers';
 import { beforeAll, describe, it, jest } from '@jest/globals';
 import { cantillonNetwork, localDockerNetwork } from './testUtils/networks';
 import WS from 'websocket';
-import { WsProvider } from '@polkadot/rpc-provider';
-import { options } from '@encointer/node-api';
-import { pallets } from '../src/config/consts';
+import { testSetup } from './testUtils/testSetup';
+import { Keyring } from '@polkadot/api';
 
 const { w3cwebsocket: WebSocket } = WS;
 
+// Todo: Cantillon does not run currently. Re-activate them once Cantillon runs.
 describe('cantillon', () => {
-  const network = localDockerNetwork();
+  const network = cantillonNetwork();
   let keyring;
   let registry;
   let worker;
 
   beforeAll(async () => {
-    jest.setTimeout(9000);
-    window.send = (_, data) => console.log(data);
+    jest.setTimeout(90000);
+
+    const setup = await testSetup(network);
     keyring = new Keyring({ type: 'sr25519' });
-    const provider = new WsProvider(network.chain);
-    try {
-      window.api = await ApiPromise.create({
-        ...options({
-          types: network.customTypes
-        }),
-        provider: provider
-      });
-      Object.assign(pallets, network.palletOverrides);
-      send('log', `${network.chain} wss connected success`);
-    } catch (err) {
-      send('log', `connect ${network.chain} failed`);
-      await provider.disconnect();
-    }
-
-    registry = window.api.registry;
+
+    registry = setup.api.registry;
     worker = new EncointerWorker(network.worker, {
       keyring: keyring,
-      api: api,
+      api: setup.api,
       createWebSocket: (url) => new WebSocket(url)
     });
     window.workerShieldingKey = await worker.getShieldingKey();