build(client, network): Browser streams/resends working correctly.

packages/client/package.json

@@ -43,7 +43,8 @@
     "eslint": "eslint --cache-location=node_modules/.cache/.eslintcache/ '*/**/*.{js,ts}'",
     "test": "jest --detectOpenHandles",
     "test-unit": "jest test/unit --verbose --useStderr --detectOpenHandles",
-    "test-types": "(cd test/exports && npm run link) && tsc --project --noEmit ./tsconfig.json",
+    "test-types": "npm run test-browser-types && (cd test/exports && npm run link && tsc --project --noEmit ./tsconfig.json)",
+    "test-browser-types": "tsc -b tsconfig.browser.json",
     "coverage": "jest --coverage",
     "test-integration": "jest --verbose --useStderr --forceExit test/integration/brubeck",
     "test-exports": "cd test/exports && npm run link && npm test",

packages/client/src/Resends.ts

@@ -5,7 +5,7 @@ import { DependencyContainer, inject, Lifecycle, scoped, delay } from 'tsyringe'
 import { SPID, SIDLike, MessageRef, StreamMessage } from 'streamr-client-protocol'
 import AbortController from 'node-abort-controller'
 import split2 from 'split2'
-import { Transform } from 'stream'
+import { Readable } from 'stream'
 
 import { instanceId, counterId } from './utils'
 import { Context, ContextError } from './utils/Context'
@@ -19,24 +19,33 @@ import Session from './Session'
 import NodeRegistry from './StorageNodeRegistry'
 import { StreamEndpoints } from './StreamEndpoints'
 import { BrubeckContainer } from './Container'
+import { ConvertBrowserStream } from './utils/ConvertBrowserStream'
 
 const MIN_SEQUENCE_NUMBER_VALUE = 0
 
 type QueryDict = Record<string, string | number | boolean | null | undefined>
 
-async function fetchStream(url: string, session: Session, opts = {}, abortController = new AbortController()) {
+async function fetchStream(url: string, session: Session, opts = {}, abortController = new AbortController()): Promise<Readable> {
     const startTime = Date.now()
     const response = await authRequest(url, session, {
         signal: abortController.signal,
         ...opts,
     })
+    if (!response.body) {
+        throw new Error('No Response Body')
+    }
+
     try {
-        const stream: Transform = response.body.pipe(split2((message: string) => {
+        const source: Readable = ConvertBrowserStream(response.body as unknown as (ReadableStream | Readable))
+
+        const stream = source.pipe(split2((message: string) => {
             return StreamMessage.deserialize(message)
         }))
+
         stream.once('close', () => {
             abortController.abort()
         })
+
         return Object.assign(stream, {
             startTime,
         })

packages/client/src/Rest.ts

@@ -8,6 +8,8 @@ import { instanceId } from './utils'
 import { ConnectionConfig, Config } from './Config'
 import authFetch, { authRequest } from './authFetch'
 import { Context } from './utils/Context'
+import { Readable } from 'stream'
+import { ConvertBrowserStream } from './utils/ConvertBrowserStream'
 
 import Session from './Session'
 import { BrubeckContainer } from './Container'
@@ -128,7 +130,7 @@ export class Rest implements Context {
         })
     }
 
-    async stream(urlParts: UrlParts, options: FetchOptions = {}, abortController = new AbortController()) {
+    async stream(urlParts: UrlParts, options: FetchOptions = {}, abortController = new AbortController()): Promise<Readable> {
         const startTime = Date.now()
         const response = await this.request(urlParts, {
             ...options,
@@ -138,10 +140,16 @@ export class Rest implements Context {
             }
         })
 
-        const stream = response.body
+        if (!response.body) {
+            throw new Error('No Response Body')
+        }
+
+        const stream = ConvertBrowserStream(response.body as unknown as (ReadableStream | Readable))
+
         stream.once('close', () => {
             abortController.abort()
         })
+
         return Object.assign(stream, {
             startTime,
         })

packages/client/src/encryption/BrowserPersistentStore.ts

@@ -1,5 +1,5 @@
 import { get, set, del, clear, keys, createStore } from 'idb-keyval'
-import { PersistentStore } from './GroupKeyStore'
+import { PersistentStore } from './PersistentStore'
 
 export default class BrowserPersistentStore implements PersistentStore<string, string> {
     readonly clientId: string

packages/client/src/encryption/GroupKeyStore.ts

@@ -1,19 +1,9 @@
 import { instanceId } from '../utils'
 import { Context } from '../utils/Context'
 import { GroupKey } from './Encryption'
-import ServerPersistentStore, { ServerPersistentStoreOptions } from './ServerPersistentStore'
+import { PersistentStore } from './PersistentStore'
 
-export interface PersistentStore<K, V> {
-    get(key: K): Promise<V | undefined>
-    set(key: K, value: V): Promise<boolean>
-    has(key: K): Promise<boolean>
-    delete(key: K): Promise<boolean>
-    clear(): Promise<boolean>
-    size(): Promise<number>
-    close(): Promise<void>
-    destroy(): Promise<void>
-    exists(): Promise<boolean>
-}
+import ServerPersistentStore, { ServerPersistentStoreOptions } from './ServerPersistentStore'
 
 type GroupKeyId = string
 

packages/client/src/encryption/PersistentStore.ts

@@ -0,0 +1,11 @@
+export interface PersistentStore<K, V> {
+    get(key: K): Promise<V | undefined>
+    set(key: K, value: V): Promise<boolean>
+    has(key: K): Promise<boolean>
+    delete(key: K): Promise<boolean>
+    clear(): Promise<boolean>
+    size(): Promise<number>
+    close(): Promise<void>
+    destroy(): Promise<void>
+    exists(): Promise<boolean>
+}

packages/client/src/encryption/ServerPersistentStore.ts

@@ -7,7 +7,7 @@ import sqlite3 from 'sqlite3'
 import { instanceId, pOnce } from '../utils'
 import { Context } from '../utils/Context'
 
-import { PersistentStore } from './GroupKeyStore'
+import { PersistentStore } from './PersistentStore'
 
 // eslint-disable-next-line promise/param-names
 const wait = (ms: number) => new Promise((resolveFn) => setTimeout(resolveFn, ms))

packages/client/src/encryption/WebCrypto.ts

@@ -0,0 +1,5 @@
+import { Crypto as WebCrypto } from 'node-webcrypto-ossl'
+
+export function getCryptoInstance(): Crypto {
+    return new WebCrypto()
+}

packages/client/src/shim/crypto.ts

@@ -0,0 +1,10 @@
+interface WebCrypto {
+    new (): Crypto
+    (): Crypto
+}
+
+const WebCryptoFn: WebCrypto = function WebCrypto() {
+    return window.crypto
+} as WebCrypto
+
+export { WebCryptoFn as Crypto }

packages/client/src/shim/node-fetch.ts

@@ -0,0 +1,7 @@
+type FetchResponse = Response
+// In browsers, the node-fetch package is replaced with this to use native fetch
+export default (
+    ((typeof fetch !== 'undefined' && fetch) || (typeof window !== 'undefined' && window.fetch) || undefined)!
+)
+
+export { FetchResponse as Response }

packages/client/src/utils/ConvertBrowserStream.ts

@@ -0,0 +1,36 @@
+import { PassThrough, Readable } from 'stream'
+
+export function ConvertBrowserStream(browserStream: ReadableStream | Readable) {
+    if ('pipe' in browserStream) {
+        return browserStream as Readable
+    }
+
+    const reader = browserStream.getReader()
+    const stream = new PassThrough()
+    // eslint-disable-next-line no-inner-declarations
+    async function pull() {
+        try {
+            // eslint-disable-next-line no-constant-condition
+            while (true) {
+                // eslint-disable-next-line no-await-in-loop
+                const { done, value } = await reader.read()
+                if (done) {
+                    return
+                }
+
+                if (!stream.writable) {
+                    return
+                }
+
+                stream.write(value)
+            }
+        } catch (err) {
+            stream.destroy(err)
+            reader.cancel()
+        } finally {
+            stream.end()
+        }
+    }
+    pull()
+    return stream
+}

packages/client/tsconfig.browser.json

@@ -0,0 +1,41 @@
+{
+    "extends": "@streamr/dev-config/ts/tsconfig.browser.json",
+    "compilerOptions": {
+        "composite": true,
+        "noEmit": true,
+        "declarationDir": "dist/types",
+        "outDir": "dist",
+        "experimentalDecorators": true,
+        "emitDecoratorMetadata": true,
+        "strictBindCallApply": true,
+        "baseUrl": ".",
+        "paths": {
+            "stream": ["readable-stream"],
+            "util": ["util"],
+            "http": ["./src/shim/http-https.js"],
+            "@ethersproject/wordlists": ["./node_modules/@ethersproject/wordlists/lib.esm/browser-wordlists.js"],
+            "https": ["./src/shim/http-https.js"],
+            "crypto": ["./node_modules/crypto-browserify"],
+            "buffer": ["./node_modules/buffer"],
+            "node-fetch": ["./src/shim/node-fetch.ts"],
+            "node-webcrypto-ossl": ["./src/shim/crypto.ts"],
+            "streamr-client-protocol/dist/contracts/NodeRegistry.json": ["./node_modules/streamr-client-protocol/dist/contracts/NodeRegistry.json"],
+            "streamr-client-protocol/*": ["./node_modules/streamr-client-protocol/src/*"],
+            "streamr-client-protocol": ["./node_modules/streamr-client-protocol/src/index.ts"],
+            "streamr-network": ["./node_modules/streamr-network/src/browser.ts"],
+            "./node_modules/streamr-network/src/connection/NodeWebRtcConnection.ts": ["./node_modules/streamr-network/src/connection/BrowserWebRtcConnection.ts"],
+            "./node_modules/streamr-network/src/connection/ws/NodeClientWsEndpoint.ts": ["./node_modules/streamr-network/src/connection/ws/BrowserClientWsEndpoint.ts"],
+            "./node_modules/streamr-network/src/connection/ws/NodeClientWsConnection.ts": ["./node_modules/streamr-network/src/connection/ws/BrowserClientWsConnection.ts"],
+            "./node_modules/streamr-network/src/helpers/logger/LoggerNode.ts": ["./node_modules/streamr-network/src/helpers/logger/LoggerBrowser.ts"]
+        }
+    },
+    "include": [
+        "package.json",
+        "src/**/*",
+        "vendor/**/*",
+        "contracts/**/*.json"
+    ],
+    "references": [
+        { "path": "node_modules/streamr-client-protocol/tsconfig.node.json" }
+    ]
+}

packages/client/tsconfig.json

@@ -5,7 +5,18 @@
         "experimentalDecorators": true,
         "emitDecoratorMetadata": true,
         "strictBindCallApply": true,
-        "outDir": "dist"
+        "outDir": "dist",
+        "lib": [
+            "ES5",
+            "ES2015",
+            "ES2016",
+            "ES2017",
+            "ES2018",
+            "ES2019",
+            "ES2020",
+            "ESNext",
+            "DOM"
+        ]
     },
     "include": [
         "package.json",
@@ -14,7 +25,12 @@
         "contracts/**/*.json",
         "test/**/*"
     ],
-    "exclude": ["test/legacy/*", "test/memory/*", "test/exports/*"],
+    "exclude": [
+        "test/legacy/*",
+        "test/memory/*",
+        "test/exports/*",
+        "src/shim/*"
+    ],
     "references": [
         { "path": "./tsconfig.node.json" }
     ]

packages/client/tsconfig.node.json

@@ -26,7 +26,7 @@
         "contracts/**/*.json"
     ],
     "exclude": [
-        "src/**/WebCryptoBrowser.ts"
+        "src/shim/*"
     ],
     "references": [
         { "path": "node_modules/streamr-network/tsconfig.node.json" },

packages/client/webpack.config.js

@@ -109,19 +109,15 @@ module.exports = (env, argv) => {
             alias: {
                 stream: 'readable-stream',
                 util: 'util',
-                http: path.resolve(__dirname, './src/shim/http-https.js'),
+                http: path.resolve(__dirname, './src/shim/http-https.ts'),
                 '@ethersproject/wordlists': path.resolve(__dirname, 'node_modules', '@ethersproject/wordlists/lib.esm/browser-wordlists.js'),
-                https: path.resolve(__dirname, './src/shim/http-https.js'),
+                https: path.resolve(__dirname, './src/shim/http-https.ts'),
                 crypto: path.resolve(__dirname, 'node_modules', 'crypto-browserify'),
                 buffer: path.resolve(__dirname, 'node_modules', 'buffer'),
-                'node-fetch': path.resolve(__dirname, './src/shim/node-fetch.js'),
-                'node-webcrypto-ossl': path.resolve(__dirname, 'src/shim/crypto.js'),
+                'node-fetch': path.resolve(__dirname, './src/shim/node-fetch.ts'),
+                'node-webcrypto-ossl': path.resolve(__dirname, 'src/shim/crypto.ts'),
                 'streamr-client-protocol/dist/contracts/NodeRegistry.json': path.resolve(__dirname, 'node_modules/streamr-client-protocol/dist/contracts/NodeRegistry.json'),
                 'streamr-client-protocol': path.resolve(__dirname, 'node_modules/streamr-client-protocol/dist/src'),
-                //'streamr-network': path.resolve(__dirname, 'node_modules/streamr-network/src/browser.ts'),
-                //[path.resolve(__dirname, 'node_modules/streamr-network/src/connection/NodeWebRtcConnection.ts')]: path.resolve(__dirname, 'node_modules/streamr-network/src/connection/BrowserWebRtcConnection.ts'),
-                //[path.resolve(__dirname, 'node_modules/streamr-network/src/connection/ws/NodeClientWsEndpoint.ts')]: path.resolve(__dirname, 'node_modules/streamr-network/src/connection/ws/BrowserClientWsEndpoint.ts'),
-                //[path.resolve(__dirname, 'node_modules/streamr-network/src/connection/ws/NodeClientWsConnection.ts')]: path.resolve(__dirname, 'node_modules/streamr-network/src/connection/ws/BrowserClientWsConnection.ts'),
                 'streamr-network': path.join(__dirname, '../network/src/browser.ts'),
                 [path.join(__dirname, '../network/src/connection/NodeWebRtcConnection.ts$')]: path.resolve(__dirname, 'node_modules/streamr-network/src/connection/BrowserWebRtcConnection.ts'),
                 [path.join(__dirname, '../network/src/connection/ws/NodeClientWsEndpoint.ts$')]: path.resolve(__dirname, 'node_modules/streamr-network/src/connection/ws/BrowserClientWsEndpoint.ts'),

packages/network/package.json

@@ -13,7 +13,7 @@
     "subscriber": "bin/subscriber.js",
     "tracker": "bin/tracker.js"
   },
-  "main": "dist/streamr-network.js",
+  "main": "dist/src/composition.js",
   "browser": "dist/streamr-network.js",
   "types": "dist/src/composition.d.ts",
   "scripts": {

packages/network/src/browser.ts

@@ -6,4 +6,3 @@ export { MetricsContext, Metrics } from './helpers/MetricsContext'
 export { Location, AbstractNodeOptions } from './identifiers'
 export { createNetworkNode, NetworkNodeOptions } from './createNetworkNode'
 export { NetworkNode } from './logic/node/NetworkNode'
-export { Tracker } from './logic/tracker/Tracker'