Backport important fixes (#2592)

* optimize the upload procedure (#2589)

- closes #2538

* fixed problem that cb is never called on mh enable/disable (#2557)

* fixed problem that cb is never called on mh enable/disable

* fix type

* added enhanced adapter license info to schema and types (#2591)

* added license info

* fix update license script

* format

* deprecate common.license and introduce common.licenseInformation

* rm local schema from controller iopack

* bring back license for adapter tests

* WIP placeholder added

* prevent issue when values with stateNs.null exist in the database (#2580)

* prevent issue when values with stateNs.null exist in the database

- closes #2579

* rm log

* rm anoither log

* allow path aliases in adapter package

* rm types from validator again

* move tsc-alias to postbuild

* use lerna to build project again as npm workspaces does not use the correct build script when building dependency packages

- and using normal workspace flag works but is ineffective and can lead to errors due to still open npm/rfcs#548

* update jsonl

* ts back to v4, accidentaly chosen by resolving merge conflict

* build local

* clean install to try to solve install module not found

* another pack-lock update

* fix linter

CHANGELOG.md

@@ -4,7 +4,7 @@
 	## __WORK IN PROGRESS__
 -->
 
-## 5.0.17 (2023-11-29) - Jana
+## __WORK IN PROGRESS__ - Jana
 **BREAKING CHANGES**
 * Support for Node.js 12 and 14 is dropped! Supported are Node.js 16.4.0+ and 18.x
 * Backups created with the new js-controller version cannot be restored on hosts with lower js-controller version!

package.json

@@ -50,14 +50,14 @@
     "@types/triple-beam": "^1.3.2",
     "@types/winston-syslog": "^2.4.0",
     "@types/yargs": "^17.0.8",
-    "@typescript-eslint/eslint-plugin": "^6.2.1",
-    "@typescript-eslint/parser": "^6.2.1",
+    "@typescript-eslint/eslint-plugin": "^6.18.1",
+    "@typescript-eslint/parser": "^6.18.1",
     "chai": "^4.3.4",
     "chai-as-promised": "^7.1.1",
     "cpy-cli": "^4.2.0",
-    "eslint": "^8.48.0",
-    "eslint-config-prettier": "^8.10.0",
-    "eslint-plugin-prettier": "^5.0.0",
+    "eslint": "^8.56.0",
+    "eslint-config-prettier": "^9.1.0",
+    "eslint-plugin-prettier": "^5.1.3",
     "fs-extra": "^11.1.0",
     "get-spdx-license-ids": "^2.1.0",
     "gulp": "^4.0.2",
@@ -71,19 +71,19 @@
     "sinon-chai": "^3.7.0",
     "ts-json-schema-generator": "~1.2.0",
     "ts-node": "^10.9.1",
-    "typedoc": "^0.24.8",
-    "typedoc-plugin-markdown": "^3.15.3",
-    "typedoc-plugin-missing-exports": "^2.0.0",
+    "tsc-alias": "^1.8.8",
+    "typedoc": "^0.25.4",
+    "typedoc-plugin-markdown": "^3.17.1",
+    "typedoc-plugin-missing-exports": "^2.1.0",
     "typescript": "~4.9.5"
   },
   "scripts": {
     "clean": "rimraf packages/*/build packages/*/*.tsbuildinfo",
     "build:doc": "lerna run build:doc",
-    "build:ts": "npm run build --workspace=iobroker.js-controller -- --verbose",
+    "build:ts": "lerna run build --ignore '@iobroker/types'",
     "build:types": "npm run build --workspace=@iobroker/types",
-    "watch:ts": "npm run build:ts -- --watch",
     "build": "npm run build:ts && npm run build:types",
-    "postbuild": "lerna run postbuild && npm run update-schema",
+    "postbuild": "npm run update-schema",
     "preinstall": "lerna run preinstall",
     "install": "lerna run install",
     "release": "release-script",

packages/adapter/package.json

@@ -39,7 +39,7 @@
     "url": "https://github.com/ioBroker/ioBroker.js-controller/packages/adapter"
   },
   "scripts": {
-    "build": "tsc -b tsconfig.build.json",
+    "build": "tsc -b tsconfig.build.json && tsc-alias",
     "postbuild": "cpy ./cert ../build/ --cwd=src",
     "build:doc": "typedoc --excludeExternals --excludeProtected --excludePrivate --plugin typedoc-plugin-missing-exports --plugin typedoc-plugin-markdown src/lib/adapter/adapter.ts --out ../controller/doc"
   },

packages/adapter/src/lib/adapter/validator.ts

@@ -1,4 +1,4 @@
-import { MAX_TIMEOUT, SYSTEM_ADMIN_USER } from './constants';
+import { MAX_TIMEOUT, SYSTEM_ADMIN_USER } from '@/lib/adapter/constants';
 import { tools, EXIT_CODES } from '@iobroker/js-controller-common';
 
 type Callback = (...args: any[]) => void | Promise<void>;
@@ -61,7 +61,7 @@ export class Validator {
                 return;
             }
 
-            const obj: any = await this.objects.getObjectAsync(id);
+            const obj = await this.objects.getObjectAsync(id);
             // at first check object existence
             if (!obj) {
                 this.log.warn(
@@ -71,7 +71,7 @@ export class Validator {
             }
 
             // for a state object, we require common.type to exist
-            if (obj.common && obj.common.type) {
+            if (obj.common?.type) {
                 // check if we are allowed to write (read-only can only be written with ack: true)
                 if (!state.ack && obj.common.write === false) {
                     this.log.warn(

packages/adapter/tsconfig.json

@@ -2,7 +2,12 @@
   "extends": "../../tsconfig.json",
   "compilerOptions": {
     "outDir": "build",
-    "rootDir": "src"
+    "rootDir": "src",
+    "paths": {
+      "@/*": [
+        "./src/*"
+      ]
+    }
   },
   "references": [
     {

packages/cli/src/lib/setup.ts

@@ -2488,11 +2488,11 @@ async function processCommand(
                         null,
                         (err, res) => {
                             if (!err && res?.rows.length) {
-                                for (let i = 0; i < res.rows.length; i++) {
-                                    const parts = res.rows[i]!.id.split('.');
+                                for (const row of res.rows) {
+                                    const parts = row.id.split('.');
                                     // ignore system.host.name.alive and so on
                                     if (parts.length === 3) {
-                                        states.pushMessage(res.rows[i]!.id, {
+                                        states.pushMessage(row.id, {
                                             command: 'checkLogging',
                                             message: null,
                                             from: 'console'
@@ -2641,7 +2641,8 @@ async function processCommand(
                     });
 
                     if (cmd === 's' || cmd === 'status') {
-                        mh.status(() => void callback(EXIT_CODES.CANNOT_CREATE_USER_OR_GROUP));
+                        mh.status();
+                        return void callback();
                     } else if (cmd === 'b' || cmd === 'browse') {
                         mh.browse((err: any, list: any) => {
                             if (err) {
@@ -2653,39 +2654,33 @@ async function processCommand(
                             }
                         });
                     } else if (cmd === 'e' || cmd === 'enable') {
-                        mh.enable(true, (err: any) => {
+                        mh.enable(true, async (err: any) => {
                             if (err) {
                                 console.error(err);
                                 return void callback(EXIT_CODES.CANNOT_ENABLE_MULTIHOST);
                             } else {
-                                states.pushMessage(
-                                    `system.host.${tools.getHostName()}`,
-                                    {
-                                        command: 'updateMultihost',
-                                        message: null,
-                                        from: 'setup'
-                                    },
-                                    // @ts-expect-error todo formally we should only call processExit with an exit code
-                                    callback
-                                );
+                                await states.pushMessage(`system.host.${tools.getHostName()}`, {
+                                    command: 'updateMultihost',
+                                    message: null,
+                                    from: 'setup'
+                                });
+
+                                callback();
                             }
                         });
                     } else if (cmd === 'd' || cmd === 'disable') {
-                        mh.enable(false, (err: any) => {
+                        mh.enable(false, async (err: any) => {
                             if (err) {
                                 console.error(err);
                                 return void callback(EXIT_CODES.CANNOT_ENABLE_MULTIHOST);
                             } else {
-                                states.pushMessage(
-                                    `system.host.${tools.getHostName()}`,
-                                    {
-                                        command: 'updateMultihost',
-                                        message: null,
-                                        from: 'setup'
-                                    },
-                                    // @ts-expect-error todo formally we should only call processExit with an exit code
-                                    callback
-                                );
+                                await states.pushMessage(`system.host.${tools.getHostName()}`, {
+                                    command: 'updateMultihost',
+                                    message: null,
+                                    from: 'setup'
+                                });
+
+                                callback();
                             }
                         });
                     } else if (cmd === 'c' || cmd === 'connect') {

packages/cli/src/lib/setup/multihostClient.ts

@@ -15,10 +15,10 @@ interface ReceivedMessage {
     slave?: boolean;
     auth?: string;
     salt?: string;
-    /** The states config of iobroker.json  */
-    states?: Record<string, any>;
-    /** The objects config of iobroker.json */
-    objects?: Record<string, any>;
+    /** The states config of ioBroker.json  */
+    states?: ioBroker.StatesDatabaseOptions;
+    /** The objects config of ioBroker.json */
+    objects?: ioBroker.ObjectsDatabaseOptions;
 }
 
 export type BrowseResultEntry = Partial<ReceivedMessage>;
@@ -192,8 +192,8 @@ export class MHClient {
         password: string,
         callback: (
             err: Error | undefined,
-            objectsConfig?: Record<string, any>,
-            statesConfig?: Record<string, any>,
+            objectsConfig?: ioBroker.ObjectsDatabaseOptions,
+            statesConfig?: ioBroker.StatesDatabaseOptions,
             address?: string
         ) => void
     ): void {

packages/cli/src/lib/setup/setupInstall.ts

@@ -894,8 +894,8 @@ export class Install {
             options.enabled === true || options.enabled === false
                 ? options.enabled
                 : instanceObj.common.enabled === true || instanceObj.common.enabled === false
-                ? instanceObj.common.enabled
-                : false;
+                  ? instanceObj.common.enabled
+                  : false;
 
         // @ts-expect-error we now convert the adapter object to an instance object
         instanceObj.common.host = options.host;

packages/cli/src/lib/setup/setupMultihost.ts

@@ -32,7 +32,7 @@ export class Multihost {
     /**
      * Retrive config (iobroker.json content)
      */
-    getConfig(): Record<string, any> {
+    getConfig(): ioBroker.IoBrokerJson {
         let config;
         // read actual configuration
         try {
@@ -92,9 +92,8 @@ export class Multihost {
      *
      * @param config iob config
      * @param changed if config has changed
-     * @param callback
      */
-    private showMHState(config: Record<string, any>, changed: boolean, callback: () => void): void {
+    private showMHState(config: ioBroker.IoBrokerJson, changed: boolean): void {
         if (config.multihostService.enabled) {
             let warningShown = false;
             if (dbTools.isLocalObjectsDbServer(config.objects.type, config.objects.host, true)) {
@@ -144,7 +143,6 @@ export class Multihost {
         );
         console.log(`Objects:                    ${config.objects.type} on ${config.objects.host}`);
         console.log(`States:                     ${config.states.type} on ${config.states.host}`);
-        callback();
     }
 
     /**
@@ -217,7 +215,7 @@ export class Multihost {
                 prompt.start();
 
                 prompt.get(schema, (err, password) => {
-                    if (password && password.password) {
+                    if (password?.password) {
                         if (password.password !== password.passwordRepeat) {
                             callback(new Error('Secret phrases are not equal!'));
                         } else {
@@ -226,29 +224,31 @@ export class Multihost {
                                     obj!.native.secret,
                                     password.password as string
                                 );
-                                this.showMHState(config, changed, callback);
+                                this.showMHState(config, changed);
+                                callback();
                             });
                         }
                     } else {
                         callback(new Error('No secret phrase entered!'));
                     }
                 });
             } else {
-                this.showMHState(config, changed, callback);
+                this.showMHState(config, changed);
+                callback();
             }
         } else {
-            this.showMHState(config, changed, callback);
+            this.showMHState(config, changed);
+            callback();
         }
     }
 
     /**
      * Show the MH status
-     * @param callback
      */
-    status(callback: () => void): void {
+    status(): void {
         const config = this.getConfig();
         config.multihostService = config.multihostService || { enabled: false, secure: true };
-        this.showMHState(config, false, callback);
+        this.showMHState(config, false);
     }
 
     /**
@@ -314,11 +314,11 @@ export class Multihost {
                 } else {
                     if (tools.isListenAllAddress(config.states.host)) {
                         // TODO: why we set the remote IP only when the local config allows full connectivity?
-                        config.states.host = ipHost;
+                        config.states.host = ipHost ?? '';
                     }
                     if (tools.isListenAllAddress(config.objects.host)) {
                         // TODO: why we set the remote IP only when the local config allows full connectivity?
-                        config.objects.host = ipHost;
+                        config.objects.host = ipHost ?? '';
                     }
 
                     fs.writeFileSync(this.configName, JSON.stringify(config, null, 2));

packages/cli/src/lib/setup/setupSetup.ts

@@ -895,8 +895,8 @@ Please DO NOT copy files manually into ioBroker storage directories!`
             const defaultSentinelName = originalConfig.states.sentinelName
                 ? originalConfig.states.sentinelName
                 : oSentinelName && oPort === sPort
-                ? oSentinelName
-                : 'mymaster';
+                  ? oSentinelName
+                  : 'mymaster';
             sSentinelName = rl.question(`States Redis Sentinel Master Name [${defaultSentinelName}]: `, {
                 defaultInput: defaultSentinelName
             });

packages/cli/src/lib/setup/setupUpload.ts

@@ -392,7 +392,6 @@ export class Upload {
         isAdmin: boolean,
         files: string[],
         id: string,
-        rev: any,
         logger: Logger | typeof console
     ): Promise<string> {
         const uploadID = `system.adapter.${adapter}.upload`;
@@ -442,18 +441,8 @@ export class Upload {
             }
 
             try {
-                await new Promise<void>((resolve, reject) => {
-                    const stream = fs.createReadStream(file);
-                    stream.on('error', e => reject(e));
-                    stream.pipe(
-                        this.objects.insert(id, attName, null, mimeType || {}, { rev }, err => {
-                            if (err) {
-                                console.log(err);
-                            }
-                            resolve();
-                        })
-                    );
-                });
+                const content = await fs.readFile(file);
+                await this.objects.writeFileAsync(id, attName, content, { mimeType: mimeType || undefined });
             } catch (e) {
                 console.error(`Error: Cannot upload ${file}: ${e.message}`);
             }
@@ -627,11 +616,9 @@ export class Upload {
             if (!isAdmin) {
                 await this.checkRestartOther(adapter);
                 await new Promise<void>(resolve => setTimeout(() => resolve(), 25));
-                // @ts-expect-error TODO rev is not required and should not exist on an object?
-                await this.upload(adapter, isAdmin, files, id, result?.rev, logger);
+                await this.upload(adapter, isAdmin, files, id, logger);
             } else {
-                // @ts-expect-error TODO rev is not required and should not exist on an object?
-                await this.upload(adapter, isAdmin, files, id, result?.rev, logger);
+                await this.upload(adapter, isAdmin, files, id, logger);
             }
         }
         return adapter;

packages/common/src/lib/common/logger.ts

@@ -230,8 +230,8 @@ export function logger(
                         transport.localTime !== undefined
                             ? transport.localTime
                             : userOptions.localTime === undefined
-                            ? true
-                            : userOptions.localTime;
+                              ? true
+                              : userOptions.localTime;
                     transport.datePattern = 'YYYY-MM-DD';
                     transport.format = winston.format.combine(winston.format.printf(formatter));
                     /*transport.logException = function (message, info, next, err) {
@@ -240,8 +240,8 @@ export function logger(
                     transport.zippedArchive = isWindows
                         ? false
                         : transport.zippedArchive !== undefined
-                        ? transport.zippedArchive
-                        : true;
+                          ? transport.zippedArchive
+                          : true;
 
                     if (transport.maxFiles === null && userOptions.maxDays) {
                         transport.maxFiles = `${userOptions.maxDays}d`;