feat(appimage): move logic to app-builder, improve linux icon resolving

Close #2570

docker/base/Dockerfile

@@ -9,15 +9,12 @@ RUN curl -L https://yarnpkg.com/latest.tar.gz | tar xvz && mv yarn-* /yarn && ln
   # git ssh for using as docker image on CircleCI
   # python for node-gyp
   # rpm is required for FPM to build rpm package
-  # libpng16-16 is required for libicns1_0.8.1-3.1 (on xenial)
   # libsecret-1-0 and libgnome-keyring-dev are required even for prebuild keytar
   # libgtk2.0-dev for snap desktop-gtk2 (see https://github.com/ubuntu/snapcraft-desktop-helpers/blob/master/snapcraft.yaml#L248)
   apt-get -qq install --no-install-recommends git qtbase5-dev bsdtar build-essential autoconf libssl-dev gcc-multilib g++-multilib lzip rpm python libcurl3 git git-lfs ssh unzip \
-  libpng16-16 icnsutils libopenjp2-7 \
   libsecret-1-0 libgnome-keyring-dev \
+  libopenjp2-tools \
   libgtk2.0-dev && \
-  # libicns
-  curl -O http://mirrors.kernel.org/ubuntu/pool/universe/libi/libicns/libicns1_0.8.1-3.1_amd64.deb && dpkg --install libicns1_0.8.1-3.1_amd64.deb && unlink libicns1_0.8.1-3.1_amd64.deb && \
   # git-lfs
   git lfs install && \
   # snap

docs/multi-platform-build.md

@@ -45,7 +45,7 @@ You can use [Docker](#docker) to avoid installing system dependencies.
 
 To build app in distributable format for Linux:
 ```
-sudo apt-get install --no-install-recommends -y icnsutils
+sudo apt-get install --no-install-recommends -y libopenjp2-tools
 ```
 
 To build rpm: `sudo apt-get install --no-install-recommends -y rpm`.
@@ -68,7 +68,7 @@ sudo apt-get install --no-install-recommends -y gcc-multilib g++-multilib
 ```
 
 ## Travis Linux
-[Trusty](https://docs.travis-ci.com/user/trusty-ci-environment/) is required — default Travis Linux dist is outdated and `icnsutils` version is non-functional.
+[Trusty](https://docs.travis-ci.com/user/trusty-ci-environment/) is required.
 ```yaml
 sudo: required
 dist: trusty

package.json

@@ -29,10 +29,10 @@
   "////": "All typings are added into root `package.json` to avoid duplication errors in the IDE compiler (several `node.d.ts` files).",
   "dependencies": {
     "7zip-bin": "~3.1.0",
-    "app-builder-bin": "1.2.1",
+    "app-builder-bin": "1.3.1",
     "archiver": "^2.1.1",
     "async-exit-hook": "^2.0.1",
-    "aws-sdk": "^2.191.0",
+    "aws-sdk": "^2.192.0",
     "bluebird-lst": "^1.0.5",
     "chalk": "^2.3.0",
     "chromium-pickle-js": "^0.2.0",
@@ -95,8 +95,8 @@
     "gitbook-plugin-github": "^2.0.0",
     "gitbook-plugin-github-buttons": "^3.0.0",
     "globby": "^7.1.1",
-    "jest-cli": "^22.2.1",
-    "jest-junit": "^3.5.0",
+    "jest-cli": "^22.2.2",
+    "jest-junit": "^3.6.0",
     "jsdoc-to-markdown": "^4.0.1",
     "path-sort": "^0.1.0",
     "ts-babel": "^4.1.8",

packages/builder-util/package.json

@@ -11,7 +11,7 @@
     "out"
   ],
   "dependencies": {
-    "app-builder-bin": "1.2.1",
+    "app-builder-bin": "1.3.1",
     "temp-file": "^3.1.1",
     "fs-extra-p": "^4.5.0",
     "is-ci": "^1.1.0",

packages/builder-util/src/fs.ts

@@ -4,7 +4,6 @@ import isCi from "is-ci"
 import * as path from "path"
 import Mode from "stat-mode"
 import { log } from "./log"
-import { exec } from "./util"
 import { orNullIfFileNotExist } from "./promise"
 
 export const MAX_FILE_REQUESTS = 8
@@ -291,20 +290,6 @@ export function copyDir(src: string, destination: string, options: CopyDirOption
     .then(() => BluebirdPromise.map(links, it => symlink(it.link, it.file), CONCURRENCY))
 }
 
-// https://unix.stackexchange.com/questions/202430/how-to-copy-a-directory-recursively-using-hardlinks-for-each-file
-export function copyDirUsingHardLinks(source: string, destination: string) {
-  const promise = ensureDir(destination)
-  if (process.platform !== "darwin") {
-    return promise
-      .then(() => exec("cp", ["-d", "--recursive", "--preserve=mode", "--link", "-T" /* to merge */, source + "/", destination + "/"]))
-  }
-
-  return promise
-    .then(() => exec("pax", ["-rwl", "-p", "amp" /* Do not preserve file access times, Do not preserve file modification times, Preserve the file mode	bits */, ".", destination], {
-      cwd: source,
-    }))
-}
-
 export const DO_NOT_USE_HARD_LINKS = (file: string) => false
 export const USE_HARD_LINKS = (file: string) => true
 

packages/electron-builder-lib/package.json

@@ -42,7 +42,7 @@
   "homepage": "https://github.com/electron-userland/electron-builder",
   "dependencies": {
     "7zip-bin": "~3.1.0",
-    "app-builder-bin": "1.2.1",
+    "app-builder-bin": "1.3.1",
     "async-exit-hook": "^2.0.1",
     "bluebird-lst": "^1.0.5",
     "chromium-pickle-js": "^0.2.0",

packages/electron-builder-lib/src/platformPackager.ts

@@ -560,7 +560,8 @@ export abstract class PlatformPackager<DC extends PlatformSpecificBuildOptions>
   protected async getOrConvertIcon(format: IconFormat): Promise<string | null> {
     const iconPath = this.platformSpecificBuildOptions.icon || this.config.icon
     if (iconPath != null) {
-      return (await this.resolveIcon([iconPath], format))[0].file
+      const iconInfos = await this.resolveIcon([iconPath], format)
+      return (iconInfos)[0].file
     }
 
     const resourceList = await this.resourceList
@@ -581,7 +582,11 @@ export abstract class PlatformPackager<DC extends PlatformSpecificBuildOptions>
 
   // convert if need, validate size (it is a reason why tool is called even if file has target extension (already specified as foo.icns for example))
   async resolveIcon(sources: Array<string>, outputFormat: IconFormat): Promise<Array<IconInfo>> {
-    const arg = ["icon", "--format", outputFormat, "--root", this.buildResourcesDir, "--root", this.projectDir]
+    const arg = [
+      "icon",
+      "--format", outputFormat,
+      "--root", this.buildResourcesDir, "--root", this.projectDir,
+    ]
     for (const source of sources) {
       arg.push("--input", source)
     }
@@ -593,7 +598,6 @@ export abstract class PlatformPackager<DC extends PlatformSpecificBuildOptions>
         // command creates temp dir amd cannot delete it automatically since result files located in and it is our responsibility remove it after use,
         // so, we just set TMPDIR to tempDirManager.rootTempDir and tempDirManager in any case will delete rootTempDir on exit
         TMPDIR: await this.info.tempDirManager.rootTempDir,
-        DEBUG: log.isDebugEnabled ? "true" : "false",
       },
     })
 

packages/electron-builder-lib/src/targets/AppImageTarget.ts

@@ -1,20 +1,21 @@
+import { path7za } from "7zip-bin"
+import { appBuilderPath } from "app-builder-bin"
 import BluebirdPromise from "bluebird-lst"
-import { Arch, debug, exec, serializeToYaml } from "builder-util"
+import { Arch, log, serializeToYaml, isEnvTrue, spawn } from "builder-util"
 import { UUID } from "builder-util-runtime"
-import { copyDir, copyOrLinkFile, unlinkIfExists, copyDirUsingHardLinks, USE_HARD_LINKS } from "builder-util/out/fs"
+import { copyOrLinkFile, unlinkIfExists } from "builder-util/out/fs"
 import * as ejs from "ejs"
-import { ensureDir, readFile, symlink, writeFile } from "fs-extra-p"
+import { ensureDir, outputFile, readFile, symlink, writeFile } from "fs-extra-p"
 import { Lazy } from "lazy-val"
 import * as path from "path"
 import { AppImageOptions } from ".."
 import { Target } from "../core"
-import { LinuxPackager, toAppImageOrSnapArch } from "../linuxPackager"
+import { LinuxPackager } from "../linuxPackager"
 import { getAppUpdatePublishConfiguration } from "../publish/PublishManager"
 import { getTemplatePath } from "../util/pathManager"
 import { appendBlockmap } from "./differentialUpdateInfoBuilder"
 import { LinuxTargetHelper } from "./LinuxTargetHelper"
 import { createStageDir } from "./targetUtil"
-import { getAppImage } from "./tools"
 
 const appRunTemplate = new Lazy<(data: any) => string>(async () => {
   return ejs.compile(await readFile(path.join(getTemplatePath("linux"), "AppRun.sh"), "utf-8"))
@@ -46,14 +47,11 @@ export default class AppImageTarget extends Target {
 
     // pax doesn't like dir with leading dot (e.g. `.__appimage`)
     const stageDir = await createStageDir(this, packager, arch)
-    const appInStageDir = stageDir.getTempFile("app")
-    await copyDirUsingHardLinks(appOutDir, appInStageDir)
-
     const resourceName = `appimagekit-${this.packager.executableName}`
     const installIcons = await this.copyIcons(stageDir.dir, resourceName)
 
     const finalDesktopFilename = `${this.packager.executableName}.desktop`
-    await BluebirdPromise.all([
+    await Promise.all([
       unlinkIfExists(artifactPath),
       writeFile(stageDir.getTempFile("/AppRun"), (await appRunTemplate.value)({
         systemIntegration: this.options.systemIntegration || "ask",
@@ -72,49 +70,29 @@ export default class AppImageTarget extends Target {
       throw new Error("Icon is not provided")
     }
 
-    //noinspection SpellCheckingInspection
-    const vendorDir = await getAppImage()
-
     if (this.packager.packagerOptions.effectiveOptionComputed != null && await this.packager.packagerOptions.effectiveOptionComputed({desktop: await this.desktopEntry.value})) {
       return
     }
 
-    if (arch === Arch.x64 || arch === Arch.ia32) {
-      await copyDir(path.join(vendorDir, "lib", arch === Arch.x64 ? "x86_64-linux-gnu" : "i386-linux-gnu"), stageDir.getTempFile("usr/lib"), {
-        isUseHardLink: USE_HARD_LINKS,
-      })
-    }
-
     const publishConfig = await getAppUpdatePublishConfiguration(packager, arch)
     if (publishConfig != null) {
-      await writeFile(path.join(packager.getResourcesDir(appInStageDir), "app-update.yml"), serializeToYaml(publishConfig))
+      await outputFile(path.join(packager.getResourcesDir(stageDir.getTempFile("app")), "app-update.yml"), serializeToYaml(publishConfig))
     }
 
-    const vendorToolDir = path.join(vendorDir, process.platform === "darwin" ? "darwin" : `linux-${process.arch}`)
-    // default gzip compression - 51.9, xz - 50.4 difference is negligible, start time - well, it seems, a little bit longer (but on Parallels VM on external SSD disk)
-    // so, to be decided later, is it worth to use xz by default
-    const args = [
-      "--runtime-file", path.join(vendorDir, `runtime-${(archToRuntimeName(arch))}`),
-      "--no-appstream",
-    ]
-    if (debug.enabled) {
-      args.push("--verbose")
-    }
+    const args = ["appimage", "--stage", stageDir.dir, "--arch", Arch[arch], "--output", artifactPath, "--app", appOutDir]
     if (packager.compression === "maximum") {
-      args.push("--comp", "xz")
+      args.push("--compression", "xz")
+    }
+    if (log.isDebugEnabled && !isEnvTrue(process.env.ELECTRON_BUILDER_REMOVE_STAGE_EVEN_IF_DEBUG)) {
+      args.push("--no-remove-stage")
     }
-    args.push(stageDir.dir, artifactPath)
-    await exec(path.join(vendorToolDir, "appimagetool"), args, {
+    await spawn(appBuilderPath, args, {
       env: {
         ...process.env,
-        PATH: `${vendorToolDir}:${process.env.PATH}`,
-        // to avoid detection by appimagetool (see extract_arch_from_text about expected arch names)
-        ARCH: toAppImageOrSnapArch(arch),
-      }
+        SZA_PATH: path7za,
+      },
     })
 
-    await stageDir.cleanup()
-
     const updateInfo = await appendBlockmap(artifactPath)
     packager.info.dispatchArtifactCreated({
       file: artifactPath,
@@ -154,23 +132,4 @@ export default class AppImageTarget extends Target {
     }
     return installIcons
   }
-}
-
-function archToRuntimeName(arch: Arch) {
-  switch (arch) {
-    case Arch.armv7l:
-      return "armv7"
-
-    case Arch.arm64:
-      return "arm64"
-
-    case Arch.ia32:
-      return "i686"
-
-    case Arch.x64:
-      return "x86_64"
-
-    default:
-      throw new Error(`AppImage for arch ${Arch[arch]} not supported`)
-  }
 }

packages/electron-builder-lib/src/targets/LinuxTargetHelper.ts

@@ -26,26 +26,23 @@ export class LinuxTargetHelper {
     const packager = this.packager
     const iconDir = packager.platformSpecificBuildOptions.icon
     const sources = [iconDir == null ? "icons" : iconDir]
-    const icnsPath = this.getIcns()
+
+    const commonConfiguration = packager.config
+    let icnsPath = (commonConfiguration.mac || {}).icon || commonConfiguration.icon
     if (icnsPath != null) {
+      if (!icnsPath.endsWith(".icns")) {
+        icnsPath += ".icns"
+      }
       sources.push(icnsPath)
     }
+
     sources.push(path.join(getTemplatePath("linux"), "electron-icons"))
 
     const result = await packager.resolveIcon(sources, "set")
     this.maxIconPath = result[result.length - 1].file
     return result
   }
 
-  private getIcns(): string | null {
-    const build = this.packager.info.config
-    let iconPath = (build.mac || {}).icon || build.icon
-    if (iconPath != null && !iconPath.endsWith(".icns")) {
-      iconPath += ".icns"
-    }
-    return iconPath == null ? null : path.resolve(this.packager.projectDir, iconPath)
-  }
-
   getDescription(options: LinuxTargetSpecificOptions) {
     return options.description || this.packager.appInfo.description
   }

packages/electron-builder-lib/src/targets/tools.ts

@@ -12,11 +12,6 @@ export function getLinuxToolsPath() {
   return getBinFromGithub("linux-tools", "mac-10.12.3", "SQ8fqIRVXuQVWnVgaMTDWyf2TLAJjJYw3tRSqQJECmgF6qdM7Kogfa6KD49RbGzzMYIFca9Uw3MdsxzOPRWcYw==")
 }
 
-export function getAppImage() {
-  //noinspection SpellCheckingInspection
-  return getBinFromGithub("appimage", "9.0.5", "AZbiBSeyow/pKCzeyIwVtogIUSWD4GxAxkqwL9GQcL1vq+EhcNPeMKOdlSI045SU4pknU4ceLwO5tzV7o0tNOw==")
-}
-
 export const fpmPath = new Lazy(() => {
   if (process.platform === "win32" || process.env.USE_SYSTEM_FPM === "true") {
     return Promise.resolve("fpm")
@@ -40,7 +35,6 @@ export const fpmPath = new Lazy(() => {
 export function prefetchBuildTools(): Promise<any> {
   // yes, we starting to use native Promise
   return Promise.all([
-    getAppImage(),
     fpmPath.value,
     getBinFromGithub("snap-template", SNAP_TEMPLATE_VERSION, SNAP_TEMPLATE_SHA512),
   ])

packages/electron-publisher-s3/package.json

@@ -12,7 +12,7 @@
   ],
   "dependencies": {
     "fs-extra-p": "^4.5.0",
-    "aws-sdk": "^2.191.0",
+    "aws-sdk": "^2.192.0",
     "mime": "^2.2.0",
     "electron-publish": "~0.0.0-semantic-release",
     "builder-util": "^0.0.0-semantic-release",

test/out/linux/__snapshots__/linuxPackagerTest.js.snap

@@ -128,6 +128,22 @@ Object {
 }
 `;
 
+exports[`icons from ICNS (mac) 1`] = `
+Object {
+  "linux": Array [
+    Object {
+      "arch": "x64",
+      "file": "TestApp-1.1.0-x86_64.AppImage",
+      "updateInfo": Object {
+        "blockMapSize": "@blockMapSize",
+        "sha512": "@sha512",
+        "size": "@size",
+      },
+    },
+  ],
+}
+`;
+
 exports[`icons from ICNS 1`] = `
 Object {
   "linux": Array [

test/src/linux/linuxPackagerTest.ts

@@ -1,5 +1,5 @@
 import { Arch, build, Platform } from "electron-builder"
-import { remove, rename } from "fs-extra-p"
+import { move, remove, rename } from "fs-extra-p"
 import * as path from "path"
 import { assertThat } from "../helpers/fileAssert"
 import { app, appThrows, modifyPackageJson } from "../helpers/packTester"
@@ -74,6 +74,24 @@ test.ifNotWindows.ifNotCiMac("AppImage - default icon, custom executable and cus
   },
 }))
 
+// test prepacked asar also https://github.com/electron-userland/electron-builder/issues/1102
+test.ifNotWindows("icons from ICNS (mac)", app({
+  targets: appImageTarget,
+  config: {
+    publish: null,
+    mac: {
+      icon: "resources/time.icns",
+    },
+  },
+}, {
+  projectDirCreated: it => move(path.join(it, "build", "icon.icns"), path.join(it, "resources", "time.icns"))
+    .then(() => remove(path.join(it, "build"))),
+  packed: async context => {
+    const projectDir = context.getResources(Platform.LINUX)
+    await assertThat(projectDir).isDirectory()
+  },
+}))
+
 // test prepacked asar also https://github.com/electron-userland/electron-builder/issues/1102
 test.ifNotWindows("icons from ICNS", app({
   targets: appImageTarget,

test/src/linux/snapTest.ts

@@ -20,7 +20,8 @@ test.ifAll.ifDevOrLinuxCi("snap", app({
 }))
 
 // custom packages to test not-prepacked snap build
-test.ifAll.ifDevOrLinuxCi("snap full", app({
+// very slow
+test.skip("snap full", app({
   targets: snapTarget,
   config: {
     extraMetadata: {