Build anki web assets from rslib-bridge/anki submodule (#217)

ankidroid · Jul 19, 2022 · ba74873 · ba74873
1 parent b279e01
commit ba74873
Show file tree

Hide file tree

Showing 16 changed files with 186 additions and 10 deletions.
diff --git a/.cargo/config.toml b/.cargo/config.toml
@@ -1,4 +1,4 @@
 [env]
 RSLIB_FTL_ROOT = { value = "ftl/core/l10n.toml", relative = true }
-BUILDINFO = { value = "rslib-bridge/buildinfo.txt", relative = true }
+BUILDINFO = { value = "rsdroid/build/generated/anki_artifacts/buildinfo.txt", relative = true }
 BAZEL = "1"
diff --git a/.github/workflows/linux_build.yml b/.github/workflows/linux_build.yml
@@ -70,6 +70,18 @@ jobs:
             ${{ runner.os }}-rust-v1-assembleRelease
             ${{ runner.os }}-rust-v1
 
+      # Caching node modules and bazel, the bazel cache path on linux, mac and windows are according to following docs
+      # https://docs.bazel.build/versions/main/output_directories.html
+      - name: Node and Bazel cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cache/bazel
+            rslib-bridge/anki/node_modules
+          key: ${{ runner.os }}-v1-node-bazel-cache-${{ hashFiles('rslib-bridge/anki/package.json') }}-${{ hashFiles('rslib-bridge/anki/defs.bzl') }}
+          restore-keys: |
+            ${{ runner.os }}-v1-node-bazel-cache
+
       - name: Build
         run: ./gradlew clean assembleRelease -DtestBuildType=release -Dorg.gradle.daemon=false -Dorg.gradle.console=plain # assembleAndroidTest
 

diff --git a/.github/workflows/macos_build.yml b/.github/workflows/macos_build.yml
@@ -69,6 +69,18 @@ jobs:
             ${{ runner.os }}-rust-v1-assembleDebug
             ${{ runner.os }}-rust-v1
 
+      # Caching node modules and bazel, the bazel cache path on linux, mac and windows are according to following docs
+      # https://docs.bazel.build/versions/main/output_directories.html
+      - name: Node and Bazel cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            /private/var/tmp/_bazel_runner
+            rslib-bridge/anki/node_modules
+          key: ${{ runner.os }}-v1-node-bazel-cache-${{ hashFiles('rslib-bridge/anki/package.json') }}-${{ hashFiles('rslib-bridge/anki/defs.bzl') }}
+          restore-keys: |
+            ${{ runner.os }}-v1-node-bazel-cache
+
       - name: Build
         run: ./gradlew clean assembleDebug -DtestBuildType=debug -Dorg.gradle.daemon=false -Dorg.gradle.console=plain
 

diff --git a/.github/workflows/publish_library.yml b/.github/workflows/publish_library.yml
@@ -50,6 +50,18 @@ jobs:
         run: |
           pip3 install --upgrade protobuf stringcase
 
+      # Caching node modules and bazel, the bazel cache path on linux, mac and windows are according to following docs
+      # https://docs.bazel.build/versions/main/output_directories.html
+      - name: Node and Bazel cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cache/bazel
+            rslib-bridge/anki/node_modules
+          key: ${{ runner.os }}-v1-node-bazel-cache-${{ hashFiles('rslib-bridge/anki/package.json') }}-${{ hashFiles('rslib-bridge/anki/defs.bzl') }}
+          restore-keys: |
+            ${{ runner.os }}-v1-node-bazel-cache
+
       - name: Build
         run: ./gradlew clean assembleRelease -DtestBuildType=release -Dorg.gradle.daemon=false -Dorg.gradle.console=plain
 

diff --git a/.github/workflows/publish_testing.yml b/.github/workflows/publish_testing.yml
@@ -52,6 +52,18 @@ jobs:
           brew install x86_64-unknown-linux-gnu
           x86_64-unknown-linux-gnu-gcc -v
 
+      # Caching node modules and bazel, the bazel cache path on linux, mac and windows are according to following docs
+      # https://docs.bazel.build/versions/main/output_directories.html
+      - name: Node and Bazel cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            /private/var/tmp/_bazel_runner
+            rslib-bridge/anki/node_modules
+          key: ${{ runner.os }}-v1-node-bazel-cache-${{ hashFiles('rslib-bridge/anki/package.json') }}-${{ hashFiles('rslib-bridge/anki/defs.bzl') }}
+          restore-keys: |
+            ${{ runner.os }}-v1-node-bazel-cache
+
       - name: Build JAR
         run: |
           export ANKIDROID_LINUX_CC=x86_64-unknown-linux-gnu-gcc

diff --git a/.github/workflows/robolectric_build.yml b/.github/workflows/robolectric_build.yml
@@ -85,6 +85,18 @@ jobs:
             ${{ runner.os }}-rust-v1-rsdroid-testing:build
             ${{ runner.os }}-rust-v1
 
+      # Caching node modules and bazel, the bazel cache path on linux, mac and windows are according to following docs
+      # https://docs.bazel.build/versions/main/output_directories.html
+      - name: Node and Bazel cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            /private/var/tmp/_bazel_runner
+            rslib-bridge/anki/node_modules
+          key: ${{ runner.os }}-v1-node-bazel-cache-${{ hashFiles('rslib-bridge/anki/package.json') }}-${{ hashFiles('rslib-bridge/anki/defs.bzl') }}
+          restore-keys: |
+            ${{ runner.os }}-v1-node-bazel-cache
+
       - name: Build JAR
         run: |
           export ANKIDROID_LINUX_CC=x86_64-unknown-linux-gnu-gcc
@@ -179,4 +191,6 @@ jobs:
 
       - name: Run Tests (win)
         if: matrix.os == 'windows-latest'
-        run: ./gradlew rsdroid:test -x jar -x cargoBuildArm -x cargoBuildX86 -x cargoBuildArm64 -x cargoBuildX86_64
+        run: | 
+          ./tools/web/set-bash-win.ps1
+          ./gradlew rsdroid:test -x jar -x cargoBuildArm -x cargoBuildX86 -x cargoBuildArm64 -x cargoBuildX86_64
diff --git a/.github/workflows/windows_build.yml b/.github/workflows/windows_build.yml
@@ -92,6 +92,18 @@ jobs:
             ${{ runner.os }}-rust-v1-assembleRelease
             ${{ runner.os }}-rust-v1
 
+      # Caching node modules and bazel, the bazel cache path on linux, mac and windows are according to following docs
+      # https://docs.bazel.build/versions/main/output_directories.html
+      - name: Node and Bazel cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            C:/Users/runneradmin/_bazel_runneradmin
+            rslib-bridge/anki/node_modules
+          key: ${{ runner.os }}-v1-node-bazel-cache-${{ hashFiles('rslib-bridge/anki/package.json') }}-${{ hashFiles('rslib-bridge/anki/defs.bzl') }}
+          restore-keys: |
+            ${{ runner.os }}-v1-node-bazel-cache
+
       # -Dorg.gradle.daemon=false has been removed
       # -Dorg.gradle.console=plain has been removed
       #      env:
@@ -105,5 +117,6 @@ jobs:
           $Env:ANDROID_NDK_HOME =  "C:\ProgramFiles\Android\android-sdk\ndk-bundle"
           $Env:ANDROID_NDK_PATH =  "C:\ProgramFiles\Android\android-sdk\ndk-bundle"
           $Env:ANDROID_SDK_ROOT =  "C:\ProgramFiles\Android\android-sdk"
-          $env:ANDROID_HOME
+          $env:ANDROID_HOME   
+          ./tools/web/set-bash-win.ps1
           ./gradlew clean assembleRelease -DtestBuildType=release # assembleAndroidTest
diff --git a/.github/workflows/windows_pure_build.yml b/.github/workflows/windows_pure_build.yml
@@ -66,5 +66,19 @@ jobs:
             ${{ runner.os }}-rust-v1-assembleRelease
             ${{ runner.os }}-rust-v1
 
+      # Caching node modules and bazel, the bazel cache path on linux, mac and windows are according to following docs
+      # https://docs.bazel.build/versions/main/output_directories.html
+      - name: Node and Bazel cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            C:/Users/runneradmin/_bazel_runneradmin
+            rslib-bridge/anki/node_modules
+          key: ${{ runner.os }}-v1-node-bazel-cache-${{ hashFiles('rslib-bridge/anki/package.json') }}-${{ hashFiles('rslib-bridge/anki/defs.bzl') }}
+          restore-keys: |
+            ${{ runner.os }}-v1-node-bazel-cache
+
       - name: Build
-        run: ./gradlew clean assembleRelease -DtestBuildType=release
+        run: | 
+          ./tools/web/set-bash-win.ps1
+          ./gradlew clean assembleRelease -DtestBuildType=release
diff --git a/build-web-assets.gradle b/build-web-assets.gradle
@@ -0,0 +1,9 @@
+task buildAnkiWebAssets(type: Exec) {
+    workingDir "$rootDir"
+    String toolPath = System.getProperty('os.name').toLowerCase(Locale.ROOT).contains('windows') ? 'tools/web/build-web-assets.ps1' : 'tools/web/build-web-assets.sh'
+    if (System.getProperty('os.name').toLowerCase(Locale.ROOT).contains('windows')) {
+        commandLine 'PowerShell', '-File', toolPath
+    } else {
+        commandLine 'sh', '-c', toolPath
+    }
+}
diff --git a/docs/TESTING.md b/docs/TESTING.md
@@ -10,6 +10,10 @@ so far.
 
 Make sure you can build AnkiDroid first.
 
+Follow the initial setup instructions in the computer version
+(rslib-bridge/anki/docs/{windows,mac,linux}.md) to install prerequisites
+like Bazelisk and the build tools for your platform.
+
 Install NDK:
 
 - Download https://developer.android.com/studio#command-tools

diff --git a/rsdroid-testing/build.gradle b/rsdroid-testing/build.gradle
@@ -19,6 +19,8 @@ apply plugin: 'kotlin'
 apply plugin: 'signing'
 apply plugin: 'com.vanniktech.maven.publish'
 
+apply from: "$rootDir/build-web-assets.gradle"
+
 dependencies {
     implementation fileTree(dir: 'libs', include: ['*.jar'])
     // obtaining the OS
@@ -118,6 +120,7 @@ def toCommandLine(Platform targetPlatform) {
 }
 
 task preBuildWindows(type: Exec) {
+    dependsOn "buildAnkiWebAssets"
     doFirst {
         logger.lifecycle(toCommandLine(Platform.WINDOWS).join(" "))
     }
@@ -126,6 +129,7 @@ task preBuildWindows(type: Exec) {
 }
 
 task preBuildMac(type: Exec) {
+    dependsOn "buildAnkiWebAssets"
     doFirst {
         logger.lifecycle(toCommandLine(Platform.MACOS).join(" "))
     }
@@ -147,6 +151,7 @@ task preBuildMac(type: Exec) {
 }
 
 task preBuildLinux(type: Exec) {
+    dependsOn "buildAnkiWebAssets"
     doFirst {
         logger.lifecycle(toCommandLine(Platform.LINUX).join(" "))
     }

diff --git a/rsdroid/build.gradle b/rsdroid/build.gradle
@@ -4,6 +4,8 @@ apply plugin: "kotlin-android"
 apply plugin: 'signing'
 apply plugin: 'com.vanniktech.maven.publish'
 
+apply from: "$rootDir/build-web-assets.gradle"
+
 def getAnkiCommitHash = { ->
     def hashStdOut = new ByteArrayOutputStream()
     exec {
@@ -45,6 +47,9 @@ android {
     sourceSets {
         main {
             kotlin.srcDirs += "build/generated/source/fluent"
+            resources {
+                srcDirs "src/main/resources", "build/generated/anki_artifacts"
+            }
         }
     }
 }
@@ -100,12 +105,19 @@ preBuild.dependsOn "generateTranslations"
 
 Boolean wantAllPlatforms = System.getenv("CURRENT_ONLY") != "true"
 
-if (wantAllPlatforms) {
-    preBuild.dependsOn "cargoBuild"
-} else {
-    preBuild.dependsOn "cargoBuildX86_64"
+task cargoBuildTask {
+    if (wantAllPlatforms) {
+        preBuild.dependsOn "cargoBuild"
+    } else {
+        preBuild.dependsOn "cargoBuildX86_64"
+    }
+
+    mustRunAfter "buildAnkiWebAssets"
 }
 
+preBuild.dependsOn "buildAnkiWebAssets"
+preBuild.dependsOn "cargoBuildTask"
+
 signing {
     def hasPrivate = project.hasProperty('SIGNING_PRIVATE_KEY')
     def hasPassword = project.hasProperty('SIGNING_PASSWORD')

diff --git a/rslib-bridge/buildinfo.txt b/rslib-bridge/buildinfo.txt
diff --git a/tools/web/build-web-assets.ps1 b/tools/web/build-web-assets.ps1
@@ -0,0 +1,21 @@
+$SRC_DIR = (Get-Item .).FullName
+$ROOT_DIR = (Get-Item D:/a/).FullName
+$ANKI_SRC = (Get-Item $SRC_DIR/rslib-bridge/anki).FullName
+$TEMP_DIR = (Get-Item $ENV:TEMP).FullName
+
+New-Item -P $SRC_DIR/rsdroid/build/generated/anki_artifacts/web -itemType Directory
+$BUILD_DIR = (Get-Item $SRC_DIR/rsdroid/build/generated/anki_artifacts).FullName
+
+Set-Location -Path $ANKI_SRC
+
+bazel --output_user_root=$ROOT_DIR/_bzl build -c opt buildinfo.txt --symlink_prefix=$TEMP_DIR/.bazel/ --verbose_failures
+Copy-Item -Path $TEMP_DIR/.bazel/bin/buildinfo.txt -Destination $BUILD_DIR/buildinfo.txt -PassThru
+
+bazel --output_user_root=$ROOT_DIR/_bzl build ts/reviewer/reviewer_extras_bundle --symlink_prefix=$TEMP_DIR/.bazel/ --verbose_failures
+Copy-Item -Path $TEMP_DIR/.bazel/bin/ts/reviewer/reviewer_extras_bundle.js -Destination $BUILD_DIR/web -PassThru
+
+bazel --output_user_root=$ROOT_DIR/_bzl build qt/aqt/data/web/pages --symlink_prefix=$TEMP_DIR/.bazel/ --verbose_failures
+Get-Childitem $TEMP_DIR/.bazel/bin/qt/aqt/data/web/pages -Recurse -Include *.html,*.js,*.css | Copy-Item -Destination $BUILD_DIR/web -PassThru
+
+Copy-Item -Path $ANKI_SRC/cargo/licenses.json -Destination $BUILD_DIR/web/licenses-cargo.json -PassThru
+Copy-Item -Path $ANKI_SRC/ts/licenses.json -Destination $BUILD_DIR/web/licenses-ts.json -PassThru
diff --git a/tools/web/build-web-assets.sh b/tools/web/build-web-assets.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+# build web assets with anki submodule in rslib-bridge
+
+# check if bazel exist or not
+if ! bazel --version > /dev/null 2>&1; then
+  echo "bazel: command not found. Please install Bazelisk. Your distro may have it,"
+  echo "or you can fetch the binary from https://github.com/bazelbuild/bazelisk/releases"
+  echo "and rename it to /usr/local/bin/bazel"
+  exit 1
+fi
+
+SRC_DIR=$(pwd)
+ANKI_SRC=$SRC_DIR/rslib-bridge/anki
+BUILD_DIR=$SRC_DIR/rsdroid/build/generated/anki_artifacts
+mkdir -p $BUILD_DIR/web
+
+# build buildinfo.txt, reviewer_extras_bundle and pages files
+cd $ANKI_SRC
+
+bazel build --symlink_prefix=/tmp/.bazel/ -c opt buildinfo.txt
+cp /tmp/.bazel/bin/buildinfo.txt $BUILD_DIR/buildinfo.txt
+
+bazel build ts/reviewer/reviewer_extras_bundle --symlink_prefix=/tmp/.bazel/
+cp /tmp/.bazel/bin/ts/reviewer/reviewer_extras_bundle.js $BUILD_DIR/web
+
+bazel build qt/aqt/data/web/pages --symlink_prefix=/tmp/.bazel/
+cp -r /tmp/.bazel/bin/qt/aqt/data/web/pages/* $BUILD_DIR/web
+
+cp $ANKI_SRC/cargo/licenses.json $BUILD_DIR/web/licenses-cargo.json
+cp $ANKI_SRC/ts/licenses.json $BUILD_DIR/web/licenses-ts.json
+
+chmod -R a+w $BUILD_DIR/web
diff --git a/tools/web/set-bash-win.ps1 b/tools/web/set-bash-win.ps1
@@ -0,0 +1,6 @@
+# wsl does not support bazel yet https://github.com/bazelbuild/bazel/issues/10941
+echo "Renaming Bash in WSL"
+mv.exe "C:/WINDOWS/system32/bash.EXE" "C:/WINDOWS/system32/bash_copy.EXE"
+Set-Alias -Name bash -Value "C:/msys64/usr/bin/bash.exe"
+$env:BAZEL_SH = "C:/msys64/usr/bin/bash.exe"
+[Environment]::SetEnvironmentVariable('BAZEL_SH', 'C:/msys64/usr/bin/bash.exe', 'Machine')