Add support for Lineageos 17.1

Dockerfile

@@ -25,6 +25,9 @@ ENV USER root
 # WARNING: disabling this may slow down a lot your builds!
 ENV USE_CCACHE 1
 
+# We need to specify the ccache binary since it is no longer packaged along with AOSP
+ENV CCACHE_EXEC /usr/bin/ccache
+
 # ccache maximum size. It should be a number followed by an optional suffix: k,
 # M, G, T (decimal), Ki, Mi, Gi or Ti (binary). The default suffix is G. Use 0
 # for no limit.
@@ -67,6 +70,9 @@ ENV CLEAN_OUTDIR false
 # For example, '0 10 * * *' means 'Every day at 10:00 UTC'
 ENV CRONTAB_TIME 'now'
 
+# Put the boot.img in the zips directory
+ENV BOOT_IMG false
+
 # Clean artifacts output after each build
 ENV CLEAN_AFTER_BUILD true
 
@@ -102,6 +108,9 @@ ENV LOGS_SUBDIR true
 # example.
 ENV SIGNATURE_SPOOFING "no"
 
+# Apply the microG unifiedNLP patch
+ENV SUPPORT_UNIFIEDNLP false
+
 # Generate delta files
 ENV BUILD_DELTA false
 
@@ -120,6 +129,10 @@ ENV DELETE_OLD_LOGS 0
 # Requires ZIP_SUBDIR.
 ENV OPENDELTA_BUILDS_JSON ''
 
+# set the java tool options max memory size
+# not sure yet if necessary or helpful
+# ENV JAVA_TOOL_OPTIONS "-Xmx6g"
+
 # You can optionally specify a USERSCRIPTS_DIR volume containing these scripts:
 #  * begin.sh, run at the very beginning
 #  * before.sh, run after the syncing and patching, before starting the builds
@@ -171,27 +184,13 @@ RUN apt-get install -y bc bison bsdmainutils build-essential ccache cgpt cron \
       libxml2-utils lsof lzop maven openjdk-8-jdk pngcrush procps python rsync \
       schedtool squashfs-tools wget xdelta3 xsltproc yasm zip zlib1g-dev
 
+#required for opengapps pico, uncomment to use
+# RUN curl http://ftp.us.debian.org/debian/pool/main/g/git-lfs/git-lfs_2.7.1-1~bpo9+2_amd64.deb -o git-lfs.deb && dpkg -i git-lfs.deb
+# RUN git lfs install
+
 RUN curl https://storage.googleapis.com/git-repo-downloads/repo > /usr/local/bin/repo
 RUN chmod a+x /usr/local/bin/repo
 
-# Download and build delta tools
-################################
-RUN cd /root/ && \
-        mkdir delta && \
-        git clone --depth=1 https://github.com/omnirom/android_packages_apps_OpenDelta.git OpenDelta && \
-        gcc -o delta/zipadjust OpenDelta/jni/zipadjust.c OpenDelta/jni/zipadjust_run.c -lz && \
-        cp OpenDelta/server/minsignapk.jar OpenDelta/server/opendelta.sh delta/ && \
-        chmod +x delta/opendelta.sh && \
-        rm -rf OpenDelta/ && \
-        sed -i -e 's|^\s*HOME=.*|HOME=/root|; \
-                   s|^\s*BIN_XDELTA=.*|BIN_XDELTA=xdelta3|; \
-                   s|^\s*FILE_MATCH=.*|FILE_MATCH=lineage-\*.zip|; \
-                   s|^\s*PATH_CURRENT=.*|PATH_CURRENT=$SRC_DIR/out/target/product/$DEVICE|; \
-                   s|^\s*PATH_LAST=.*|PATH_LAST=$SRC_DIR/delta_last/$DEVICE|; \
-                   s|^\s*KEY_X509=.*|KEY_X509=$KEYS_DIR/releasekey.x509.pem|; \
-                   s|^\s*KEY_PK8=.*|KEY_PK8=$KEYS_DIR/releasekey.pk8|; \
-                   s|publish|$DELTA_DIR|g' /root/delta/opendelta.sh
-
 # Set the work directory
 ########################
 WORKDIR $SRC_DIR

README.md

@@ -76,11 +76,21 @@ The signature spoofing patch can be optionally included with:
 If in doubt, use `restricted`: note that packages that requires the
 FAKE_SIGNATURE permission must be embedded in the build by adding them in
 
+### UnifiedNLP support
+
+If you enable signature spoofing and want to use unifiednlp from microg, the patche
+to support is can be enabled using:
+ * `UNIFIEDNLP_SUPPORT` (false): `true` to patch in unifiednlp support
+
+### Custom Packages
+
  * `CUSTOM_PACKAGES`
 
 Extra packages can be included in the tree by adding the corresponding manifest
 XML to the local_manifests volume.
 
+
+
 ### Proprietary files
 
 Some proprietary files are needed to create a LineageOS build, but they're not
@@ -123,6 +133,8 @@ Other useful settings are:
  * `WITH_SU (false)`: set to `true` to embed `su` in the build (note that, even
     when set to `false`, you can still enable root by flashing the
     [su installable ZIP][los-extras])
+    NOTE: in lineageos 17.1 the WITH_SU flag was dropped, as it did not functioning properly in Android 10
+    The suggested replacement is Magisk https://www.xda-developers.com/lineageos-dropping-superuser-addonsu-implementation-favor-magisk-manager/
  * `RELEASE_TYPE (UNOFFICIAL)`: change the release type of your builds
  * `BUILD_OVERLAY (false)`: normally each build is done on the source tree, then
     the tree is cleaned with `mka clean`. If you want to be sure that each build
@@ -132,6 +144,8 @@ Other useful settings are:
     mirror of the LineageOS source (> 200 GB)
  * `CRONTAB_TIME (now)`: instead of building immediately and exit, build at the
     specified time (uses standard cron format)
+ * `BOOT_IMG (false)`: copy the build boot.img into the zips folder. This is useful
+    as you may need to flash it first if your device doesn't support custom recoveries
 
 The full list of settings, including the less interesting ones not mentioned in
 this guide, can be found in the [Dockerfile][dockerfile].
@@ -159,6 +173,47 @@ When `BUILD_OVERLAY` is `true`
 When `LOCAL_MIRROR` is `true`:
 
  * `/srv/mirror`, for the LineageOS mirror
+
+
+## Including MicroG and other package pre-builts
+
+The microG and FDroid packages are not present in the LineageOS repositories,
+and must be provided through an XML in the `/home/user/manifests`.
+
+#### LineageOS < 17.1
+
+[This][prebuiltapks] repo contains some of the most common packages for these
+kind of builds for lineageOS < 17.1: to include it create an XML (the name is irrelevant, as long as
+it ends with `.xml`) in the `/home/user/manifests` folder with this content:
+
+```
+<?xml version="1.0" encoding="UTF-8"?>
+<manifest>
+  <project name="lineageos4microg/android_prebuilts_prebuiltapks" path="prebuilts/prebuiltapks" remote="github" revision="master" />
+</manifest>
+```
+
+#### LineageOS 17.1
+Some (most) of the packages in the prebuiltapks repo are not updated for Android 10/ LineageOS 17.1
+An example of this is the MicroG apk, and the fdroid extension.
+
+```
+<?xml version="1.0" encoding="UTF-8"?>
+<manifest>
+  <remote name="fdroid" fetch="https://gitlab.com/fdroid/" />
+  <project path="packages/apps/F-DroidPrivilegedExtension"
+           name="privileged-extension.git" remote="fdroid"
+           revision="refs/tags/0.2.11" />
+
+</manifest>
+```
+NOTE: due to https://github.com/microg/android_packages_apps_GmsCore/issues/1059 including microg in the build is unsupported at this time
+get the apks from github and install them manually, or add the microg fdroid repo to fdroid. 
+
+Then include any other pre-built app name in the `CUSTOM_PACKAGES` environment variable like this: 
+```
+    -e "CUSTOM_PACKAGES=F-DroidPrivilegedExtension" \
+```
 
 ## Examples
 

src/build.sh

@@ -124,6 +124,10 @@ for branch in ${BRANCH_NAME//,/ }; do
         themuppets_branch=lineage-15.1
       elif [[ $branch =~ .*lineage-16\.0.* ]]; then
         themuppets_branch=lineage-16.0
+      elif [[ $branch =~ .*lineage-17\.0.* ]]; then
+        themuppets_branch=lineage-17.0
+      elif [[ $branch =~ .*lineage-17\.1.* ]]; then
+        themuppets_branch=lineage-17.1
       else
         themuppets_branch=lineage-15.1
         echo ">> [$(date)] Can't find a matching branch on github.com/TheMuppets, using $themuppets_branch"
@@ -137,14 +141,18 @@ for branch in ${BRANCH_NAME//,/ }; do
     builddate=$(date +%Y%m%d)
     repo sync -c --force-sync &>> "$repo_log"
 
-    android_version=$(sed -n -e 's/^\s*PLATFORM_VERSION\.OPM1 := //p' build/core/version_defaults.mk)
+
+    android_version=$(sed -n -e 's/^\s*PLATFORM_VERSION\.QP1A := //p' build/core/version_defaults.mk)
     if [ -z $android_version ]; then
-      android_version=$(sed -n -e 's/^\s*PLATFORM_VERSION\.PPR1 := //p' build/core/version_defaults.mk)
+      android_version=$(sed -n -e 's/^\s*PLATFORM_VERSION\.OPM1 := //p' build/core/version_defaults.mk)
       if [ -z $android_version ]; then
-        android_version=$(sed -n -e 's/^\s*PLATFORM_VERSION := //p' build/core/version_defaults.mk)
+        android_version=$(sed -n -e 's/^\s*PLATFORM_VERSION\.PPR1 := //p' build/core/version_defaults.mk)
         if [ -z $android_version ]; then
-          echo ">> [$(date)] Can't detect the android version"
-          exit 1
+          android_version=$(sed -n -e 's/^\s*PLATFORM_VERSION := //p' build/core/version_defaults.mk)
+          if [ -z $android_version ]; then
+            echo ">> [$(date)] Can't detect the android version"
+            exit 1
+          fi
         fi
       fi
     fi
@@ -185,6 +193,7 @@ for branch in ${BRANCH_NAME//,/ }; do
         7.*  )    patch_name="android_frameworks_base-N.patch" ;;
         8.*  )    patch_name="android_frameworks_base-O.patch" ;;
 	9*  )    patch_name="android_frameworks_base-P.patch" ;; #not sure why 9 not 9.0 but here's a fix that will work until android 90
+	10*  )    patch_name="android_frameworks_base-Q.patch" ;;
       esac
 
       if ! [ -z $patch_name ]; then
@@ -209,6 +218,25 @@ for branch in ${BRANCH_NAME//,/ }; do
       fi
     fi
 
+    if [ "$SUPPORT_UNIFIEDNLP" = "true" ]; then
+      # Determine which patch should be applied to the current Android source tree
+      patch_name=""
+      case $android_version in
+	      10*  )    patch_name="android_frameworks_base-Q.patch" ;;
+      esac
+
+      if ! [ -z $patch_name ]; then
+        cd frameworks/base
+          echo ">> [$(date)] Applying location services patch"
+          patch --quiet -p1 -i "/root/location_services_patches/$patch_name"
+        git clean -q -f
+        cd ../..
+      else
+        echo ">> [$(date)] ERROR: can't find a unifiednlp support patch for the current Android version ($android_version)"
+        exit 1
+      fi
+    fi
+
     echo ">> [$(date)] Setting \"$RELEASE_TYPE\" as release type"
     sed -i "/\$(filter .*\$(${vendor^^}_BUILDTYPE)/,+2d" "vendor/$vendor/config/common.mk"
 
@@ -240,21 +268,28 @@ for branch in ${BRANCH_NAME//,/ }; do
       echo ">> [$(date)] Adding keys path ($KEYS_DIR)"
       # Soong (Android 9+) complains if the signing keys are outside the build path
       ln -sf "$KEYS_DIR" user-keys
-      sed -i "1s;^;PRODUCT_DEFAULT_DEV_CERTIFICATE := user-keys/releasekey\nPRODUCT_OTA_PUBLIC_KEYS := user-keys/releasekey\nPRODUCT_EXTRA_RECOVERY_KEYS := user-keys/releasekey\n\n;" "vendor/$vendor/config/common.mk"
+      if [ "$android_version_major" -lt "10" ]; then
+        sed -i "1s;^;PRODUCT_DEFAULT_DEV_CERTIFICATE := user-keys/releasekey\nPRODUCT_OTA_PUBLIC_KEYS := user-keys/releasekey\nPRODUCT_EXTRA_RECOVERY_KEYS := user-keys/releasekey\n\n;" "vendor/$vendor/config/common.mk"
+      fi
+
+      if [ "$android_version_major" -ge "10" ]; then
+        sed -i "1s;^;PRODUCT_DEFAULT_DEV_CERTIFICATE := user-keys/releasekey\nPRODUCT_OTA_PUBLIC_KEYS := user-keys/releasekey\n\n;" "vendor/$vendor/config/common.mk"
+      fi
     fi
 
     # Prepare the environment
     echo ">> [$(date)] Preparing build environment"
     source build/envsetup.sh > /dev/null
 
-    if [ -f /root/userscripts/before.sh ]; then
-      echo ">> [$(date)] Running before.sh"
-      /root/userscripts/before.sh
-    fi
-
     for codename in ${devices//,/ }; do
       if ! [ -z "$codename" ]; then
 
+        if [ -f /root/userscripts/before.sh ]; then
+            echo ">> [$(date)] Running before.sh"
+            breakfast $codename
+            /root/userscripts/before.sh
+        fi
+
         currentdate=$(date +%Y%m%d)
         if [ "$builddate" != "$currentdate" ]; then
           # Sync the source code
@@ -309,34 +344,17 @@ for branch in ${BRANCH_NAME//,/ }; do
             find out/target/product/$codename -maxdepth 1 -name "lineage-*-$currentdate-*.zip*" -type f -exec sh /root/fix_build_date.sh {} $currentdate $builddate \; &>> "$DEBUG_LOG"
           fi
 
-          if [ "$BUILD_DELTA" = true ]; then
-            if [ -d "delta_last/$codename/" ]; then
-              # If not the first build, create delta files
-              echo ">> [$(date)] Generating delta files for $codename" | tee -a "$DEBUG_LOG"
-              cd /root/delta
-              if ./opendelta.sh $codename &>> "$DEBUG_LOG"; then
-                echo ">> [$(date)] Delta generation for $codename completed" | tee -a "$DEBUG_LOG"
-              else
-                echo ">> [$(date)] Delta generation for $codename failed" | tee -a "$DEBUG_LOG"
-              fi
-              if [ "$DELETE_OLD_DELTAS" -gt "0" ]; then
-                /usr/bin/python /root/clean_up.py -n $DELETE_OLD_DELTAS -V $los_ver -N 1 "$DELTA_DIR/$codename" &>> $DEBUG_LOG
-              fi
-              cd "$source_dir"
-            else
-              # If the first build, copy the current full zip in $source_dir/delta_last/$codename/
-              echo ">> [$(date)] No previous build for $codename; using current build as base for the next delta" | tee -a "$DEBUG_LOG"
-              mkdir -p delta_last/$codename/ &>> "$DEBUG_LOG"
-              find out/target/product/$codename -maxdepth 1 -name 'lineage-*.zip' -type f -exec cp {} "$source_dir/delta_last/$codename/" \; &>> "$DEBUG_LOG"
-            fi
-          fi
           # Move produced ZIP files to the main OUT directory
           echo ">> [$(date)] Moving build artifacts for $codename to '$ZIP_DIR/$zipsubdir'" | tee -a "$DEBUG_LOG"
           cd out/target/product/$codename
           for build in lineage-*.zip; do
             sha256sum "$build" > "$ZIP_DIR/$zipsubdir/$build.sha256sum"
           done
           find . -maxdepth 1 -name 'lineage-*.zip*' -type f -exec mv {} "$ZIP_DIR/$zipsubdir/" \; &>> "$DEBUG_LOG"
+
+          if [ "$BOOT_IMG" = true ]; then
+            find . -maxdepth 1 -name 'boot.img' -type f -exec mv {} "$ZIP_DIR/$zipsubdir/" \; &>> "$DEBUG_LOG"
+          fi
           cd "$source_dir"
           build_successful=true
         else
@@ -396,15 +414,6 @@ for branch in ${BRANCH_NAME//,/ }; do
   fi
 done
 
-# Create the OpenDelta's builds JSON file
-if ! [ -z "$OPENDELTA_BUILDS_JSON" ]; then
-  echo ">> [$(date)] Creating OpenDelta's builds JSON file (ZIP_DIR/$OPENDELTA_BUILDS_JSON)"
-  if [ "$ZIP_SUBDIR" != true ]; then
-    echo ">> [$(date)] WARNING: OpenDelta requires zip builds separated per device! You should set ZIP_SUBDIR to true"
-  fi
-  /usr/bin/python /root/opendelta_builds_json.py "$ZIP_DIR" -o "$ZIP_DIR/$OPENDELTA_BUILDS_JSON"
-fi
-
 if [ "$DELETE_OLD_LOGS" -gt "0" ]; then
   find "$LOGS_DIR" -maxdepth 1 -name repo-*.log | sort | head -n -$DELETE_OLD_LOGS | xargs -r rm
 fi

src/build_manifest.py

@@ -20,9 +20,9 @@
 import argparse
 
 try:
-    from urllib.request import urlopen
+    from urllib.request import urlopen, Request
 except ImportError:
-    from urllib2 import urlopen
+    from urllib2 import urlopen, Request
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description='Build an Android repo manifest')
@@ -33,7 +33,8 @@
 
     args = parser.parse_args()
 
-    source_manifest = urlopen(args.url).read()
+    request = Request(args.url, headers={"User-Agent": "Mozilla/5.0"})
+    source_manifest = urlopen(request).read()
 
     xmlin = ET.fromstring(source_manifest)
     xmlout = ET.Element("manifest")

src/init.sh

@@ -35,12 +35,12 @@ git config --global user.email $USER_MAIL
 if [ "$SIGN_BUILDS" = true ]; then
   if [ -z "$(ls -A "$KEYS_DIR")" ]; then
     echo ">> [$(date)] SIGN_BUILDS = true but empty \$KEYS_DIR, generating new keys"
-    for c in releasekey platform shared media; do
+    for c in releasekey platform shared media networkstack; do
       echo ">> [$(date)]  Generating $c..."
       /root/make_key "$KEYS_DIR/$c" "$KEYS_SUBJECT" <<< '' &> /dev/null
     done
   else
-    for c in releasekey platform shared media; do
+    for c in releasekey platform shared media networkstack; do
       for e in pk8 x509.pem; do
         if [ ! -f "$KEYS_DIR/$c.$e" ]; then
           echo ">> [$(date)] SIGN_BUILDS = true and not empty \$KEYS_DIR, but \"\$KEYS_DIR/$c.$e\" is missing"

src/location_services_patches/android_frameworks_base-N.patch

@@ -0,0 +1,14 @@
+diff --git a/services/core/java/com/android/server/ServiceWatcher.java b/services/core/java/com/android/server/ServiceWatcher.java
+index 383e25a..31ae918 100644
+--- a/services/core/java/com/android/server/ServiceWatcher.java
++++ b/services/core/java/com/android/server/ServiceWatcher.java
+@@ -92,8 +92,7 @@ public class ServiceWatcher implements ServiceConnection {
+             String pkg = initialPackageNames.get(i);
+             try {
+                 HashSet<Signature> set = new HashSet<Signature>();
+-                Signature[] sigs = pm.getPackageInfo(pkg, PackageManager.MATCH_SYSTEM_ONLY
+-                        | PackageManager.GET_SIGNATURES).signatures;
++                Signature[] sigs = pm.getPackageInfo(pkg, PackageManager.GET_SIGNATURES).signatures;
+                 set.addAll(Arrays.asList(sigs));
+                 sigSets.add(set);
+             } catch (NameNotFoundException e) {

src/location_services_patches/android_frameworks_base-Q.patch

@@ -0,0 +1,16 @@
+diff --git a/services/core/java/com/android/server/ServiceWatcher.java b/services/core/java/com/android/server/ServiceWatcher.java
+index e3dc3b7a984..d72fc64dbcd 100644
+--- a/services/core/java/com/android/server/ServiceWatcher.java
++++ b/services/core/java/com/android/server/ServiceWatcher.java
+@@ -87,9 +87,8 @@ public class ServiceWatcher implements ServiceConnection {
+         ArrayList<HashSet<Signature>> signatureSets = new ArrayList<>(packageNames.length);
+         for (String packageName : packageNames) {
+             try {
+-                Signature[] signatures = pm.getPackageInfo(packageName,
+-                        PackageManager.MATCH_SYSTEM_ONLY
+-                                | PackageManager.GET_SIGNATURES).signatures;
++		Signature[] signatures = pm.getPackageInfo(packageName, PackageManager.GET_SIGNATURES).signatures;
++
+
+                 HashSet<Signature> set = new HashSet<>();
+                 Collections.addAll(set, signatures);