From 3e8c4527c0f8d95594b4b3a0255f819e8c5f5730 Mon Sep 17 00:00:00 2001 From: SolidHal Date: Mon, 3 Feb 2020 13:47:08 -0800 Subject: [PATCH 01/21] update to support lineage 17.1 - fix ccache - fix getting android version - fix getting muppets --- Dockerfile | 3 +++ src/build.sh | 19 ++++++++++++++----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index f7fdf685..d6a7fd85 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,6 +25,9 @@ ENV USER root # WARNING: disabling this may slow down a lot your builds! ENV USE_CCACHE 1 +# We need to specify the ccache binary since it is no longer packaged along with AOSP +ENV CACHE_EXEC /usr/bin/ccache + # ccache maximum size. It should be a number followed by an optional suffix: k, # M, G, T (decimal), Ki, Mi, Gi or Ti (binary). The default suffix is G. Use 0 # for no limit. diff --git a/src/build.sh b/src/build.sh index c6f089fd..19c76315 100755 --- a/src/build.sh +++ b/src/build.sh @@ -124,6 +124,10 @@ for branch in ${BRANCH_NAME//,/ }; do themuppets_branch=lineage-15.1 elif [[ $branch =~ .*lineage-16\.0.* ]]; then themuppets_branch=lineage-16.0 + elif [[ $branch =~ .*lineage-17\.0.* ]]; then + themuppets_branch=lineage-17.0 + elif [[ $branch =~ .*lineage-17\.1.* ]]; then + themuppets_branch=lineage-17.1 else themuppets_branch=lineage-15.1 echo ">> [$(date)] Can't find a matching branch on github.com/TheMuppets, using $themuppets_branch" @@ -137,14 +141,18 @@ for branch in ${BRANCH_NAME//,/ }; do builddate=$(date +%Y%m%d) repo sync -c --force-sync &>> "$repo_log" - android_version=$(sed -n -e 's/^\s*PLATFORM_VERSION\.OPM1 := //p' build/core/version_defaults.mk) + + android_version=$(sed -n -e 's/^\s*PLATFORM_VERSION\.QP1A := //p' build/core/version_defaults.mk) if [ -z $android_version ]; then - android_version=$(sed -n -e 's/^\s*PLATFORM_VERSION\.PPR1 := //p' build/core/version_defaults.mk) + android_version=$(sed -n -e 's/^\s*PLATFORM_VERSION\.OPM1 := //p' build/core/version_defaults.mk) if [ -z $android_version ]; then - android_version=$(sed -n -e 's/^\s*PLATFORM_VERSION := //p' build/core/version_defaults.mk) + android_version=$(sed -n -e 's/^\s*PLATFORM_VERSION\.PPR1 := //p' build/core/version_defaults.mk) if [ -z $android_version ]; then - echo ">> [$(date)] Can't detect the android version" - exit 1 + android_version=$(sed -n -e 's/^\s*PLATFORM_VERSION := //p' build/core/version_defaults.mk) + if [ -z $android_version ]; then + echo ">> [$(date)] Can't detect the android version" + exit 1 + fi fi fi fi @@ -185,6 +193,7 @@ for branch in ${BRANCH_NAME//,/ }; do 7.* ) patch_name="android_frameworks_base-N.patch" ;; 8.* ) patch_name="android_frameworks_base-O.patch" ;; 9* ) patch_name="android_frameworks_base-P.patch" ;; #not sure why 9 not 9.0 but here's a fix that will work until android 90 + 10* ) patch_name="android_frameworks_base-Q.patch" ;; esac if ! [ -z $patch_name ]; then From 64b6d197060e42128ff244720eb1f1601fded60b Mon Sep 17 00:00:00 2001 From: leonghui Date: Sun, 6 Oct 2019 04:40:38 +0000 Subject: [PATCH 02/21] Add keys for networkstack Resolves error FAILED: ninja: 'user-keys/networkstack.x509.pem', needed by 'out/soong/.intermediates/packages/modules/CaptivePortalLogin/CaptivePortalLogin/android_common/CaptivePortalLogin.apk', missing and no known rule to make it See: https://android-review.googlesource.com/q/I7257a472d702e82acdadffc4e0535c63a8bd591d --- src/init.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/init.sh b/src/init.sh index a6e5b033..37461083 100755 --- a/src/init.sh +++ b/src/init.sh @@ -35,12 +35,12 @@ git config --global user.email $USER_MAIL if [ "$SIGN_BUILDS" = true ]; then if [ -z "$(ls -A "$KEYS_DIR")" ]; then echo ">> [$(date)] SIGN_BUILDS = true but empty \$KEYS_DIR, generating new keys" - for c in releasekey platform shared media; do + for c in releasekey platform shared media networkstack; do echo ">> [$(date)] Generating $c..." /root/make_key "$KEYS_DIR/$c" "$KEYS_SUBJECT" <<< '' &> /dev/null done else - for c in releasekey platform shared media; do + for c in releasekey platform shared media networkstack; do for e in pk8 x509.pem; do if [ ! -f "$KEYS_DIR/$c.$e" ]; then echo ">> [$(date)] SIGN_BUILDS = true and not empty \$KEYS_DIR, but \"\$KEYS_DIR/$c.$e\" is missing" From 72e7a84f816c9120c2bf5564c8e9421f70f0c480 Mon Sep 17 00:00:00 2001 From: SolidHal Date: Wed, 5 Feb 2020 14:35:03 -0800 Subject: [PATCH 03/21] Don't include keys in PRODUCT_EXTRA_RECOVERY_KEYS, causes build signing error --- src/build.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/build.sh b/src/build.sh index 19c76315..adbbd0e7 100755 --- a/src/build.sh +++ b/src/build.sh @@ -249,7 +249,13 @@ for branch in ${BRANCH_NAME//,/ }; do echo ">> [$(date)] Adding keys path ($KEYS_DIR)" # Soong (Android 9+) complains if the signing keys are outside the build path ln -sf "$KEYS_DIR" user-keys - sed -i "1s;^;PRODUCT_DEFAULT_DEV_CERTIFICATE := user-keys/releasekey\nPRODUCT_OTA_PUBLIC_KEYS := user-keys/releasekey\nPRODUCT_EXTRA_RECOVERY_KEYS := user-keys/releasekey\n\n;" "vendor/$vendor/config/common.mk" + if [ "$android_version_major" -lt "10" ]; then + sed -i "1s;^;PRODUCT_DEFAULT_DEV_CERTIFICATE := user-keys/releasekey\nPRODUCT_OTA_PUBLIC_KEYS := user-keys/releasekey\nPRODUCT_EXTRA_RECOVERY_KEYS := user-keys/releasekey\n\n;" "vendor/$vendor/config/common.mk" + fi + + if [ "$android_version_major" -ge "10" ]; then + sed -i "1s;^;PRODUCT_DEFAULT_DEV_CERTIFICATE := user-keys/releasekey\nPRODUCT_OTA_PUBLIC_KEYS := user-keys/releasekey\n\n;" "vendor/$vendor/config/common.mk" + fi fi # Prepare the environment From 9a0b702b420bafb236729337baacc6e21f14c45d Mon Sep 17 00:00:00 2001 From: SolidHal Date: Wed, 5 Feb 2020 14:36:52 -0800 Subject: [PATCH 04/21] breakfast before userscripts so that all repos are available for patching --- src/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/build.sh b/src/build.sh index adbbd0e7..35685153 100755 --- a/src/build.sh +++ b/src/build.sh @@ -264,6 +264,7 @@ for branch in ${BRANCH_NAME//,/ }; do if [ -f /root/userscripts/before.sh ]; then echo ">> [$(date)] Running before.sh" + breakfast $codename /root/userscripts/before.sh fi From 5e5cf823c11445eaafe3b2f8b36f8e2342fcd886 Mon Sep 17 00:00:00 2001 From: SolidHal Date: Wed, 5 Feb 2020 14:42:13 -0800 Subject: [PATCH 05/21] Add android Q sig spoofing patch --- .../android_frameworks_base-Q.patch | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 src/signature_spoofing_patches/android_frameworks_base-Q.patch diff --git a/src/signature_spoofing_patches/android_frameworks_base-Q.patch b/src/signature_spoofing_patches/android_frameworks_base-Q.patch new file mode 100644 index 00000000..09cb74cd --- /dev/null +++ b/src/signature_spoofing_patches/android_frameworks_base-Q.patch @@ -0,0 +1,88 @@ +diff --git a/core/res/AndroidManifest.xml b/core/res/AndroidManifest.xml +index d79a23d5..6cf49ecd 100644 +--- a/core/res/AndroidManifest.xml ++++ b/core/res/AndroidManifest.xml +@@ -2637,6 +2637,13 @@ + android:description="@string/permdesc_getPackageSize" + android:protectionLevel="normal" /> + ++ ++ ++ + +diff --git a/core/res/res/values/config.xml b/core/res/res/values/config.xml +index 0db2e140..4a42e50d 100644 +--- a/core/res/res/values/config.xml ++++ b/core/res/res/values/config.xml +@@ -1856,6 +1856,8 @@ + + + com.android.location.fused ++ ++ com.google.android.gms + + + +diff --git a/core/res/res/values/strings.xml b/core/res/res/values/strings.xml +index 152b131a..f8d87a37 100644 +--- a/core/res/res/values/strings.xml ++++ b/core/res/res/values/strings.xml +@@ -827,6 +827,11 @@ + + + ++ ++ Spoof package signature ++ ++ Allows the app to pretend to be a different app. Malicious applications might be able to use this to access private application data. Legitimate uses include an emulator pretending to be what it emulates. Grant this permission with caution only! ++ + + disable or modify status bar + +diff --git a/services/core/java/com/android/server/pm/PackageManagerService.java b/services/core/java/com/android/server/pm/PackageManagerService.java +index 1b5a03b0..0f4f97ab 100644 +--- a/services/core/java/com/android/server/pm/PackageManagerService.java ++++ b/services/core/java/com/android/server/pm/PackageManagerService.java +@@ -4105,8 +4105,9 @@ public class PackageManagerService extends IPackageManager.Stub + final Set permissions = ArrayUtils.isEmpty(p.requestedPermissions) + ? Collections.emptySet() : permissionsState.getPermissions(userId); + +- PackageInfo packageInfo = PackageParser.generatePackageInfo(p, gids, flags, +- ps.firstInstallTime, ps.lastUpdateTime, permissions, state, userId); ++ PackageInfo packageInfo = mayFakeSignature(p, PackageParser.generatePackageInfo(p, gids, flags, ++ ps.firstInstallTime, ps.lastUpdateTime, permissions, state, userId), ++ permissions); + + if (packageInfo == null) { + return null; +@@ -4142,6 +4143,24 @@ public class PackageManagerService extends IPackageManager.Stub + } + } + ++ private PackageInfo mayFakeSignature(PackageParser.Package p, PackageInfo pi, ++ Set permissions) { ++ try { ++ if (permissions.contains("android.permission.FAKE_PACKAGE_SIGNATURE") ++ && p.applicationInfo.targetSdkVersion > Build.VERSION_CODES.LOLLIPOP_MR1 ++ && p.mAppMetaData != null) { ++ String sig = p.mAppMetaData.getString("fake-signature"); ++ if (sig != null) { ++ pi.signatures = new Signature[] {new Signature(sig)}; ++ } ++ } ++ } catch (Throwable t) { ++ // We should never die because of any failures, this is system code! ++ Log.w("PackageManagerService.FAKE_PACKAGE_SIGNATURE", t); ++ } ++ return pi; ++ } ++ + @Override + public void checkPackageStartable(String packageName, int userId) { + final int callingUid = Binder.getCallingUid(); From 009cbfa3f14d030ce5088bf6133b640d690f37e9 Mon Sep 17 00:00:00 2001 From: SolidHal Date: Wed, 5 Feb 2020 14:54:42 -0800 Subject: [PATCH 06/21] Add env var to extract the boot image --- Dockerfile | 3 +++ src/build.sh | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/Dockerfile b/Dockerfile index d6a7fd85..ef153aa5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -70,6 +70,9 @@ ENV CLEAN_OUTDIR false # For example, '0 10 * * *' means 'Every day at 10:00 UTC' ENV CRONTAB_TIME 'now' +# Put the boot.img in the zips directory +ENV BOOT_IMG false + # Clean artifacts output after each build ENV CLEAN_AFTER_BUILD true diff --git a/src/build.sh b/src/build.sh index 35685153..b6c3258f 100755 --- a/src/build.sh +++ b/src/build.sh @@ -353,6 +353,10 @@ for branch in ${BRANCH_NAME//,/ }; do sha256sum "$build" > "$ZIP_DIR/$zipsubdir/$build.sha256sum" done find . -maxdepth 1 -name 'lineage-*.zip*' -type f -exec mv {} "$ZIP_DIR/$zipsubdir/" \; &>> "$DEBUG_LOG" + + if [ "$BOOT_IMG" = true ]; then + find . -maxdepth 1 -name 'boot.img' -type f -exec mv {} "$ZIP_DIR/$zipsubdir/" \; &>> "$DEBUG_LOG" + fi cd "$source_dir" build_successful=true else From 510c2dd91c7eb3214095af2d159987fd55c1cbc1 Mon Sep 17 00:00:00 2001 From: SolidHal Date: Wed, 5 Feb 2020 20:12:47 -0800 Subject: [PATCH 07/21] Move before script and breakfast to inside the device loop so $codename can be used --- src/build.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/build.sh b/src/build.sh index b6c3258f..48bfa56a 100755 --- a/src/build.sh +++ b/src/build.sh @@ -262,15 +262,15 @@ for branch in ${BRANCH_NAME//,/ }; do echo ">> [$(date)] Preparing build environment" source build/envsetup.sh > /dev/null - if [ -f /root/userscripts/before.sh ]; then - echo ">> [$(date)] Running before.sh" - breakfast $codename - /root/userscripts/before.sh - fi - for codename in ${devices//,/ }; do if ! [ -z "$codename" ]; then + if [ -f /root/userscripts/before.sh ]; then + echo ">> [$(date)] Running before.sh" + breakfast $codename + /root/userscripts/before.sh + fi + currentdate=$(date +%Y%m%d) if [ "$builddate" != "$currentdate" ]; then # Sync the source code From 0e79af958dbf3e0639e600f127d4c276db0c738f Mon Sep 17 00:00:00 2001 From: SolidHal Date: Sat, 8 Feb 2020 21:25:16 -0800 Subject: [PATCH 08/21] Add in support for unifiednlp patch --- README.md | 10 ++++++++++ src/build.sh | 19 +++++++++++++++++++ .../android_frameworks_base-N.patch | 14 ++++++++++++++ .../android_frameworks_base-Q.patch | 16 ++++++++++++++++ 4 files changed, 59 insertions(+) create mode 100644 src/location_services_patches/android_frameworks_base-N.patch create mode 100644 src/location_services_patches/android_frameworks_base-Q.patch diff --git a/README.md b/README.md index 5bf46da7..4a3a5793 100644 --- a/README.md +++ b/README.md @@ -76,11 +76,21 @@ The signature spoofing patch can be optionally included with: If in doubt, use `restricted`: note that packages that requires the FAKE_SIGNATURE permission must be embedded in the build by adding them in +### UnifiedNLP support + +If you enable signature spoofing and want to use unifiednlp from microg, the patche +to support is can be enabled using: + * `UNIFIEDNLP_SUPPORT` (false): `true` to patch in unifiednlp support + +### Custom Packages + * `CUSTOM_PACKAGES` Extra packages can be included in the tree by adding the corresponding manifest XML to the local_manifests volume. + + ### Proprietary files Some proprietary files are needed to create a LineageOS build, but they're not diff --git a/src/build.sh b/src/build.sh index 48bfa56a..950f99db 100755 --- a/src/build.sh +++ b/src/build.sh @@ -218,6 +218,25 @@ for branch in ${BRANCH_NAME//,/ }; do fi fi + if [ "$SUPPORT_UNIFIEDNLP" = "true" ]; then + # Determine which patch should be applied to the current Android source tree + patch_name="" + case $android_version in + 10* ) patch_name="android_frameworks_base-Q.patch" ;; + esac + + if ! [ -z $patch_name ]; then + cd frameworks/base + echo ">> [$(date)] Applying location services patch" + patch --quiet -p1 -i "/root/location_services_patches/$patch_name" + git clean -q -f + cd ../.. + else + echo ">> [$(date)] ERROR: can't find a unifiednlp support patch for the current Android version ($android_version)" + exit 1 + fi + fi + echo ">> [$(date)] Setting \"$RELEASE_TYPE\" as release type" sed -i "/\$(filter .*\$(${vendor^^}_BUILDTYPE)/,+2d" "vendor/$vendor/config/common.mk" diff --git a/src/location_services_patches/android_frameworks_base-N.patch b/src/location_services_patches/android_frameworks_base-N.patch new file mode 100644 index 00000000..df410e07 --- /dev/null +++ b/src/location_services_patches/android_frameworks_base-N.patch @@ -0,0 +1,14 @@ +diff --git a/services/core/java/com/android/server/ServiceWatcher.java b/services/core/java/com/android/server/ServiceWatcher.java +index 383e25a..31ae918 100644 +--- a/services/core/java/com/android/server/ServiceWatcher.java ++++ b/services/core/java/com/android/server/ServiceWatcher.java +@@ -92,8 +92,7 @@ public class ServiceWatcher implements ServiceConnection { + String pkg = initialPackageNames.get(i); + try { + HashSet set = new HashSet(); +- Signature[] sigs = pm.getPackageInfo(pkg, PackageManager.MATCH_SYSTEM_ONLY +- | PackageManager.GET_SIGNATURES).signatures; ++ Signature[] sigs = pm.getPackageInfo(pkg, PackageManager.GET_SIGNATURES).signatures; + set.addAll(Arrays.asList(sigs)); + sigSets.add(set); + } catch (NameNotFoundException e) { diff --git a/src/location_services_patches/android_frameworks_base-Q.patch b/src/location_services_patches/android_frameworks_base-Q.patch new file mode 100644 index 00000000..30a956b7 --- /dev/null +++ b/src/location_services_patches/android_frameworks_base-Q.patch @@ -0,0 +1,16 @@ +diff --git a/services/core/java/com/android/server/ServiceWatcher.java b/services/core/java/com/android/server/ServiceWatcher.java +index e3dc3b7a984..d72fc64dbcd 100644 +--- a/services/core/java/com/android/server/ServiceWatcher.java ++++ b/services/core/java/com/android/server/ServiceWatcher.java +@@ -87,9 +87,8 @@ public class ServiceWatcher implements ServiceConnection { + ArrayList> signatureSets = new ArrayList<>(packageNames.length); + for (String packageName : packageNames) { + try { +- Signature[] signatures = pm.getPackageInfo(packageName, +- PackageManager.MATCH_SYSTEM_ONLY +- | PackageManager.GET_SIGNATURES).signatures; ++ Signature[] signatures = pm.getPackageInfo(packageName, PackageManager.GET_SIGNATURES).signatures; ++ + + HashSet set = new HashSet<>(); + Collections.addAll(set, signatures); From e1548e0f9025cb3f7c97ccccdcb90f62d3f5eb9a Mon Sep 17 00:00:00 2001 From: SolidHal Date: Sat, 8 Feb 2020 21:28:10 -0800 Subject: [PATCH 09/21] Add readme entry for BOOT_IMG env var --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 5bf46da7..02697411 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,8 @@ Other useful settings are: mirror of the LineageOS source (> 200 GB) * `CRONTAB_TIME (now)`: instead of building immediately and exit, build at the specified time (uses standard cron format) + * `BOOT_IMG (false)`: copy the build boot.img into the zips folder. This is useful + as you may need to flash it first if your device doesn't support custom recoveries The full list of settings, including the less interesting ones not mentioned in this guide, can be found in the [Dockerfile][dockerfile]. From ad8c85400fdfca089bfd455c1b7b2a23d96ae56f Mon Sep 17 00:00:00 2001 From: SolidHal Date: Sat, 8 Feb 2020 21:30:54 -0800 Subject: [PATCH 10/21] Add default unifiednlp support value --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index ef153aa5..6874c09d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -108,6 +108,9 @@ ENV LOGS_SUBDIR true # example. ENV SIGNATURE_SPOOFING "no" +# Apply the microG unifiedNLP patch +ENV SUPPORT_UNIFIEDNLP false + # Generate delta files ENV BUILD_DELTA false From ea1282f3a5b66e73301e7e2abceff75954c094ff Mon Sep 17 00:00:00 2001 From: SolidHal Date: Wed, 18 Mar 2020 15:11:01 -0700 Subject: [PATCH 11/21] Fix type in ccache exec path --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6874c09d..1a77a5a8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,7 +26,7 @@ ENV USER root ENV USE_CCACHE 1 # We need to specify the ccache binary since it is no longer packaged along with AOSP -ENV CACHE_EXEC /usr/bin/ccache +ENV CCACHE_EXEC /usr/bin/ccache # ccache maximum size. It should be a number followed by an optional suffix: k, # M, G, T (decimal), Ki, Mi, Gi or Ti (binary). The default suffix is G. Use 0 From 2346c004461aa861621506c77c6fe0f851b4316b Mon Sep 17 00:00:00 2001 From: SolidHal Date: Fri, 20 Mar 2020 10:56:21 -0700 Subject: [PATCH 12/21] Add info for WITH_SU flag on lineage-17.1 --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 6ac0fa9f..6d38ffbf 100644 --- a/README.md +++ b/README.md @@ -133,6 +133,8 @@ Other useful settings are: * `WITH_SU (false)`: set to `true` to embed `su` in the build (note that, even when set to `false`, you can still enable root by flashing the [su installable ZIP][los-extras]) + NOTE: in lineageos 17.1 the WITH_SU flag was dropped, as it did not functioning properly in Android 10 + The suggested replacement is Magisk https://www.xda-developers.com/lineageos-dropping-superuser-addonsu-implementation-favor-magisk-manager/ * `RELEASE_TYPE (UNOFFICIAL)`: change the release type of your builds * `BUILD_OVERLAY (false)`: normally each build is done on the source tree, then the tree is cleaned with `mka clean`. If you want to be sure that each build From 7107f639b01d17a13eabeb901134988e18b34550 Mon Sep 17 00:00:00 2001 From: SolidHal Date: Mon, 30 Mar 2020 19:09:37 -0700 Subject: [PATCH 13/21] Update README to include how to build in fork of microg for lineageos 17.1 --- README.md | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/README.md b/README.md index 6d38ffbf..e772f3a0 100644 --- a/README.md +++ b/README.md @@ -173,6 +173,57 @@ When `BUILD_OVERLAY` is `true` When `LOCAL_MIRROR` is `true`: * `/srv/mirror`, for the LineageOS mirror + + +## Including MicroG and other packages + +The microG and FDroid packages are not present in the LineageOS repositories, +and must be provided through an XML in the `/home/user/manifests`. + +#### LineageOS < 17.1 + +[This][prebuiltapks] repo contains some of the most common packages for these +kind of builds for lineageOS < 17.1: to include it create an XML (the name is irrelevant, as long as +it ends with `.xml`) in the `/home/user/manifests` folder with this content: + +``` + + + + +``` + +#### LineageOS 17.1 +Some of the packages in the prebuiltapks repo are not updated for Android 10/ LineageOS 17.1 +An example of this is the MicroG apk. +Instead of getting prebuilts, we can build it ourselves. To have it built, create an XML (the name is irrelevant, as long as +it ends with `.xml`) in the `/home/user/manifests` folder with this content: + +``` + + + + + +``` + +NOTE: This specifies a fork of MicroG with patches to fix UnifiedNLP on LineageOS 17.1 +When these patches are merged into the main repo, the following can be used instead: + +``` + + + + + +``` + +Then include `GmsCore` in the `CUSTOM_PACKAGES` environment variable like this: +``` + -e "CUSTOM_PACKAGES=GmsCore" \ +``` ## Examples From d040c9e3ae52206a4006b1a16bd955be22acac23 Mon Sep 17 00:00:00 2001 From: SolidHal Date: Tue, 31 Mar 2020 13:33:59 -0700 Subject: [PATCH 14/21] Don't redefine the github remote --- README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.md b/README.md index e772f3a0..ece2773e 100644 --- a/README.md +++ b/README.md @@ -202,8 +202,6 @@ it ends with `.xml`) in the `/home/user/manifests` folder with this content: ``` - ``` @@ -214,8 +212,6 @@ When these patches are merged into the main repo, the following can be used inst ``` - ``` From c229eb72d976d456226974838ded909c02a67be3 Mon Sep 17 00:00:00 2001 From: SolidHal Date: Wed, 1 Apr 2020 16:23:50 -0700 Subject: [PATCH 15/21] Suggest using prebuilts provided by OmniRom for now, until new builds with Android 10 patches are available in the prebuilts repos --- README.md | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index ece2773e..bd739f5f 100644 --- a/README.md +++ b/README.md @@ -175,7 +175,7 @@ When `LOCAL_MIRROR` is `true`: * `/srv/mirror`, for the LineageOS mirror -## Including MicroG and other packages +## Including MicroG and other package pre-builts The microG and FDroid packages are not present in the LineageOS repositories, and must be provided through an XML in the `/home/user/manifests`. @@ -196,27 +196,16 @@ it ends with `.xml`) in the `/home/user/manifests` folder with this content: #### LineageOS 17.1 Some of the packages in the prebuiltapks repo are not updated for Android 10/ LineageOS 17.1 An example of this is the MicroG apk. -Instead of getting prebuilts, we can build it ourselves. To have it built, create an XML (the name is irrelevant, as long as -it ends with `.xml`) in the `/home/user/manifests` folder with this content: ``` - + ``` -NOTE: This specifies a fork of MicroG with patches to fix UnifiedNLP on LineageOS 17.1 -When these patches are merged into the main repo, the following can be used instead: - -``` - - - - -``` -Then include `GmsCore` in the `CUSTOM_PACKAGES` environment variable like this: +Then include `GmsCore` or any other pre-built app name in the `CUSTOM_PACKAGES` environment variable like this: ``` -e "CUSTOM_PACKAGES=GmsCore" \ ``` From bb3816b9d8db0cfaae5a99d0eff729a55e72d734 Mon Sep 17 00:00:00 2001 From: SolidHal Date: Fri, 3 Apr 2020 13:46:16 -0700 Subject: [PATCH 16/21] use fork of prebuilt apps to remove the MICROG build flag requirement --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index bd739f5f..9e130a26 100644 --- a/README.md +++ b/README.md @@ -200,12 +200,12 @@ An example of this is the MicroG apk. ``` - + ``` -Then include `GmsCore` or any other pre-built app name in the `CUSTOM_PACKAGES` environment variable like this: +Then include `GmsCore` and/or any other pre-built app name in the `CUSTOM_PACKAGES` environment variable like this: ``` -e "CUSTOM_PACKAGES=GmsCore" \ ``` From 4b10fb288f9752a70495032e5284de092832a35e Mon Sep 17 00:00:00 2001 From: SolidHal Date: Mon, 20 Apr 2020 18:36:28 -0700 Subject: [PATCH 17/21] increase the java vm heap memory limit, not doing this breaks builds for some devices --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index 1a77a5a8..bd8ad124 100644 --- a/Dockerfile +++ b/Dockerfile @@ -129,6 +129,9 @@ ENV DELETE_OLD_LOGS 0 # Requires ZIP_SUBDIR. ENV OPENDELTA_BUILDS_JSON '' +# set the java tool options max memory size +ENV JAVA_TOOL_OPTIONS "-Xmx6g" + # You can optionally specify a USERSCRIPTS_DIR volume containing these scripts: # * begin.sh, run at the very beginning # * before.sh, run after the syncing and patching, before starting the builds From 2095f4e0a3e07d639da02210b669787aac1b5cee Mon Sep 17 00:00:00 2001 From: SolidHal Date: Mon, 20 Apr 2020 18:38:50 -0700 Subject: [PATCH 18/21] remove opendelta support, opendelta seems to have been dropped from omnirom --- Dockerfile | 18 ------------- src/build.sh | 30 ---------------------- src/opendelta_builds_json.py | 49 ------------------------------------ 3 files changed, 97 deletions(-) delete mode 100755 src/opendelta_builds_json.py diff --git a/Dockerfile b/Dockerfile index bd8ad124..309e2c9c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -186,24 +186,6 @@ RUN apt-get install -y bc bison bsdmainutils build-essential ccache cgpt cron \ RUN curl https://storage.googleapis.com/git-repo-downloads/repo > /usr/local/bin/repo RUN chmod a+x /usr/local/bin/repo -# Download and build delta tools -################################ -RUN cd /root/ && \ - mkdir delta && \ - git clone --depth=1 https://github.com/omnirom/android_packages_apps_OpenDelta.git OpenDelta && \ - gcc -o delta/zipadjust OpenDelta/jni/zipadjust.c OpenDelta/jni/zipadjust_run.c -lz && \ - cp OpenDelta/server/minsignapk.jar OpenDelta/server/opendelta.sh delta/ && \ - chmod +x delta/opendelta.sh && \ - rm -rf OpenDelta/ && \ - sed -i -e 's|^\s*HOME=.*|HOME=/root|; \ - s|^\s*BIN_XDELTA=.*|BIN_XDELTA=xdelta3|; \ - s|^\s*FILE_MATCH=.*|FILE_MATCH=lineage-\*.zip|; \ - s|^\s*PATH_CURRENT=.*|PATH_CURRENT=$SRC_DIR/out/target/product/$DEVICE|; \ - s|^\s*PATH_LAST=.*|PATH_LAST=$SRC_DIR/delta_last/$DEVICE|; \ - s|^\s*KEY_X509=.*|KEY_X509=$KEYS_DIR/releasekey.x509.pem|; \ - s|^\s*KEY_PK8=.*|KEY_PK8=$KEYS_DIR/releasekey.pk8|; \ - s|publish|$DELTA_DIR|g' /root/delta/opendelta.sh - # Set the work directory ######################## WORKDIR $SRC_DIR diff --git a/src/build.sh b/src/build.sh index 950f99db..9480365e 100755 --- a/src/build.sh +++ b/src/build.sh @@ -344,27 +344,6 @@ for branch in ${BRANCH_NAME//,/ }; do find out/target/product/$codename -maxdepth 1 -name "lineage-*-$currentdate-*.zip*" -type f -exec sh /root/fix_build_date.sh {} $currentdate $builddate \; &>> "$DEBUG_LOG" fi - if [ "$BUILD_DELTA" = true ]; then - if [ -d "delta_last/$codename/" ]; then - # If not the first build, create delta files - echo ">> [$(date)] Generating delta files for $codename" | tee -a "$DEBUG_LOG" - cd /root/delta - if ./opendelta.sh $codename &>> "$DEBUG_LOG"; then - echo ">> [$(date)] Delta generation for $codename completed" | tee -a "$DEBUG_LOG" - else - echo ">> [$(date)] Delta generation for $codename failed" | tee -a "$DEBUG_LOG" - fi - if [ "$DELETE_OLD_DELTAS" -gt "0" ]; then - /usr/bin/python /root/clean_up.py -n $DELETE_OLD_DELTAS -V $los_ver -N 1 "$DELTA_DIR/$codename" &>> $DEBUG_LOG - fi - cd "$source_dir" - else - # If the first build, copy the current full zip in $source_dir/delta_last/$codename/ - echo ">> [$(date)] No previous build for $codename; using current build as base for the next delta" | tee -a "$DEBUG_LOG" - mkdir -p delta_last/$codename/ &>> "$DEBUG_LOG" - find out/target/product/$codename -maxdepth 1 -name 'lineage-*.zip' -type f -exec cp {} "$source_dir/delta_last/$codename/" \; &>> "$DEBUG_LOG" - fi - fi # Move produced ZIP files to the main OUT directory echo ">> [$(date)] Moving build artifacts for $codename to '$ZIP_DIR/$zipsubdir'" | tee -a "$DEBUG_LOG" cd out/target/product/$codename @@ -435,15 +414,6 @@ for branch in ${BRANCH_NAME//,/ }; do fi done -# Create the OpenDelta's builds JSON file -if ! [ -z "$OPENDELTA_BUILDS_JSON" ]; then - echo ">> [$(date)] Creating OpenDelta's builds JSON file (ZIP_DIR/$OPENDELTA_BUILDS_JSON)" - if [ "$ZIP_SUBDIR" != true ]; then - echo ">> [$(date)] WARNING: OpenDelta requires zip builds separated per device! You should set ZIP_SUBDIR to true" - fi - /usr/bin/python /root/opendelta_builds_json.py "$ZIP_DIR" -o "$ZIP_DIR/$OPENDELTA_BUILDS_JSON" -fi - if [ "$DELETE_OLD_LOGS" -gt "0" ]; then find "$LOGS_DIR" -maxdepth 1 -name repo-*.log | sort | head -n -$DELETE_OLD_LOGS | xargs -r rm fi diff --git a/src/opendelta_builds_json.py b/src/opendelta_builds_json.py deleted file mode 100755 index 9ae338ea..00000000 --- a/src/opendelta_builds_json.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python - -# Copyright (C) 2017 Nicola Corna -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from sys import argv -from argparse import ArgumentParser -import os -import json - -if __name__ == "__main__": - parser = ArgumentParser(description='Generate an OpenDelta\'s builds.json ' - 'file') - parser.add_argument('path', metavar='PATH', type=str, help='the directory ' - 'containing the zips') - parser.add_argument('-o', "--output", type=str, help='output file; ' - 'if unspecified, print to stdout') - args = parser.parse_args() - - data = {} - builddirs = ['./' + s for s in os.listdir(args.path)] - for builddir in builddirs: - try: - builds = os.listdir(os.path.join(args.path, builddir)) - data[builddir] = [dict() for x in range(len(builds))] - for i in range(0, len(builds)): - data[builddir][i]["filename"] = builds[i] - data[builddir][i]["timestamp"] = int(os.path.getmtime( - os.path.join(args.path, builddir, builds[i]))) - except OSError: - pass - - if args.output: - with open(args.output, "w") as f: - f.write(json.dumps(data, separators=(',',':'))) - else: - print(json.dumps(data, separators=(',',':'))) From 77f20ef9b484e3f500246884a81153a497d0ecec Mon Sep 17 00:00:00 2001 From: Evan Goode Date: Sat, 11 Apr 2020 10:14:54 -0400 Subject: [PATCH 19/21] Send some User-Agent other than "Python-urllib" Looks like GitLab is using CloudFlare to block requests of raw files from user agents matching "Python-urllib*". So we'll just send a generic browser user agent instead. --- src/build_manifest.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/build_manifest.py b/src/build_manifest.py index d24af590..32f9411a 100755 --- a/src/build_manifest.py +++ b/src/build_manifest.py @@ -20,9 +20,9 @@ import argparse try: - from urllib.request import urlopen + from urllib.request import urlopen, Request except ImportError: - from urllib2 import urlopen + from urllib2 import urlopen, Request if __name__ == "__main__": parser = argparse.ArgumentParser(description='Build an Android repo manifest') @@ -33,7 +33,8 @@ args = parser.parse_args() - source_manifest = urlopen(args.url).read() + request = Request(args.url, headers={"User-Agent": "Mozilla/5.0"}) + source_manifest = urlopen(request).read() xmlin = ET.fromstring(source_manifest) xmlout = ET.Element("manifest") From a6724344899f78091e600d16c85297bdfaf15889 Mon Sep 17 00:00:00 2001 From: SolidHal Date: Sun, 2 Aug 2020 10:21:34 -0700 Subject: [PATCH 20/21] initial new microg build --- Dockerfile | 7 ++++++- README.md | 13 +++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 309e2c9c..8c7c1090 100644 --- a/Dockerfile +++ b/Dockerfile @@ -130,7 +130,8 @@ ENV DELETE_OLD_LOGS 0 ENV OPENDELTA_BUILDS_JSON '' # set the java tool options max memory size -ENV JAVA_TOOL_OPTIONS "-Xmx6g" +# not sure yet if necessary or helpful +# ENV JAVA_TOOL_OPTIONS "-Xmx6g" # You can optionally specify a USERSCRIPTS_DIR volume containing these scripts: # * begin.sh, run at the very beginning @@ -183,6 +184,10 @@ RUN apt-get install -y bc bison bsdmainutils build-essential ccache cgpt cron \ libxml2-utils lsof lzop maven openjdk-8-jdk pngcrush procps python rsync \ schedtool squashfs-tools wget xdelta3 xsltproc yasm zip zlib1g-dev +#required for opengapps pico, uncomment to use +# RUN curl http://ftp.us.debian.org/debian/pool/main/g/git-lfs/git-lfs_2.7.1-1~bpo9+2_amd64.deb -o git-lfs.deb && dpkg -i git-lfs.deb +# RUN git lfs install + RUN curl https://storage.googleapis.com/git-repo-downloads/repo > /usr/local/bin/repo RUN chmod a+x /usr/local/bin/repo diff --git a/README.md b/README.md index 9e130a26..586615d0 100644 --- a/README.md +++ b/README.md @@ -194,20 +194,25 @@ it ends with `.xml`) in the `/home/user/manifests` folder with this content: ``` #### LineageOS 17.1 -Some of the packages in the prebuiltapks repo are not updated for Android 10/ LineageOS 17.1 -An example of this is the MicroG apk. +Some (most) of the packages in the prebuiltapks repo are not updated for Android 10/ LineageOS 17.1 +An example of this is the MicroG apk, and the fdroid extension. ``` - + + + + ``` Then include `GmsCore` and/or any other pre-built app name in the `CUSTOM_PACKAGES` environment variable like this: ``` - -e "CUSTOM_PACKAGES=GmsCore" \ + -e "CUSTOM_PACKAGES=GmsCore F-DroidPrivilegedExtension" \ ``` ## Examples From 306f66b1798750392d38d42f97063d00252b52d7 Mon Sep 17 00:00:00 2001 From: SolidHal Date: Sun, 2 Aug 2020 11:36:08 -0700 Subject: [PATCH 21/21] don't suggest building in microg --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 586615d0..524e4b88 100644 --- a/README.md +++ b/README.md @@ -205,14 +205,14 @@ An example of this is the MicroG apk, and the fdroid extension. name="privileged-extension.git" remote="fdroid" revision="refs/tags/0.2.11" /> - ``` +NOTE: due to https://github.com/microg/android_packages_apps_GmsCore/issues/1059 including microg in the build is unsupported at this time +get the apks from github and install them manually, or add the microg fdroid repo to fdroid. - -Then include `GmsCore` and/or any other pre-built app name in the `CUSTOM_PACKAGES` environment variable like this: +Then include any other pre-built app name in the `CUSTOM_PACKAGES` environment variable like this: ``` - -e "CUSTOM_PACKAGES=GmsCore F-DroidPrivilegedExtension" \ + -e "CUSTOM_PACKAGES=F-DroidPrivilegedExtension" \ ``` ## Examples