Implement support for resource_sets API in bazel-common (#132)

Fixes #132
grab · Nov 11, 2024 · 94b4c87 · 94b4c87
1 parent 6d44b7f
commit 94b4c87
Show file tree

Hide file tree

Showing 15 changed files with 276 additions and 153 deletions.
diff --git a/.bazel/.default.bazelrc b/.bazel/.default.bazelrc
@@ -1,12 +1,15 @@
 # BzlMod
 common --enable_bzlmod=false
 
-# Cache
+# Cache setup
 common --disk_cache=bazel-cache
+common --remote_cache_compression=true
+common --remote_upload_local_results=false # Don't upload results by default
 startup --digest_function=blake3
-common --experimental_action_cache_store_output_metadata
+common --experimental_remote_cache_eviction_retries=10
 common --modify_execution_info=JavaDeployJar=+no-remote-cache # Don't cache Java deploy jar which is huge in size
 common --remote_cache_compression=true
+common --experimental_disk_cache_gc_max_size=4G
 
 # Error config
 common --verbose_failures
@@ -16,16 +19,12 @@ common --incompatible_strict_action_env
 common --reuse_sandbox_directories
 common --repo_env=RJE_VERBOSE=true
 
-# Resources
+# JVM External rules
+common --@rules_jvm_external//settings:stamp_manifest=False
+
 # Resources
 common --local_resources=cpu=HOST_CPUS*.75 # Android actions start their own threads which can overwhelm the system
 
-# Workers
-common --worker_verbose
-common --experimental_worker_multiplex
-common --experimental_shrink_worker_pool
-common --experimental_worker_for_repo_fetching=platform
-
 # JAVA - START
 common --experimental_strict_java_deps=off # Turn off strict java deps
 common --java_runtime_version=remotejdk_17 # Use inbuilt Java 17 for hermeticity
@@ -39,6 +38,7 @@ common --experimental_java_header_input_pruning
 # JAVA - END
 
 # Android
+common --fat_apk_cpu=arm64-v8a,x86
 common --experimental_google_legacy_api
 common --noincompatible_enable_android_toolchain_resolution
 common --noincompatible_enable_cc_toolchain_resolution
@@ -54,30 +54,53 @@ common --output_library_merged_assets=false # Turn off asset merging artifact
 # common --experimental_use_package_aware_rtxt=true # Use package aware R.txt files (required for databinding)
 # common --define=nontransitive_r_class=1 # Adapt bazel common rules for non transitive R class
 
+# Workers
+common --worker_verbose
+common --worker_multiplex
+common --experimental_shrink_worker_pool
+common --experimental_worker_for_repo_fetching=platform
+common --experimental_collect_worker_data_in_profiler
+
+## Android Resource Workers
 common --experimental_persistent_aar_extractor
-common --persistent_multiplex_android_tools
 common --persistent_android_dex_desugar
+common --persistent_android_resource_processor
+common --persistent_android_dex_desugar
+common --persistent_multiplex_android_dex_desugar
+common --persistent_multiplex_android_resource_processor
+common --persistent_multiplex_android_tools
+
+# Action Strategies
+common --strategy=AARGenerator=worker
 common --strategy=DatabindingStubs=worker
-common --worker_max_instances=Javac=1
-common --worker_max_instances=KotlinCompile=1
-common --worker_max_instances=KotlinKapt=1
+common --modify_execution_info=GenerateDataBindingBaseClasses=+supports-multiplex-workers=1
+## Java Workers
+common --strategy=KotlinCompile=worker
+common --strategy=Javac=worker
+# common --strategy=Turbine=worker
+## Worker configuration to avoid CPU thrashing https://github.com/bazelbuild/bazel/issues/8586#issuecomment-500070549
+common --worker_max_instances=Aapt2Optimize=1
 common --worker_max_instances=AaptPackage=1
-common --worker_max_instances=AndroidResourceParser=1
+common --worker_max_instances=AndroidAapt2=1
+common --worker_max_instances=AndroidAssetMerger=1
+common --worker_max_instances=AndroidCompiledResourceMerger=1
 common --worker_max_instances=AndroidResourceCompiler=1
+common --worker_max_instances=AndroidResourceMerger=1
+common --worker_max_instances=AndroidResourceParser=1
 common --worker_max_instances=AndroidResourceValidator=1
 common --worker_max_instances=AndroidLintAnalyze=1
 common --worker_max_instances=AndroidLint=1
-common --worker_max_instances=RClassGenerator=1
-common --worker_max_instances=AndroidAapt2=1
-common --worker_max_instances=AndroidAssetMerger=1
-common --worker_max_instances=AndroidResourceMerger=1
-common --worker_max_instances=AndroidCompiledResourceMerger=1
-common --worker_max_instances=Aapt2Optimize=1
+common --worker_max_instances=BuildConfigGenerationWorker=1
 common --worker_max_instances=DatabindingStubs=1
-common --worker_max_instances=GenerateDataBindingBaseClasses=1
-common --worker_max_instances=DexBuilder=1
+common --worker_max_instances=DatabindingWorker=1
 common --worker_max_instances=Desugar=1
-
-# JVM
-common --strategy=KotlinCompile=worker
-common --strategy=Javac=worker
+common --worker_max_instances=DexBuilder=1
+common --worker_max_instances=GenerateDataBindingBaseClasses=1
+common --worker_max_instances=Javac=1
+common --worker_max_instances=JdepsMerge=1
+common --worker_max_instances=KotlinCompile=1
+common --worker_max_instances=KotlinKapt=1
+common --worker_max_instances=MergeSourceSets=1
+common --worker_max_instances=ProcessDatabinding=1
+common --worker_max_instances=RClassGenerator=1
+common --worker_max_instances=Turbine=1
diff --git a/.bazelversion b/.bazelversion
@@ -1 +1 @@
-7.2.0
+7.4.0
diff --git a/build.gradle b/build.gradle
@@ -155,7 +155,7 @@ idea {
     module {
         excludeDirs += [
             file("bazel-bin"),
-            file("bazel-grazel"),
+            file("bazel-" + projectDir.name),
             file("bazel-cache"),
             file("bazel-out"),
             file("bazel-testlogs"),

diff --git a/flavors/sample-android-flavor/BUILD.bazel b/flavors/sample-android-flavor/BUILD.bazel
@@ -19,12 +19,17 @@ android_library(
             "generated_value": "generated",
         },
     },
-    resources = {
-        "src/demo/res": {
+    resource_sets = {
+        "demo": {
+            "res": "src/demo/res",
         },
-        "src/free/res": {
+        "free": {
+            "res": "src/free/res",
         },
-        "src/main/res": {
+        "main": {
+            "res": "src/main/res",
+            "assets": "src/main/assets",
+            "manifest": "src/main/AndroidManifest.xml",
         },
     },
     visibility = [
@@ -60,10 +65,14 @@ android_library(
             "generated_value": "generated",
         },
     },
-    resources = {
-        "src/demo/res": {
+    resource_sets = {
+        "demo": {
+            "res": "src/demo/res",
         },
-        "src/main/res": {
+        "main": {
+            "res": "src/main/res",
+            "assets": "src/main/assets",
+            "manifest": "src/main/AndroidManifest.xml",
         },
     },
     visibility = [
@@ -99,12 +108,17 @@ android_library(
             "generated_value": "generated",
         },
     },
-    resources = {
-        "src/full/res": {
+    resource_sets = {
+        "full": {
+            "res": "src/full/res",
         },
-        "src/free/res": {
+        "free": {
+            "res": "src/free/res",
         },
-        "src/main/res": {
+        "main": {
+            "res": "src/main/res",
+            "assets": "src/main/assets",
+            "manifest": "src/main/AndroidManifest.xml",
         },
     },
     visibility = [
@@ -140,10 +154,14 @@ android_library(
             "generated_value": "generated",
         },
     },
-    resources = {
-        "src/full/res": {
+    resource_sets = {
+        "full": {
+            "res": "src/full/res",
         },
-        "src/main/res": {
+        "main": {
+            "res": "src/main/res",
+            "assets": "src/main/assets",
+            "manifest": "src/main/AndroidManifest.xml",
         },
     },
     visibility = [

diff --git a/grazel-gradle-plugin/src/main/kotlin/com/grab/grazel/bazel/rules/AndroidRules.kt b/grazel-gradle-plugin/src/main/kotlin/com/grab/grazel/bazel/rules/AndroidRules.kt
@@ -133,7 +133,7 @@ internal fun StatementsBuilder.androidBinary(
     enableCompose: Boolean = false,
     visibility: Visibility = Visibility.Public,
     resourceFiles: List<Assignee> = emptyList(),
-    resources: Assignee? = null,
+    resourceSets: Assignee? = null,
     resValuesData: ResValuesData,
     deps: List<BazelDependency>,
     assetsGlob: List<String> = emptyList(),
@@ -172,7 +172,7 @@ internal fun StatementsBuilder.androidBinary(
         resConfigFilters.notEmpty {
             "resource_configuration_filters" `=` resConfigFilters.quote
         }
-        resources?.let { "resources" `=` resources }
+        resourceSets?.let { "resource_sets" `=` resourceSets }
         deps.notEmpty {
             "deps" `=` array(deps.map(BazelDependency::toString).quote)
         }
@@ -199,7 +199,7 @@ internal fun StatementsBuilder.androidLibrary(
     manifest: String? = null,
     srcsGlob: List<String> = emptyList(),
     visibility: Visibility = Visibility.Public,
-    resources: Assignee? = null,
+    resorceSets: Assignee? = null,
     resourceFiles: List<Assignee> = emptyList(),
     enableDataBinding: Boolean = false,
     enableCompose: Boolean = false,
@@ -226,7 +226,7 @@ internal fun StatementsBuilder.androidLibrary(
                 transform = Assignee::asString
             )
         }
-        resources?.let { "resources" `=` resources }
+        resorceSets?.let { "resource_sets" `=` resorceSets }
         deps.notEmpty {
             "deps" `=` array(deps.map(BazelDependency::toString).map(String::quote))
         }

diff --git a/grazel-gradle-plugin/src/main/kotlin/com/grab/grazel/migrate/android/AndroidData.kt b/grazel-gradle-plugin/src/main/kotlin/com/grab/grazel/migrate/android/AndroidData.kt
@@ -19,13 +19,18 @@ package com.grab.grazel.migrate.android
 import com.grab.grazel.bazel.rules.Multidex
 import com.grab.grazel.bazel.starlark.BazelDependency
 
+internal data class BazelSourceSet(
+    val name: String,
+    val res: String?,
+    val assets: String?,
+    val manifest: String?,
+)
+
 internal interface AndroidData {
     val name: String
     val srcs: List<String>
-    val res: List<String>
+    val resourceSets: Set<BazelSourceSet>
     val resValuesData: ResValuesData
-    val assets: List<String>
-    val assetsDir: String?
     val manifestFile: String?
 
     // Custom package used for detecting Java/Kotlin sources root
@@ -45,10 +50,8 @@ internal interface AndroidData {
 internal data class AndroidLibraryData(
     override val name: String,
     override val srcs: List<String> = emptyList(),
-    override val res: List<String> = emptyList(),
+    override val resourceSets: Set<BazelSourceSet> = emptySet(),
     override val resValuesData: ResValuesData = ResValuesData(),
-    override val assets: List<String> = emptyList(),
-    override val assetsDir: String? = null,
     override val manifestFile: String? = null,
     override val customPackage: String,
     override val packageName: String,
@@ -64,10 +67,8 @@ internal data class AndroidLibraryData(
 internal data class AndroidBinaryData(
     override val name: String,
     override val srcs: List<String> = emptyList(),
-    override val res: List<String> = emptyList(),
+    override val resourceSets: Set<BazelSourceSet> = emptySet(),
     override val resValuesData: ResValuesData = ResValuesData(),
-    override val assets: List<String> = emptyList(),
-    override val assetsDir: String? = null,
     override val manifestFile: String? = null,
     override val customPackage: String,
     override val packageName: String,

diff --git a/grazel-gradle-plugin/src/main/kotlin/com/grab/grazel/migrate/android/AndroidExtractor.kt b/grazel-gradle-plugin/src/main/kotlin/com/grab/grazel/migrate/android/AndroidExtractor.kt
@@ -34,10 +34,7 @@ import com.grab.grazel.gradle.variant.AndroidVariantDataSource
 import com.grab.grazel.gradle.variant.MatchedVariant
 import com.grab.grazel.gradle.variant.getMigratableBuildVariants
 import com.grab.grazel.gradle.variant.nameSuffix
-import com.grab.grazel.migrate.android.PathResolveMode.DIRECTORY
-import com.grab.grazel.migrate.android.SourceSetType.ASSETS
 import com.grab.grazel.migrate.android.SourceSetType.JAVA_KOTLIN
-import com.grab.grazel.migrate.android.SourceSetType.RESOURCES
 import com.grab.grazel.migrate.dependencies.calculateDirectDependencyTags
 import com.grab.grazel.migrate.kotlin.kotlinParcelizeDeps
 import dagger.Lazy
@@ -103,7 +100,6 @@ constructor(
         matchedVariant: MatchedVariant,
         extension: BaseExtension,
         deps: List<BazelDependency>,
-        sourceSetType: SourceSetType = JAVA_KOTLIN
     ): AndroidLibraryData {
         // Only consider source sets from migratable variants
         val migratableSourceSets = matchedVariant.variant.sourceSets
@@ -113,13 +109,14 @@ constructor(
             extension,
             migratableSourceSets
         ) ?: ""
-        val srcs = androidSources(migratableSourceSets, sourceSetType).toList()
-        val res = androidSources(migratableSourceSets, RESOURCES, DIRECTORY).toList().reversed()
+        val srcs = androidSources(migratableSourceSets, JAVA_KOTLIN).toList()
 
-        val assets = androidSources(migratableSourceSets, ASSETS).toList()
-        val assetsDir = assetsDirectory(migratableSourceSets, assets)
+        val resourceSets = migratableSourceSets.flatMap { it.toResourceSet(project) }
+            .reversed()
+            .toSet()
 
-        val manifestFile = androidManifestParser.androidManifestFile(migratableSourceSets)
+        val manifestFile = androidManifestParser
+            .androidManifestFile(migratableSourceSets)
             ?.let(::relativePath)
 
         val tags = if (grazelExtension.rules.kotlin.enabledTransitiveReduction) {
@@ -138,9 +135,7 @@ constructor(
         return AndroidLibraryData(
             name = name + matchedVariant.nameSuffix,
             srcs = srcs,
-            res = res,
-            assets = assets,
-            assetsDir = assetsDir,
+            resourceSets = resourceSets,
             manifestFile = manifestFile,
             customPackage = packageName,
             packageName = packageName,
@@ -153,19 +148,6 @@ constructor(
             lintConfigData = lintConfigs
         )
     }
-
-    private fun Project.assetsDirectory(
-        sourceSets: List<AndroidSourceSet>,
-        assets: List<String>
-    ): String? {
-        return if (assets.isNotEmpty()) {
-            val assetItem = assets.first()
-            sourceSets
-                .flatMap { it.assets.srcDirs }
-                .map { relativePath(it) }
-                .first { assetItem.contains(it) }
-        } else null
-    }
 }
 
 internal interface AndroidBinaryDataExtractor : AndroidExtractor<AndroidBinaryData>

diff --git a/grazel-gradle-plugin/src/main/kotlin/com/grab/grazel/migrate/android/AndroidLibraryData.kt b/grazel-gradle-plugin/src/main/kotlin/com/grab/grazel/migrate/android/AndroidLibraryData.kt
@@ -38,14 +38,25 @@ internal fun StatementsBuilder.buildResFiles(
 }
 
 internal fun buildResources(
-    resDirs: List<String>,
-) = if (resDirs.isEmpty()) null else
+    resourceSets: Set<BazelSourceSet>,
+) = if (resourceSets.isEmpty()) null else
     Assignee {
         add(
-            statement = resDirs
-                .groupBy { it }
-                .mapValues { emptyMap<String, String>() }
-                .toObject(quoteKeys = true, quoteValues = true, allowEmpty = true)
+            statement = resourceSets.groupBy(
+                keySelector = { it.name },
+                valueTransform = {
+                    mapOf(
+                        "res" to it.res,
+                        "assets" to it.assets,
+                        "manifest" to it.manifest
+                    )
+                })
+                .mapValues { it.value.first() }
+                .toObject(
+                    quoteKeys = true,
+                    quoteValues = true,
+                    allowEmpty = false
+                )
         )
     }