Add used_by_wrap_script for cxx_library

Summary:
When using wrap.sh to preload ASAN, you must also preload the C++
standard library to avoid android/ndk#988,
which implies that the library must be placed inside the primary APK
(since that's where wrap.sh needs to exist). If you're building the C++
standard library from source, you need cxx_library to support this.

This is a mostly mechanical change to add a new category of native
linkables to the packager and update all the relevant places. A few
potentially interesting points:
* We ignore libraries used by wrap scripts for native library merging,
  and just return them unchanged. This is for simplicity, since these
  libraries will be used in very limited circumstances and are not
  expected to be merged. Note that a library used by a wrap script
  cannot be an asset, so it would be ignored by native_library_merge_sequence
  anyway.
* If a library is used by a wrap script, it would likely also make
  sense for all its dependents to be marked as such. We do not enforce
  this in any way (either by automatically marking all dependents or by
  erroring out if they aren't), again for simplicity, with the idea that
  this feature will be used in very limited circumstances by people
  aware of this requirement.

Reviewed By: IanChilds

fbshipit-source-id: e8e2dc46f26a421937d2a9b5383dc7440d019ef4

docs/rule/cxx_library.soy

@@ -156,6 +156,21 @@ that the <code>cxx_library</code> depends on <em>might not be built</em>.
   {/param}
 {/call}
 
+{call buck.arg}
+ {param name : 'used_by_wrap_script' /}
+ {param default : 'False' /}
+ {param desc}
+ When using an <a href="{ROOT}article/exopackage.html">exopackage</a> for
+ Android, if this parameter is set to <code>True</code>, then the library is
+ included in the primary APK even if native libraries would otherwise not be
+ placed in it. This is intended for native libraries that are used by a
+ {sp}<a href="https://developer.android.com/ndk/guides/wrap-script">wrap.sh</a>
+ {sp}script, which must be placed in the primary APK. Only one of
+ {sp}<code>can_be_asset</code> and <code>used_by_wrap_script</code> can be set
+ for a rule.
+ {/param}
+{/call}
+
 {call cxx_common.supported_platforms_regex_arg /}
 
 {call cxx_common.force_static /}

src/com/facebook/buck/android/AndroidBinaryGraphEnhancer.java

@@ -131,6 +131,8 @@ public class AndroidBinaryGraphEnhancer {
   private final ImmutableCollection<SourcePath> nativeLibAssetsToExclude;
   private final ImmutableCollection<NativeLinkableGroup> nativeLinkablesAssetsToExcludeGroup;
   private final ImmutableCollection<SourcePath> nativeLibsForSystemLoaderToExclude;
+  private final ImmutableCollection<NativeLinkableGroup>
+      nativeLinkablesUsedByWrapScriptToExcludeGroup;
   private final JavaBuckConfig javaBuckConfig;
   private final JavaCDBuckConfig javaCDBuckConfig;
   private final DownwardApiConfig downwardApiConfig;
@@ -185,6 +187,7 @@ public class AndroidBinaryGraphEnhancer {
       ImmutableCollection<SourcePath> nativeLibAssetsToExclude,
       ImmutableCollection<NativeLinkableGroup> nativeLinkableGroupAssetsToExclude,
       ImmutableCollection<SourcePath> nativeLibsForSystemLoaderToExclude,
+      ImmutableCollection<NativeLinkableGroup> nativeLinkablesUsedByWrapScriptToExcludeGroup,
       boolean skipCrunchPngs,
       boolean includesVectorDrawables,
       boolean noAutoVersionResources,
@@ -249,6 +252,8 @@ public class AndroidBinaryGraphEnhancer {
     this.nativeLibAssetsToExclude = nativeLibAssetsToExclude;
     this.nativeLinkablesAssetsToExcludeGroup = nativeLinkableGroupAssetsToExclude;
     this.nativeLibsForSystemLoaderToExclude = nativeLibsForSystemLoaderToExclude;
+    this.nativeLinkablesUsedByWrapScriptToExcludeGroup =
+        nativeLinkablesUsedByWrapScriptToExcludeGroup;
     this.javaBuckConfig = javaBuckConfig;
     this.javaCDBuckConfig = javaCDBuckConfig;
     this.javacOptions = javacOptions;
@@ -340,6 +345,7 @@ AndroidGraphEnhancementResult createAdditionalBuildables() {
             nativeLibAssetsToExclude,
             nativeLinkablesAssetsToExcludeGroup,
             nativeLibsForSystemLoaderToExclude,
+            nativeLinkablesUsedByWrapScriptToExcludeGroup,
             apkModuleGraph,
             AndroidPackageableFilterFactory.createFromConfigurationMatcher(
                 originalBuildTarget, androidNativeTargetConfigurationMatcher),

src/com/facebook/buck/android/AndroidBinaryGraphEnhancerFactory.java

@@ -180,6 +180,7 @@ public AndroidBinaryGraphEnhancer create(
         /* nativeLibAssetsToExclude */ ImmutableSet.of(),
         /* nativeLinkableAssetsToExclude */ ImmutableSet.of(),
         /* nativeLibsForSystemLoaderToExclude */ ImmutableSet.of(),
+        /* nativeLinkablesUsedByWrapScriptToExclude */ ImmutableSet.of(),
         shouldSkipCrunchPngs,
         args.isIncludesVectorDrawables(),
         args.isNoAutoVersionResources(),

src/com/facebook/buck/android/AndroidInstrumentationApkDescription.java

@@ -167,6 +167,9 @@ public BuildRule createBuildRule(
     ImmutableCollection<SourcePath> nativeLibsForSystemLoaderToExclude =
         apkUnderTest.getAndroidPackageableCollection().getNativeLibsDirectoriesForSystemLoader();
 
+    ImmutableCollection<NativeLinkableGroup> nativeLinkablesUsedByWrapScriptToExcludeGroup =
+        apkUnderTest.getAndroidPackageableCollection().getNativeLinkablesUsedByWrapScript();
+
     ListeningExecutorService dxExecutorService =
         toolchainProvider
             .getByName(
@@ -239,6 +242,7 @@ public BuildRule createBuildRule(
             nativeLibAssetsToExclude,
             nativeLinkableGroupAssetsToExclude,
             nativeLibsForSystemLoaderToExclude,
+            nativeLinkablesUsedByWrapScriptToExcludeGroup,
             /* skipCrunchPngs */ false,
             args.getIncludesVectorDrawables(),
             /* noAutoVersionResources */ false,

src/com/facebook/buck/android/AndroidNativeLibsPackageableGraphEnhancer.java

@@ -153,30 +153,43 @@ interface AndroidNativeLibsGraphEnhancementResult {
   }
 
   // Populates an immutable map builder with all given linkables set to the given cpu type.
-  // Returns true iff linkables is not empty.
   private void getNativeLinkableMetadata(
       ImmutableMultimap<APKModule, NativeLinkable> linkables,
       ImmutableMap.Builder<AndroidLinkableMetadata, SourcePath> builder,
       BiConsumer<AndroidLinkableMetadata, BuildTarget> duplicateReporter,
       TargetCpuType targetCpuType)
       throws HumanReadableException {
     for (Map.Entry<APKModule, NativeLinkable> linkableEntry : linkables.entries()) {
-      NativeLinkable nativeLinkable = linkableEntry.getValue();
-      if (nativeLinkable.getPreferredLinkage() != NativeLinkableGroup.Linkage.STATIC) {
-        if (!androidNativeTargetConfigurationMatcher.nativeTargetConfigurationMatchesCpuType(
-            linkableEntry.getValue().getBuildTarget(), targetCpuType)) {
-          continue;
-        }
-
-        getSharedLibrariesAndMetadata(
-            targetCpuType,
-            nativeLinkable,
-            linkableEntry.getKey(),
-            (metadata, libraryPath) -> {
-              builder.put(metadata, libraryPath);
-              duplicateReporter.accept(metadata, nativeLinkable.getBuildTarget());
-            });
+      getMetadataForNativeLinkable(
+          linkableEntry.getKey(),
+          linkableEntry.getValue(),
+          builder,
+          duplicateReporter,
+          targetCpuType);
+    }
+  }
+
+  private void getMetadataForNativeLinkable(
+      APKModule apkModule,
+      NativeLinkable nativeLinkable,
+      ImmutableMap.Builder<AndroidLinkableMetadata, SourcePath> builder,
+      BiConsumer<AndroidLinkableMetadata, BuildTarget> duplicateReporter,
+      TargetCpuType targetCpuType)
+      throws HumanReadableException {
+    if (nativeLinkable.getPreferredLinkage() != NativeLinkableGroup.Linkage.STATIC) {
+      if (!androidNativeTargetConfigurationMatcher.nativeTargetConfigurationMatchesCpuType(
+          nativeLinkable.getBuildTarget(), targetCpuType)) {
+        return;
       }
+
+      getSharedLibrariesAndMetadata(
+          targetCpuType,
+          nativeLinkable,
+          apkModule,
+          (metadata, libraryPath) -> {
+            builder.put(metadata, libraryPath);
+            duplicateReporter.accept(metadata, nativeLinkable.getBuildTarget());
+          });
     }
   }
 
@@ -199,11 +212,15 @@ public AndroidNativeLibsGraphEnhancementResult enhance(
         packageableCollection.getNativeLinkables();
     ImmutableMultimap<APKModule, NativeLinkableGroup> nativeLinkableGroupsAssets =
         packageableCollection.getNativeLinkablesAssets();
+    ImmutableList<NativeLinkableGroup> nativeLinkableGroupsUsedByWrapScript =
+        packageableCollection.getNativeLinkablesUsedByWrapScript();
 
     // TODO(cjhopman): The linkables handling is much more complex and we probably should split it
     // out into its own function.
     boolean hasLinkables =
-        !(nativeLinkableGroups.isEmpty() && nativeLinkableGroupsAssets.isEmpty());
+        !(nativeLinkableGroups.isEmpty()
+            && nativeLinkableGroupsAssets.isEmpty()
+            && nativeLinkableGroupsUsedByWrapScript.isEmpty());
     boolean hasNativeLibDirs =
         !(packageableCollection.getNativeLibsDirectories().isEmpty()
             && packageableCollection.getNativeLibAssetsDirectories().isEmpty()
@@ -248,7 +265,8 @@ public AndroidNativeLibsGraphEnhancementResult enhance(
           expandLinkableGroups(
               nativePlatforms.get(cpuType).getCxxPlatform(),
               nativeLinkableGroups,
-              nativeLinkableGroupsAssets));
+              nativeLinkableGroupsAssets,
+              nativeLinkableGroupsUsedByWrapScript));
     }
 
     ImmutableMap<TargetCpuType, NativeLinkableEnhancementResult> nativeLinkables =
@@ -292,16 +310,19 @@ public AndroidNativeLibsGraphEnhancementResult enhance(
 
     // Make sure we process the root module last so that we know if any of the module contain
     // libraries that depend on a non-system runtime and add it to the root module if needed.
+    APKModule rootAPKModule = apkModuleGraph.getRootAPKModule();
     ImmutableSet<APKModule> apkModules =
         FluentIterable.from(apkModuleGraph.getAPKModules())
             .filter(input -> !input.isRootModule())
-            .append(apkModuleGraph.getRootAPKModule())
+            .append(rootAPKModule)
             .toSet();
 
     ImmutableMap.Builder<AndroidLinkableMetadata, SourcePath> nativeLinkableLibsBuilder =
         ImmutableMap.builder();
     ImmutableMap.Builder<AndroidLinkableMetadata, SourcePath> nativeLinkableLibsAssetsBuilder =
         ImmutableMap.builder();
+    ImmutableMap.Builder<AndroidLinkableMetadata, SourcePath>
+        nativeLinkableLibsUsedByWrapScriptBuilder = ImmutableMap.builder();
 
     BiConsumer<AndroidLinkableMetadata, BuildTarget> duplicateReporter =
         new BiConsumer<AndroidLinkableMetadata, BuildTarget>() {
@@ -330,12 +351,27 @@ public void accept(AndroidLinkableMetadata metadata, BuildTarget buildTarget) {
           nativeLinkableLibsAssetsBuilder,
           duplicateReporter,
           targetCpuType);
+      for (NativeLinkable nativeLinkable :
+          nativeLinkables.get(targetCpuType).getNativeLinkablesUsedByWrapScript()) {
+        getMetadataForNativeLinkable(
+            rootAPKModule,
+            nativeLinkable,
+            nativeLinkableLibsUsedByWrapScriptBuilder,
+            duplicateReporter,
+            targetCpuType);
+      }
     }
 
     // Adds a cxxruntime linkable to the nativeLinkableLibsBuilder for every platform that needs it.
     ImmutableMap<AndroidLinkableMetadata, SourcePath> nativeLinkableLibsAssets =
         nativeLinkableLibsAssetsBuilder.build();
-    addCxxRuntimeLinkables(nativePlatforms, nativeLinkableLibsBuilder, nativeLinkableLibsAssets);
+    ImmutableMap<AndroidLinkableMetadata, SourcePath> nativeLinkableLibsUsedByWrapScript =
+        nativeLinkableLibsUsedByWrapScriptBuilder.build();
+    addCxxRuntimeLinkables(
+        nativePlatforms,
+        nativeLinkableLibsBuilder,
+        nativeLinkableLibsAssets,
+        nativeLinkableLibsUsedByWrapScript);
 
     ImmutableMap<AndroidLinkableMetadata, SourcePath> nativeLinkableLibs =
         nativeLinkableLibsBuilder.build();
@@ -354,10 +390,12 @@ public void accept(AndroidLinkableMetadata metadata, BuildTarget buildTarget) {
               nativePlatforms,
               nativeLinkableLibs,
               nativeLinkableLibsAssets,
+              nativeLinkableLibsUsedByWrapScript,
               relinkerWhitelist);
 
       nativeLinkableLibs = relinker.getRelinkedLibs();
       nativeLinkableLibsAssets = relinker.getRelinkedLibsAssets();
+      nativeLinkableLibsUsedByWrapScript = relinker.getRelinkedLibsUsedByWrapScript();
       for (BuildRule rule : relinker.getRules()) {
         graphBuilder.addToIndex(rule);
       }
@@ -367,10 +405,14 @@ public void accept(AndroidLinkableMetadata metadata, BuildTarget buildTarget) {
         generateStripRules(nativePlatforms, nativeLinkableLibs);
     ImmutableMap<StripLinkable, CopyNativeLibraries.StrippedObjectDescription>
         strippedLibsAssetsMap = generateStripRules(nativePlatforms, nativeLinkableLibsAssets);
+    ImmutableMap<StripLinkable, CopyNativeLibraries.StrippedObjectDescription>
+        strippedLibsUsedByWrapScriptMap =
+            generateStripRules(nativePlatforms, nativeLinkableLibsUsedByWrapScript);
 
     ImmutableSortedSet<SourcePath> unstrippedLibraries =
         RichStream.from(nativeLinkableLibs.values())
             .concat(nativeLinkableLibsAssets.values().stream())
+            .concat(nativeLinkableLibsUsedByWrapScript.values().stream())
             .toImmutableSortedSet(Ordering.natural());
 
     Optional<CopyNativeLibraries> nativeLibrariesForPrimaryApk = Optional.empty();
@@ -388,6 +430,12 @@ public void accept(AndroidLinkableMetadata metadata, BuildTarget buildTarget) {
                   FluentIterable.from(strippedLibsAssetsMap.entrySet())
                       .filter(entry -> module.equals(entry.getValue().getApkModule())));
 
+      ImmutableMap<StripLinkable, CopyNativeLibraries.StrippedObjectDescription>
+          filteredStrippedLibsUsedByWrapScriptMap =
+              ImmutableMap.copyOf(
+                  FluentIterable.from(strippedLibsUsedByWrapScriptMap.entrySet())
+                      .filter(entry -> module.equals(entry.getValue().getApkModule())));
+
       ImmutableCollection<SourcePath> nativeLibsDirectories =
           packageableCollection.getNativeLibsDirectories().get(module);
 
@@ -399,12 +447,14 @@ public void accept(AndroidLinkableMetadata metadata, BuildTarget buildTarget) {
           module.isRootModule()
               ? packageableCollection.getNativeLibsDirectoriesForSystemLoader()
               : ImmutableList.of();
-      if (module.isRootModule() && !nativeLibsDirectoriesForPrimaryDexModule.isEmpty()) {
+      if (module.isRootModule()
+          && !(filteredStrippedLibsUsedByWrapScriptMap.isEmpty()
+              && nativeLibsDirectoriesForPrimaryDexModule.isEmpty())) {
         nativeLibrariesForPrimaryApk =
             Optional.of(
                 createCopyNativeLibraries(
                     module,
-                    ImmutableMap.of(),
+                    filteredStrippedLibsUsedByWrapScriptMap,
                     ImmutableMap.of(),
                     nativeLibsDirectoriesForPrimaryDexModule,
                     ImmutableList.of(),
@@ -520,12 +570,15 @@ public ImmutableSet<BuildRule> addNativeMergeMapGenCode(
   private NativeLinkableEnhancementResult expandLinkableGroups(
       CxxPlatform cxxPlatform,
       ImmutableMultimap<APKModule, NativeLinkableGroup> nativeLinkableGroups,
-      ImmutableMultimap<APKModule, NativeLinkableGroup> nativeLinkableGroupssAssets) {
+      ImmutableMultimap<APKModule, NativeLinkableGroup> nativeLinkableGroupssAssets,
+      ImmutableList<NativeLinkableGroup> nativeLinkableGroupsUsedByWrapScript) {
 
     ImmutableMultimap.Builder<APKModule, NativeLinkable> linkablesBuilder =
         ImmutableMultimap.builder();
     ImmutableMultimap.Builder<APKModule, NativeLinkable> linkableAssetsBuilder =
         ImmutableMultimap.builder();
+    ImmutableList.Builder<NativeLinkable> linkablesUsedByWrapScriptBuilder =
+        ImmutableList.builder();
 
     nativeLinkableGroups.forEach(
         (module, group) ->
@@ -535,16 +588,25 @@ private NativeLinkableEnhancementResult expandLinkableGroups(
         (module, group) ->
             linkableAssetsBuilder.put(module, group.getNativeLinkable(cxxPlatform, graphBuilder)));
 
+    nativeLinkableGroupsUsedByWrapScript.forEach(
+        group ->
+            linkablesUsedByWrapScriptBuilder.add(
+                group.getNativeLinkable(cxxPlatform, graphBuilder)));
+
     return NativeLinkableEnhancementResult.of(
-        linkablesBuilder.build(), linkableAssetsBuilder.build());
+        linkablesBuilder.build(),
+        linkableAssetsBuilder.build(),
+        linkablesUsedByWrapScriptBuilder.build());
   }
 
   private void addCxxRuntimeLinkables(
       ImmutableMap<TargetCpuType, NdkCxxPlatform> nativePlatforms,
       ImmutableMap.Builder<AndroidLinkableMetadata, SourcePath> nativeLinkableLibsBuilder,
-      ImmutableMap<AndroidLinkableMetadata, SourcePath> nativeLinkableLibsAssets) {
+      ImmutableMap<AndroidLinkableMetadata, SourcePath> nativeLinkableLibsAssets,
+      ImmutableMap<AndroidLinkableMetadata, SourcePath> nativeLinkableLibsUsedByWrapScript) {
     RichStream.from(nativeLinkableLibsBuilder.build().keySet())
         .concat(RichStream.from(nativeLinkableLibsAssets.keySet()))
+        .concat(RichStream.from(nativeLinkableLibsUsedByWrapScript.keySet()))
         .map(AndroidLinkableMetadata::getTargetCpuType)
         .distinct()
         .forEach(

src/com/facebook/buck/android/NativeLibraryMergeEnhancer.java

@@ -287,7 +287,10 @@ public static NativeLibraryMergeEnhancementResult enhance(
       mergedLinkablesBuilder.put(
           cpuType,
           NativeLinkableEnhancementResult.of(
-              moduleLinkablesBuilder.build(), moduleAssetLinkablesBuilder.build()));
+              moduleLinkablesBuilder.build(),
+              moduleAssetLinkablesBuilder.build(),
+              // Linkables used by the wrap script are not intended to be merged
+              baseResult.getNativeLinkablesUsedByWrapScript()));
     }
 
     ImmutableSortedMap.Builder<String, ImmutableSortedSet<String>> finalSonameTargetsBuilder =

src/com/facebook/buck/android/packageable/AndroidPackageableCollection.java

@@ -83,6 +83,12 @@ public boolean hasResources() {
   /** Native libraries to be packaged as assets. */
   ImmutableMultimap<APKModule, NativeLinkableGroup> getNativeLinkablesAssets();
 
+  /**
+   * Native libraries that are used by a wrap script and must be packaged in the primary APK. Since
+   * these must be in the root module, we use a list instead of a map.
+   */
+  ImmutableList<NativeLinkableGroup> getNativeLinkablesUsedByWrapScript();
+
   /** Directories containing native libraries. */
   ImmutableMultimap<APKModule, SourcePath> getNativeLibsDirectories();