From 0dc0c7e3053fa7f5b976a011d4008173b0336046 Mon Sep 17 00:00:00 2001 From: Googler Date: Wed, 19 Apr 2023 05:24:12 -0700 Subject: [PATCH] Starlarkify CcToolchainProviderHelper. PiperOrigin-RevId: 525419738 Change-Id: Ib887b0859edadf3b207f698d902e810cd11fbea7 --- .../build/lib/rules/cpp/CcModule.java | 4 + .../lib/rules/cpp/CcStarlarkInternal.java | 277 ++++++++++++++++++ .../build/lib/rules/cpp/CcToolchain.java | 17 +- .../cpp/CcToolchainAttributesProvider.java | 100 +++++++ .../lib/rules/cpp/CcToolchainConfigInfo.java | 45 +++ .../lib/rules/cpp/CcToolchainFeatures.java | 3 +- .../build/lib/rules/cpp/CcToolchainSuite.java | 18 +- .../lib/starlarkbuildapi/cpp/CcModuleApi.java | 7 + .../builtins_bzl/common/cc/cc_common.bzl | 9 +- .../builtins_bzl/common/cc/cc_helper.bzl | 1 + .../cc/cc_toolchain_provider_helper.bzl | 270 +++++++++++++++++ .../starlark/builtins_bzl/common/exports.bzl | 2 + .../build/lib/rules/cpp/CcToolchainTest.java | 2 +- 13 files changed, 749 insertions(+), 6 deletions(-) create mode 100644 src/main/starlark/builtins_bzl/common/cc/cc_toolchain_provider_helper.bzl diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcModule.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcModule.java index 5d666601c318b1..78d6a819695e32 100755 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcModule.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcModule.java @@ -800,6 +800,7 @@ public CcCompilationContext createCcCompilationContext( Sequence directPublicHdrs, Sequence directPrivateHdrs, Object purposeNoneable, + Object moduleMap, StarlarkThread thread) throws EvalException { isCalledFromStarlarkCcCommon(thread); @@ -844,6 +845,9 @@ public CcCompilationContext createCcCompilationContext( && purposeNoneable != Starlark.NONE) { ccCompilationContext.setPurpose((String) purposeNoneable); } + if (moduleMap != null && moduleMap != Starlark.UNBOUND && moduleMap != Starlark.NONE) { + ccCompilationContext.setCppModuleMap((CppModuleMap) moduleMap); + } return ccCompilationContext.build(); } diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcStarlarkInternal.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcStarlarkInternal.java index f136905df5fbc6..00118673825f19 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcStarlarkInternal.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcStarlarkInternal.java @@ -14,13 +14,21 @@ package com.google.devtools.build.lib.rules.cpp; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableMap.toImmutableMap; + import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.devtools.build.docgen.annot.DocCategory; import com.google.devtools.build.lib.actions.Artifact; import com.google.devtools.build.lib.actions.CommandLineExpansionException; +import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue; import com.google.devtools.build.lib.analysis.starlark.StarlarkActionFactory; import com.google.devtools.build.lib.analysis.starlark.StarlarkRuleContext; import com.google.devtools.build.lib.cmdline.Label; +import com.google.devtools.build.lib.collect.nestedset.Depset; +import com.google.devtools.build.lib.collect.nestedset.Depset.TypeException; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; import com.google.devtools.build.lib.packages.Attribute.ComputedDefault; import com.google.devtools.build.lib.packages.AttributeMap; import com.google.devtools.build.lib.packages.RuleClass.ConfiguredTargetFactory.RuleErrorException; @@ -29,12 +37,18 @@ import com.google.devtools.build.lib.rules.cpp.CcLinkingContext.Linkstamp; import com.google.devtools.build.lib.starlarkbuildapi.NativeComputedDefaultApi; import com.google.devtools.build.lib.starlarkbuildapi.core.ProviderApi; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.vfs.PathFragment; import javax.annotation.Nullable; import net.starlark.java.annot.Param; import net.starlark.java.annot.ParamType; import net.starlark.java.annot.StarlarkBuiltin; import net.starlark.java.annot.StarlarkMethod; +import net.starlark.java.eval.Dict; import net.starlark.java.eval.EvalException; +import net.starlark.java.eval.NoneType; +import net.starlark.java.eval.Sequence; +import net.starlark.java.eval.Starlark; import net.starlark.java.eval.StarlarkValue; import net.starlark.java.syntax.Location; @@ -44,6 +58,269 @@ public class CcStarlarkInternal implements StarlarkValue { public static final String NAME = "cc_internal"; + @Nullable + private PathFragment getPathfragmentOrNone(Object o) { + String pathString = CcModule.convertFromNoneable(o, null); + if (pathString == null) { + return null; + } + return PathFragment.create(pathString); + } + + private ImmutableMap castDict(Dict d) throws EvalException { + return Dict.cast(d, String.class, String.class, "tool_paths").entrySet().stream() + .map(p -> Pair.of(p.getKey(), PathFragment.create(p.getValue()))) + .collect(toImmutableMap(Pair::getFirst, Pair::getSecond)); + } + + @StarlarkMethod( + name = "construct_toolchain_provider", + documented = false, + parameters = { + @Param(name = "ctx", positional = false, named = true), + @Param(name = "cpp_config", positional = false, named = true), + @Param(name = "toolchain_features", positional = false, named = true), + @Param(name = "tools_directory", positional = false, named = true), + @Param(name = "attributes", positional = false, named = true), + @Param( + name = "static_runtime_link_inputs", + positional = false, + named = true, + allowedTypes = { + @ParamType(type = Depset.class), + @ParamType(type = NoneType.class), + }), + @Param( + name = "dynamic_runtime_link_symlinks", + positional = false, + named = true, + allowedTypes = { + @ParamType(type = Depset.class), + @ParamType(type = NoneType.class), + }), + @Param(name = "runtime_solib_dir", positional = false, named = true), + @Param(name = "cc_compilation_context", positional = false, named = true), + @Param(name = "builtin_include_files", positional = false, named = true), + @Param(name = "target_builtin_include_files", positional = false, named = true), + @Param(name = "builtin_include_directories", positional = false, named = true), + @Param(name = "sysroot", positional = false, named = true), + @Param(name = "target_sysroot", positional = false, named = true), + @Param(name = "fdo_context", positional = false, named = true), + @Param(name = "is_tool_configuration", positional = false, named = true), + @Param(name = "tool_paths", positional = false, named = true), + @Param(name = "toolchain_config_info", positional = false, named = true), + @Param(name = "default_sysroot", positional = false, named = true), + @Param(name = "runtime_sysroot", positional = false, named = true), + @Param(name = "solib_directory", positional = false, named = true), + @Param(name = "additional_make_variables", positional = false, named = true), + @Param(name = "legacy_cc_flags_make_variable", positional = false, named = true), + @Param(name = "objcopy", positional = false, named = true), + @Param(name = "compiler", positional = false, named = true), + @Param(name = "preprocessor", positional = false, named = true), + @Param(name = "nm", positional = false, named = true), + @Param(name = "objdump", positional = false, named = true), + @Param(name = "ar", positional = false, named = true), + @Param(name = "strip", positional = false, named = true), + @Param(name = "ld", positional = false, named = true), + @Param(name = "gcov", positional = false, named = true), + }) + public CcToolchainProvider getCcToolchainProvider( + StarlarkRuleContext ruleContext, + Object cppConfigurationObject, + CcToolchainFeatures toolchainFeatures, + String toolsDirectoryStr, + CcToolchainAttributesProvider attributes, + Object staticRuntimeLinkInputsObject, + Object dynamicRuntimeLinkInputsObject, + String dynamicRuntimeSolibDirStr, + CcCompilationContext ccCompilationContext, + Sequence builtinIncludeFiles, + Sequence targetBuiltinIncludeFiles, + Sequence builtInIncludeDirectoriesStr, + Object sysrootObject, + Object targetSysrootObject, + FdoContext fdoContext, + boolean isToolConfiguration, + Dict toolPathsDict, + CcToolchainConfigInfo toolchainConfigInfo, + Object defaultSysrootObject, + Object runtimeSysrootObject, + String solibDirectory, + Dict additionalMakeVariablesDict, + String legacyCcFlagsMakeVariable, + String objcopyExecutable, + String compilerExecutable, + String preprocessorExecutable, + String nmExecutable, + String objdumpExecutable, + String arExecutable, + String stripExecutable, + String ldExecutable, + String gcovExecutable) + throws EvalException { + CppConfiguration cppConfiguration = CcModule.convertFromNoneable(cppConfigurationObject, null); + PathFragment toolsDirectory = PathFragment.create(toolsDirectoryStr); + NestedSet staticRuntimeLinkInputsSet = null; + NestedSet dynamicRuntimeLinkInputsSet = null; + try { + if (staticRuntimeLinkInputsObject != Starlark.NONE) { + staticRuntimeLinkInputsSet = + ((Depset) staticRuntimeLinkInputsObject).getSet(Artifact.class); + } + if (dynamicRuntimeLinkInputsObject != Starlark.NONE) { + dynamicRuntimeLinkInputsSet = + ((Depset) dynamicRuntimeLinkInputsObject).getSet(Artifact.class); + } + } catch (TypeException e) { + throw new EvalException(e); + } + PathFragment dynamicRuntimeSolibDir = PathFragment.create(dynamicRuntimeSolibDirStr); + ImmutableList builtInIncludeDirectories = + Sequence.cast(builtInIncludeDirectoriesStr, String.class, "builtin_include_directories") + .stream() + .map(PathFragment::create) + .collect(toImmutableList()); + PathFragment sysroot = getPathfragmentOrNone(sysrootObject); + PathFragment targetSysroot = getPathfragmentOrNone(targetSysrootObject); + Dict additionalMakeVariables = + Dict.cast(additionalMakeVariablesDict, String.class, String.class, "tool_paths"); + PathFragment defaultSysroot = getPathfragmentOrNone(defaultSysrootObject); + PathFragment runtimeSysroot = getPathfragmentOrNone(runtimeSysrootObject); + + return new CcToolchainProvider( + /* cppConfiguration= */ cppConfiguration, + /* toolchainFeatures= */ toolchainFeatures, + /* crosstoolTopPathFragment= */ toolsDirectory, + /* allFiles= */ attributes.getAllFiles(), + /* allFilesIncludingLibc= */ attributes.getFullInputsForCrosstool(), + /* compilerFiles= */ attributes.getCompilerFiles(), + /* compilerFilesWithoutIncludes= */ attributes.getCompilerFilesWithoutIncludes(), + /* stripFiles= */ attributes.getStripFiles(), + /* objcopyFiles= */ attributes.getObjcopyFiles(), + /* asFiles= */ attributes.getAsFiles(), + /* arFiles= */ attributes.getArFiles(), + /* linkerFiles= */ attributes.getFullInputsForLink(), + /* interfaceSoBuilder= */ attributes.getIfsoBuilder(), + /* dwpFiles= */ attributes.getDwpFiles(), + /* coverageFiles= */ attributes.getCoverage(), + /* libcLink= */ attributes.getLibc(), + /* targetLibcLink= */ attributes.getTargetLibc(), + /* staticRuntimeLinkInputs= */ staticRuntimeLinkInputsSet, + /* dynamicRuntimeLinkInputs= */ dynamicRuntimeLinkInputsSet, + /* dynamicRuntimeSolibDir= */ dynamicRuntimeSolibDir, + /* ccCompilationContext= */ ccCompilationContext, + /* supportsParamFiles= */ attributes.isSupportsParamFiles(), + /* supportsHeaderParsing= */ attributes.isSupportsHeaderParsing(), + /* additionalBuildVariablesComputer= */ attributes.getAdditionalBuildVariablesComputer(), + /* buildVariables= */ CcToolchainProviderHelper.getBuildVariables( + ruleContext.getRuleContext().getConfiguration().getOptions(), + cppConfiguration, + sysroot, + attributes.getAdditionalBuildVariablesComputer()), + /* builtinIncludeFiles= */ Sequence.cast( + builtinIncludeFiles, Artifact.class, "builtin_include_files") + .getImmutableList(), + /* targetBuiltinIncludeFiles= */ Sequence.cast( + targetBuiltinIncludeFiles, Artifact.class, "target_builtin_include_files") + .getImmutableList(), + /* linkDynamicLibraryTool= */ attributes.getLinkDynamicLibraryTool(), + /* builtInIncludeDirectories= */ builtInIncludeDirectories, + /* sysroot= */ sysroot, + /* targetSysroot= */ targetSysroot, + /* fdoContext= */ fdoContext, + /* isToolConfiguration= */ isToolConfiguration, + /* licensesProvider= */ attributes.getLicensesProvider(), + /* toolPaths= */ castDict(toolPathsDict), + /* toolchainIdentifier= */ toolchainConfigInfo.getToolchainIdentifier(), + /* compiler= */ toolchainConfigInfo.getCompiler(), + /* abiGlibcVersion= */ toolchainConfigInfo.getAbiLibcVersion(), + /* targetCpu= */ toolchainConfigInfo.getTargetCpu(), + /* targetOS= */ toolchainConfigInfo.getCcTargetOs(), + /* defaultSysroot= */ defaultSysroot, + /* runtimeSysroot= */ runtimeSysroot, + /* targetLibc= */ toolchainConfigInfo.getTargetLibc(), + /* ccToolchainLabel= */ ruleContext.getRuleContext().getLabel(), + /* solibDirectory= */ solibDirectory, + /* abi= */ toolchainConfigInfo.getAbiVersion(), + /* targetSystemName= */ toolchainConfigInfo.getTargetSystemName(), + /* additionalMakeVariables= */ ImmutableMap.copyOf(additionalMakeVariables), + /* legacyCcFlagsMakeVariable= */ legacyCcFlagsMakeVariable, + /* allowlistForLayeringCheck= */ attributes.getAllowlistForLayeringCheck(), + /* allowListForLooseHeaderCheck= */ attributes.getAllowlistForLooseHeaderCheck(), + /* objcopyExecutable= */ objcopyExecutable, + /* compilerExecutable= */ compilerExecutable, + /* preprocessorExecutable= */ preprocessorExecutable, + /* nmExecutable= */ nmExecutable, + /* objdumpExecutable= */ objdumpExecutable, + /* arExecutable= */ arExecutable, + /* stripExecutable= */ stripExecutable, + /* ldExecutable= */ ldExecutable, + /* gcovExecutable= */ gcovExecutable); + } + + @StarlarkMethod( + name = "solib_symlink_action", + documented = false, + parameters = { + @Param(name = "ctx", positional = false, named = true), + @Param(name = "artifact", positional = false, named = true), + @Param(name = "solib_directory", positional = false, named = true), + @Param(name = "runtime_solib_dir_base", positional = false, named = true), + }) + public Artifact solibSymlinkAction( + StarlarkRuleContext ruleContext, + Artifact artifact, + String solibDirectory, + String runtimeSolibDirBase) { + return SolibSymlinkAction.getCppRuntimeSymlink( + ruleContext.getRuleContext(), artifact, solibDirectory, runtimeSolibDirBase); + } + + @StarlarkMethod( + name = "fdo_context", + documented = false, + parameters = { + @Param(name = "ctx", positional = false, named = true), + @Param(name = "attributes", positional = false, named = true), + @Param(name = "configuration", positional = false, named = true), + @Param(name = "cpp_config", positional = false, named = true), + @Param(name = "tool_paths", positional = false, named = true), + }, + allowReturnNones = true) + @Nullable + public FdoContext fdoContext( + StarlarkRuleContext ruleContext, + CcToolchainAttributesProvider attributes, + BuildConfigurationValue configuration, + CppConfiguration cppConfiguration, + Dict toolPathsDict) + throws EvalException, InterruptedException { + try { + return FdoHelper.getFdoContext( + ruleContext.getRuleContext(), + attributes, + configuration, + cppConfiguration, + castDict(toolPathsDict)); + } catch (RuleErrorException e) { + throw new EvalException(e); + } + } + + @StarlarkMethod( + name = "cc_toolchain_features", + documented = false, + parameters = { + @Param(name = "toolchain_config_info", positional = false, named = true), + @Param(name = "tools_directory", positional = false, named = true), + }) + public CcToolchainFeatures ccToolchainFeatures( + CcToolchainConfigInfo ccToolchainConfigInfo, String toolsDirectoryPathString) + throws EvalException { + return new CcToolchainFeatures( + ccToolchainConfigInfo, PathFragment.create(toolsDirectoryPathString)); + } + @StarlarkMethod( name = "is_package_headers_checking_mode_set", documented = false, diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchain.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchain.java index a9aeee569da65f..75f1eb359ee8cb 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchain.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchain.java @@ -13,6 +13,7 @@ // limitations under the License. package com.google.devtools.build.lib.rules.cpp; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.actions.MutableActionGraph.ActionConflictException; import com.google.devtools.build.lib.analysis.ConfiguredTarget; @@ -29,6 +30,8 @@ import java.io.Serializable; import java.util.HashMap; import javax.annotation.Nullable; +import net.starlark.java.eval.Starlark; +import net.starlark.java.eval.StarlarkFunction; import net.starlark.java.syntax.Location; /** @@ -86,10 +89,22 @@ public ConfiguredTarget create(RuleContext ruleContext) .build(); } + StarlarkFunction getCcToolchainProvider = + (StarlarkFunction) ruleContext.getStarlarkDefinedBuiltin("get_cc_toolchain_provider"); + ruleContext.initStarlarkRuleContext(); + Object starlarkCcToolchainProvider = + ruleContext.callStarlarkOrThrowRuleError( + getCcToolchainProvider, + ImmutableList.of( + /* ctx */ ruleContext.getStarlarkRuleContext(), /* attributes */ attributes), + ImmutableMap.of()); + // This is a platforms-backed build, we will not analyze cc_toolchain_suite at all, and we are // sure current cc_toolchain is the one selected. We can create CcToolchainProvider here. CcToolchainProvider ccToolchainProvider = - CcToolchainProviderHelper.getCcToolchainProvider(ruleContext, attributes); + starlarkCcToolchainProvider != Starlark.NONE + ? (CcToolchainProvider) starlarkCcToolchainProvider + : null; if (ccToolchainProvider == null) { // Skyframe restart diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainAttributesProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainAttributesProvider.java index effab4e7b48465..426ad196fd0ccf 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainAttributesProvider.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainAttributesProvider.java @@ -29,6 +29,7 @@ import com.google.devtools.build.lib.analysis.RuleContext; import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; import com.google.devtools.build.lib.cmdline.Label; +import com.google.devtools.build.lib.collect.nestedset.Depset; import com.google.devtools.build.lib.collect.nestedset.NestedSet; import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; import com.google.devtools.build.lib.collect.nestedset.Order; @@ -40,6 +41,9 @@ import com.google.devtools.build.lib.packages.Type; import com.google.devtools.build.lib.rules.cpp.CcToolchain.AdditionalBuildVariablesComputer; import javax.annotation.Nullable; +import net.starlark.java.annot.StarlarkMethod; +import net.starlark.java.eval.EvalException; +import net.starlark.java.eval.StarlarkThread; /** * Provider encapsulating all the information from the cc_toolchain rule that affects creation of @@ -231,6 +235,12 @@ public String getPurposePrefix() { return purposePrefix; } + @StarlarkMethod(name = "runtime_solib_dir_base", documented = false, useStarlarkThread = true) + public String getRuntimeSolibDirBaseForStarlark(StarlarkThread thread) throws EvalException { + CcModule.checkPrivateStarlarkificationAllowlist(thread); + return getRuntimeSolibDirBase(); + } + public String getRuntimeSolibDirBase() { return runtimeSolibDirBase; } @@ -251,6 +261,13 @@ public Label getToolchainType() { return toolchainType; } + @StarlarkMethod(name = "cc_toolchain_config_info", documented = false, useStarlarkThread = true) + public CcToolchainConfigInfo getCcToolchainConfigInfoForStarlark(StarlarkThread thread) + throws EvalException { + CcModule.checkPrivateStarlarkificationAllowlist(thread); + return getCcToolchainConfigInfo(); + } + public CcToolchainConfigInfo getCcToolchainConfigInfo() { return ccToolchainConfigInfo; } @@ -263,10 +280,34 @@ public LicensesProvider getLicensesProvider() { return licensesProvider; } + @StarlarkMethod( + name = "static_runtime_lib", + documented = false, + useStarlarkThread = true, + allowReturnNones = true) + @Nullable + public TransitiveInfoCollection getStaticRuntimeLibForStarlark(StarlarkThread thread) + throws EvalException { + CcModule.checkPrivateStarlarkificationAllowlist(thread); + return getStaticRuntimeLib(); + } + public TransitiveInfoCollection getStaticRuntimeLib() { return staticRuntimeLib; } + @StarlarkMethod( + name = "dynamic_runtime_lib", + documented = false, + useStarlarkThread = true, + allowReturnNones = true) + @Nullable + public TransitiveInfoCollection getDynamicRuntimeLibForStarlark(StarlarkThread thread) + throws EvalException { + CcModule.checkPrivateStarlarkificationAllowlist(thread); + return getDynamicRuntimeLib(); + } + public TransitiveInfoCollection getDynamicRuntimeLib() { return dynamicRuntimeLib; } @@ -303,6 +344,18 @@ public Artifact getLinkDynamicLibraryTool() { return linkDynamicLibraryTool; } + @StarlarkMethod( + name = "module_map", + documented = false, + useStarlarkThread = true, + allowReturnNones = true) + @Nullable + public TransitiveInfoCollection getModuleMapForStarlark(StarlarkThread thread) + throws EvalException { + CcModule.checkPrivateStarlarkificationAllowlist(thread); + return getModuleMap(); + } + public TransitiveInfoCollection getModuleMap() { return moduleMap; } @@ -331,6 +384,12 @@ public FdoProfileProvider getFdoOptimizeProvider() { return fdoOptimizeProvider; } + @StarlarkMethod(name = "module_map_artifact", documented = false, useStarlarkThread = true) + public Artifact getModuleMapArtifactForStarlark(StarlarkThread thread) throws EvalException { + CcModule.checkPrivateStarlarkificationAllowlist(thread); + return getModuleMapArtifact(); + } + public Artifact getModuleMapArtifact() { return moduleMapArtifact; } @@ -359,6 +418,13 @@ public NestedSet getFullInputsForLink() { return fullInputsForLink; } + @StarlarkMethod(name = "cc_toolchain_label", documented = false, useStarlarkThread = true) + public Label getCcToolchainLabelForStarlark(StarlarkThread thread) throws EvalException { + CcModule.checkPrivateStarlarkificationAllowlist(thread); + return getCcToolchainLabel(); + } + + @Override public Label getCcToolchainLabel() { return ccToolchainLabel; } @@ -371,6 +437,18 @@ public NestedSet getCompilerFilesWithoutIncludes() { return compilerFilesWithoutIncludes; } + @StarlarkMethod(name = "libc", documented = false, useStarlarkThread = true) + public Depset getLibcForStarlark(StarlarkThread thread) throws EvalException { + CcModule.checkPrivateStarlarkificationAllowlist(thread); + return Depset.of(Artifact.class, getLibc()); + } + + @StarlarkMethod(name = "target_libc", documented = false, useStarlarkThread = true) + public Depset getTargetLibcForstarlark(StarlarkThread thread) throws EvalException { + CcModule.checkPrivateStarlarkificationAllowlist(thread); + return Depset.of(Artifact.class, getTargetLibc()); + } + public NestedSet getLibc() { return libc; } @@ -383,11 +461,33 @@ public TransitiveInfoCollection getTargetLibcTop() { return targetLibcTop; } + @StarlarkMethod( + name = "libc_top_label", + documented = false, + allowReturnNones = true, + useStarlarkThread = true) + @Nullable + public Label getLibcTopLabelForStarlark(StarlarkThread thread) throws EvalException { + CcModule.checkPrivateStarlarkificationAllowlist(thread); + return getLibcTopLabel(); + } + @Nullable public Label getLibcTopLabel() { return getLibcTop() == null ? null : getLibcTop().getLabel(); } + @StarlarkMethod( + name = "target_libc_top_label", + documented = false, + allowReturnNones = true, + useStarlarkThread = true) + @Nullable + public Label getTargetLibcTopLabelForStarlark(StarlarkThread thread) throws EvalException { + CcModule.checkPrivateStarlarkificationAllowlist(thread); + return getTargetLibcTopLabel(); + } + @Nullable public Label getTargetLibcTopLabel() { return getTargetLibcTop() == null ? null : getTargetLibcTop().getLabel(); diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainConfigInfo.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainConfigInfo.java index 4764fbf9f4d34a..15275008ab836b 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainConfigInfo.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainConfigInfo.java @@ -14,6 +14,8 @@ package com.google.devtools.build.lib.rules.cpp; +import static com.google.common.collect.ImmutableList.toImmutableList; + import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -38,7 +40,11 @@ import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig; import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig.CToolchain; import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig.ToolPath; +import java.util.List; +import net.starlark.java.annot.StarlarkMethod; import net.starlark.java.eval.EvalException; +import net.starlark.java.eval.StarlarkThread; +import net.starlark.java.eval.Tuple; /** Information describing C++ toolchain derived from CROSSTOOL file. */ @Immutable @@ -175,6 +181,16 @@ public ArtifactNamePatternMapper getArtifactNamePatterns() { return artifactNamePatterns; } + @StarlarkMethod( + name = "cxx_builtin_include_directories", + documented = false, + useStarlarkThread = true) + public List getCxxBuiltinIncludeDirectoriesForStarlark(StarlarkThread thread) + throws EvalException { + CcModule.checkPrivateStarlarkificationAllowlist(thread); + return getCxxBuiltinIncludeDirectories(); + } + public ImmutableList getCxxBuiltinIncludeDirectories() { return cxxBuiltinIncludeDirectories; } @@ -187,6 +203,12 @@ public String getTargetSystemName() { return targetSystemName; } + @StarlarkMethod(name = "target_cpu", documented = false, useStarlarkThread = true) + public String getTargetCpuForStarlark(StarlarkThread thread) throws EvalException { + CcModule.checkPrivateStarlarkificationAllowlist(thread); + return getTargetCpu(); + } + public String getTargetCpu() { return targetCpu; } @@ -207,16 +229,39 @@ public String getAbiLibcVersion() { return abiLibcVersion; } + @StarlarkMethod(name = "tool_paths", documented = false, useStarlarkThread = true) + public ImmutableList getToolPathsForStarlark(StarlarkThread thread) throws EvalException { + CcModule.checkPrivateStarlarkificationAllowlist(thread); + return getToolPaths().stream() + .map(p -> Tuple.of(p.getFirst(), p.getSecond())) + .collect(toImmutableList()); + } + /** Returns a list of paths of the tools in the form Pair. */ public ImmutableList> getToolPaths() { return toolPaths; } + @StarlarkMethod(name = "make_variables", documented = false, useStarlarkThread = true) + public ImmutableList getMakevariablesForStarlark(StarlarkThread thread) + throws EvalException { + CcModule.checkPrivateStarlarkificationAllowlist(thread); + return getMakeVariables().stream() + .map(p -> Tuple.of(p.getFirst(), p.getSecond())) + .collect(toImmutableList()); + } + /** Returns a list of make variables that have the form Pair. */ public ImmutableList> getMakeVariables() { return makeVariables; } + @StarlarkMethod(name = "builtin_sysroot", documented = false, useStarlarkThread = true) + public String getBuiltinSysrootForStarlark(StarlarkThread thread) throws EvalException { + CcModule.checkPrivateStarlarkificationAllowlist(thread); + return getBuiltinSysroot(); + } + public String getBuiltinSysroot() { return builtinSysroot; } diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainFeatures.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainFeatures.java index 5cc589b2c037b0..24b4394266930c 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainFeatures.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainFeatures.java @@ -56,6 +56,7 @@ import javax.annotation.Nullable; import net.starlark.java.eval.EvalException; import net.starlark.java.eval.Starlark; +import net.starlark.java.eval.StarlarkValue; /** * Provides access to features supported by a specific toolchain. @@ -74,7 +75,7 @@ * them from build variables). */ @Immutable -public class CcToolchainFeatures { +public class CcToolchainFeatures implements StarlarkValue { /** * Thrown when a flag value cannot be expanded under a set of build variables. diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainSuite.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainSuite.java index a7468c766dbdf8..6e9a8234140ddd 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainSuite.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainSuite.java @@ -14,6 +14,8 @@ package com.google.devtools.build.lib.rules.cpp; import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.actions.MutableActionGraph.ActionConflictException; import com.google.devtools.build.lib.analysis.ConfiguredTarget; import com.google.devtools.build.lib.analysis.LicensesProvider; @@ -29,6 +31,8 @@ import com.google.devtools.build.lib.packages.BuiltinProvider; import java.util.Map; import javax.annotation.Nullable; +import net.starlark.java.eval.Starlark; +import net.starlark.java.eval.StarlarkFunction; /** * Implementation of the {@code cc_toolchain_suite} rule. @@ -74,8 +78,20 @@ public ConfiguredTarget create(RuleContext ruleContext) transformedCpu, compiler, selectedCcToolchain); + StarlarkFunction getCcToolchainProvider = + (StarlarkFunction) ruleContext.getStarlarkDefinedBuiltin("get_cc_toolchain_provider"); + ruleContext.initStarlarkRuleContext(); + Object starlarkCcToolchainProvider = + ruleContext.callStarlarkOrThrowRuleError( + getCcToolchainProvider, + ImmutableList.of( + /* ctx */ ruleContext.getStarlarkRuleContext(), /* attributes */ + selectedAttributes), + ImmutableMap.of()); ccToolchainProvider = - CcToolchainProviderHelper.getCcToolchainProvider(ruleContext, selectedAttributes); + starlarkCcToolchainProvider != Starlark.NONE + ? (CcToolchainProvider) starlarkCcToolchainProvider + : null; if (ccToolchainProvider == null) { // Skyframe restart diff --git a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/cpp/CcModuleApi.java b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/cpp/CcModuleApi.java index d186124bf78cd2..e33b6d259e1091 100755 --- a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/cpp/CcModuleApi.java +++ b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/cpp/CcModuleApi.java @@ -1241,6 +1241,12 @@ LinkingContextT createCcLinkingInfo( positional = false, named = true, defaultValue = "unbound"), + @Param( + name = "module_map", + documented = false, + positional = false, + named = true, + defaultValue = "unbound"), }) CompilationContextT createCcCompilationContext( Object headers, @@ -1254,6 +1260,7 @@ CompilationContextT createCcCompilationContext( Sequence directPublicHdrs, Sequence directPrivateHdrs, Object purpose, + Object moduleMap, StarlarkThread thread) throws EvalException; diff --git a/src/main/starlark/builtins_bzl/common/cc/cc_common.bzl b/src/main/starlark/builtins_bzl/common/cc/cc_common.bzl index 8246510ac474ca..9a057c93ad8406 100644 --- a/src/main/starlark/builtins_bzl/common/cc/cc_common.bzl +++ b/src/main/starlark/builtins_bzl/common/cc/cc_common.bzl @@ -419,11 +419,15 @@ def _create_compilation_context( direct_textual_headers = [], direct_public_headers = [], direct_private_headers = [], - purpose = _UNBOUND): - if purpose != _UNBOUND: + purpose = _UNBOUND, + module_map = _UNBOUND): + if purpose != _UNBOUND or \ + module_map != _UNBOUND: cc_common_internal.check_private_api(allowlist = _PRIVATE_STARLARKIFICATION_ALLOWLIST) if purpose == _UNBOUND: purpose = None + if module_map == _UNBOUND: + module_map = None return cc_common_internal.create_compilation_context( headers = headers, system_includes = system_includes, @@ -436,6 +440,7 @@ def _create_compilation_context( direct_public_headers = direct_public_headers, direct_private_headers = direct_private_headers, purpose = purpose, + module_map = module_map, ) def _legacy_cc_flags_make_variable_do_not_use(*, cc_toolchain): diff --git a/src/main/starlark/builtins_bzl/common/cc/cc_helper.bzl b/src/main/starlark/builtins_bzl/common/cc/cc_helper.bzl index a3cef8b333e894..54139b5bc9a829 100644 --- a/src/main/starlark/builtins_bzl/common/cc/cc_helper.bzl +++ b/src/main/starlark/builtins_bzl/common/cc/cc_helper.bzl @@ -1267,4 +1267,5 @@ cc_helper = struct( local_defines = _local_defines, linker_scripts = _linker_scripts, copts_filter = _copts_filter, + package_exec_path = _package_exec_path, ) diff --git a/src/main/starlark/builtins_bzl/common/cc/cc_toolchain_provider_helper.bzl b/src/main/starlark/builtins_bzl/common/cc/cc_toolchain_provider_helper.bzl new file mode 100644 index 00000000000000..3c9f9e3a74a150 --- /dev/null +++ b/src/main/starlark/builtins_bzl/common/cc/cc_toolchain_provider_helper.bzl @@ -0,0 +1,270 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A helper for creating CcToolchainProvider.""" + +load(":common/cc/cc_helper.bzl", "cc_helper") +load(":common/paths.bzl", "paths") +load(":common/cc/cc_common.bzl", "cc_common") + +cc_internal = _builtins.internal.cc_internal + +_TOOL_PATH_ONLY_TOOLS = [ + "gcov-tool", + "gcov", + "llvm-profdata", + "llvm-cov", +] + +_REQUIRED_TOOLS = [ + "ar", + "cpp", + "gcc", + "ld", + "nm", + "objdump", + "strip", +] + +_SYSROOT_START = "%sysroot%/" +_WORKSPACE_START = "%workspace%/" +_CROSSTOOL_START = "%crosstool_top%/" +_PACKAGE_START = "%package(" +_PACKAGE_END = ")%" +_BUILTIN_INCLUDE_FILE_SUFFIX = "include/stdc-predef.h" + +def _builtin_includes(libc): + result = [] + for artifact in libc.to_list(): + if artifact.path.endswith(_BUILTIN_INCLUDE_FILE_SUFFIX): + result.append(artifact) + return result + +def _legacy_cc_flags_make_variable(toolchain_make_vars): + legacy_cc_flags = "" + for variable in toolchain_make_vars: + if variable[0] == "CC_FLAGS": + legacy_cc_flags = variable[1] + return legacy_cc_flags + +def _additional_make_variables(toolchain_make_vars): + make_vars = {} + + # The following are to be used to allow some build rules to avoid the limits on stack frame + # sizes and variable-length arrays. + # These variables are initialized here, but may be overridden by the getMakeVariables() checks. + make_vars["STACK_FRAME_UNLIMITED"] = "" + for variable in toolchain_make_vars: + make_vars[variable[0]] = variable[1] + make_vars.pop("CC_FLAGS", None) + return make_vars + +def _compute_tool_paths(toolchain_config_info, crosstool_top_path): + tool_paths_collector = {} + for tool in toolchain_config_info.tool_paths(): + path_str = tool[1] + if not paths.is_normalized(path_str): + fail("The include path '" + path_str + "' is not normalized.") + tool_paths_collector[tool[0]] = paths.get_relative(crosstool_top_path, path_str) + + # These tools can only be declared using tool paths, so action-only toolchains should still + # be allowed to declared them while still being treated as an action-only toolchain. If a tool + # that can be specified with actions is declared using paths, the toolchain will be treated as + # a tool-path toolchain to enforce users to migrate their toolchain fully. + contains_all = True + for key in tool_paths_collector.keys(): + if key not in _TOOL_PATH_ONLY_TOOLS: + contains_all = False + break + if contains_all: + for tool in _REQUIRED_TOOLS: + tool_paths_collector[tool] = paths.get_relative(crosstool_top_path, tool) + else: + for tool in _REQUIRED_TOOLS: + if tool not in tool_paths_collector.keys(): + fail("Tool path for '" + tool + "' is missing") + return tool_paths_collector + +def _resolve_include_dir(target_label, s, sysroot, crosstool_path): + """ Resolve the given include directory. + + If it starts with %sysroot%/, that part is replaced with the actual sysroot. + + If it starts with %workspace%/, that part is replaced with the empty string (essentially + making it relative to the build directory). + + If it starts with %crosstool_top%/ or is any relative path, it is interpreted relative to + the crosstool top. The use of assumed-crosstool-relative specifications is considered + deprecated, and all such uses should eventually be replaced by "%crosstool_top%/". + + If it is of the form %package(@repository//my/package)%/folder, then it is interpreted as + the named folder in the appropriate package. All of the normal package syntax is supported. The + /folder part is optional. + + It is illegal if it starts with a % and does not match any of the above forms to avoid + accidentally silently ignoring misspelled prefixes. + + If it is absolute, it remains unchanged. + """ + package_end_index = s.find(_PACKAGE_END) + if package_end_index != -1 and s.startswith(_PACKAGE_START): + package = s[len(_PACKAGE_START):package_end_index] + if package.find(":") >= 0: + fail("invalid package identifier '" + package + "': contains ':'") + + # This is necessary to avoid the hard work of parsing, + # and use an already existing API. + dummy_label = target_label.relative(package + ":dummy_target") + repo_prefix = dummy_label.workspace_root + path_prefix = paths.get_relative(repo_prefix, dummy_label.package) + path_start_index = package_end_index + len(_PACKAGE_END) + if path_start_index + 1 < len(s): + if s[path_start_index] != "/": + fail("The path in the package for '" + s + "' is not valid") + path_string = s[path_start_index + 1:] + else: + path_string = "" + elif s.startswith(_SYSROOT_START): + if sysroot == None: + fail("A %sysroot% prefix is only allowed if the default_sysroot option is set") + path_prefix = sysroot + path_string = s[len(_SYSROOT_START):len(s)] + elif s.startswith(_WORKSPACE_START): + path_prefix = "" + path_string = s[len(_WORKSPACE_START):len(s)] + else: + path_prefix = crosstool_path + if s.startswith(_CROSSTOOL_START): + path_string = s[len(_CROSSTOOL_START):len(s)] + elif s.startswith("%"): + fail("The include path '" + s + "' has an " + "unrecognized %prefix%") + else: + path_string = s + + return paths.get_relative(path_prefix, path_string) + +def get_cc_toolchain_provider(ctx, attributes): + """Constructs a CcToolchainProvider instance. + + Args: + ctx: rule context. + attributes: an instance of CcToolchainAttributesProvider. + Returns: + A constructed CcToolchainProvider instance. + """ + toolchain_config_info = attributes.cc_toolchain_config_info() + tools_directory = cc_helper.package_exec_path( + ctx, + attributes.cc_toolchain_label().package, + ctx.configuration.is_sibling_repository_layout(), + ) + tool_paths = _compute_tool_paths(toolchain_config_info, tools_directory) + toolchain_features = cc_internal.cc_toolchain_features(toolchain_config_info = toolchain_config_info, tools_directory = tools_directory) + fdo_context = cc_internal.fdo_context( + ctx = ctx, + attributes = attributes, + configuration = ctx.configuration, + cpp_config = ctx.fragments.cpp, + tool_paths = tool_paths, + ) + if fdo_context == None: + return None + runtime_solib_dir_base = attributes.runtime_solib_dir_base() + runtime_solib_dir = paths.get_relative(ctx.bin_dir.path, runtime_solib_dir_base) + solib_directory = "_solib_" + toolchain_config_info.target_cpu() + default_sysroot = None + if toolchain_config_info.builtin_sysroot() != "": + default_sysroot = toolchain_config_info.builtin_sysroot() + if attributes.libc_top_label() == None: + sysroot = default_sysroot + else: + sysroot = attributes.libc_top_label().package + + if attributes.target_libc_top_label() == None: + target_sysroot = sysroot + else: + target_sysroot = attributes.target_libc_top_label().package + + static_runtime_lib = attributes.static_runtime_lib() + if static_runtime_lib != None: + static_runtime_link_inputs = static_runtime_lib[DefaultInfo].files + else: + static_runtime_link_inputs = None + + dynamic_runtime_lib = attributes.dynamic_runtime_lib() + if dynamic_runtime_lib != None: + dynamic_runtime_link_symlinks_elems = [] + for artifact in dynamic_runtime_lib[DefaultInfo].files.to_list(): + if cc_helper.is_valid_shared_library_artifact(artifact): + dynamic_runtime_link_symlinks_elems.append(cc_internal.solib_symlink_action( + ctx = ctx, + artifact = artifact, + solib_directory = solib_directory, + runtime_solib_dir_base = runtime_solib_dir_base, + )) + if len(dynamic_runtime_link_symlinks_elems) == 0: + dynamic_runtime_link_symlinks = depset() + else: + dynamic_runtime_link_symlinks = depset(direct = dynamic_runtime_link_symlinks_elems) + else: + dynamic_runtime_link_symlinks = None + + module_map = None + if attributes.module_map() != None and attributes.module_map_artifact() != None: + module_map = cc_common.create_module_map(file = attributes.module_map_artifact(), name = "crosstool") + + cc_compilation_context = cc_common.create_compilation_context(module_map = module_map) + + builtin_include_directories = [] + for s in toolchain_config_info.cxx_builtin_include_directories(): + builtin_include_directories.append(_resolve_include_dir(ctx.label, s, sysroot, tools_directory)) + + return cc_internal.construct_toolchain_provider( + ctx = ctx, + cpp_config = ctx.fragments.cpp, + toolchain_features = toolchain_features, + tools_directory = tools_directory, + attributes = attributes, + static_runtime_link_inputs = static_runtime_link_inputs, + dynamic_runtime_link_symlinks = dynamic_runtime_link_symlinks, + runtime_solib_dir = runtime_solib_dir, + cc_compilation_context = cc_compilation_context, + builtin_include_files = _builtin_includes(attributes.libc()), + target_builtin_include_files = _builtin_includes(attributes.target_libc()), + builtin_include_directories = builtin_include_directories, + sysroot = sysroot, + target_sysroot = target_sysroot, + fdo_context = fdo_context, + is_tool_configuration = ctx.configuration.is_tool_configuration(), + tool_paths = tool_paths, + toolchain_config_info = toolchain_config_info, + default_sysroot = default_sysroot, + # The runtime sysroot should really be set from --grte_top. However, currently libc has + # no way to set the sysroot. The CROSSTOOL file does set the runtime sysroot, in the + # builtin_sysroot field. This implies that you can not arbitrarily mix and match + # Crosstool and libc versions, you must always choose compatible ones. + runtime_sysroot = default_sysroot, + solib_directory = solib_directory, + additional_make_variables = _additional_make_variables(toolchain_config_info.make_variables()), + legacy_cc_flags_make_variable = _legacy_cc_flags_make_variable(toolchain_config_info.make_variables()), + objcopy = tool_paths.get("objcopy", ""), + compiler = tool_paths.get("gcc", ""), + preprocessor = tool_paths.get("cpp", ""), + nm = tool_paths.get("nm", ""), + objdump = tool_paths.get("objdump", ""), + ar = tool_paths.get("ar", ""), + strip = tool_paths.get("strip", ""), + ld = tool_paths.get("ld", ""), + gcov = tool_paths.get("gcov", ""), + ) diff --git a/src/main/starlark/builtins_bzl/common/exports.bzl b/src/main/starlark/builtins_bzl/common/exports.bzl index 2a07fbaf5d62ff..5e1efd108fe8f2 100755 --- a/src/main/starlark/builtins_bzl/common/exports.bzl +++ b/src/main/starlark/builtins_bzl/common/exports.bzl @@ -16,6 +16,7 @@ load("@_builtins//:common/cc/cc_import.bzl", "cc_import") load("@_builtins//:common/cc/cc_binary_wrapper.bzl", "cc_binary") +load("@_builtins//:common/cc/cc_toolchain_provider_helper.bzl", "get_cc_toolchain_provider") load("@_builtins//:common/cc/cc_test_wrapper.bzl", cc_test = "cc_test_wrapper") load("@_builtins//:common/cc/experimental_cc_shared_library.bzl", "CcSharedLibraryHintInfo", "CcSharedLibraryInfo", "cc_shared_library") load("@_builtins//:common/objc/objc_import.bzl", "objc_import") @@ -79,4 +80,5 @@ exported_to_java = { "proto_common_experimental_should_generate_code": proto_common_do_not_use.experimental_should_generate_code, "proto_common_experimental_filter_sources": proto_common_do_not_use.experimental_filter_sources, "link_multi_arch_static_library": linking_support.link_multi_arch_static_library, + "get_cc_toolchain_provider": get_cc_toolchain_provider, } diff --git a/src/test/java/com/google/devtools/build/lib/rules/cpp/CcToolchainTest.java b/src/test/java/com/google/devtools/build/lib/rules/cpp/CcToolchainTest.java index 1f300369262489..4ef441c27ce48a 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/cpp/CcToolchainTest.java +++ b/src/test/java/com/google/devtools/build/lib/rules/cpp/CcToolchainTest.java @@ -306,7 +306,7 @@ public void assertInvalidIncludeDirectoryMessage(String entry, String messageReg public void testInvalidIncludeDirectory() throws Exception { assertInvalidIncludeDirectoryMessage("%package(//a", "has an unrecognized %prefix%"); assertInvalidIncludeDirectoryMessage( - "%package(//a:@@a)%", "The package '//a:@@a' is not valid"); + "%package(//a:@@a)%", "invalid package identifier '//a:@@a': contains ':'"); assertInvalidIncludeDirectoryMessage( "%package(//a)%foo", "The path in the package.*is not valid"); }