Skip to content

Commit

Permalink
[antlir2][chef_solo] restructure so all deps are in chef_solo feature
Browse files Browse the repository at this point in the history
Summary:
Instead of installing some files into the build appliance, just install them
all into the target image using the chef_solo feature.

This not only consolidates (almost) all chef nonsense into a specific directory
in antlir2 (where before it's been growing a bit out of control in a bunch of
different places), this also brings pretty significant devx wins to anyone
working on antlir2, because this diff no longer invalidates the cache of
anything that comes before `feature.chef_solo`.

Test Plan:
```
❯ buck2 test fbcode//antlir/antlir2/test_images/facebook/chef_solo:
Buck UI: https://www.internalfb.com/buck2/0e3aecdc-1da0-40a9-ac19-0f1eb343393a
Test UI: https://www.internalfb.com/intern/testinfra/testrun/17732923535603330
Tests finished: Pass 36. Fail 0. Fatal 0. Skip 0. Build failure 0
```

Reviewed By: sergeyfd

Differential Revision: D49466968

fbshipit-source-id: 14b1bf95e4695c32187864d4708d2d94adb9cb85
  • Loading branch information
vmagro authored and facebook-github-bot committed Sep 21, 2023
1 parent 5be09a5 commit 1f845e9
Show file tree
Hide file tree
Showing 12 changed files with 28 additions and 12 deletions.
4 changes: 3 additions & 1 deletion antlir/antlir2/bzl/feature/clone.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,8 @@ clone_record = record(
def clone_analyze(
src_path: str,
dst_path: str,
deps: dict[str, Dependency]) -> FeatureAnalysis:
deps: dict[str, Dependency],
impl: RunInfo) -> FeatureAnalysis:
omit_outer_dir = src_path.endswith("/")
pre_existing_dest = dst_path.endswith("/")
if omit_outer_dir and not pre_existing_dest:
Expand All @@ -105,4 +106,5 @@ def clone_analyze(
pre_existing_dest = pre_existing_dest,
),
required_layers = [src_layer[LayerInfo]],
impl_run_info = impl,
)
2 changes: 1 addition & 1 deletion antlir/antlir2/bzl/feature/dot_meta.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ dot_meta_record = record(

def dot_meta_analyze(
build_info: [dict[str, typing.Any], None],
impl: [RunInfo, None] = None) -> FeatureAnalysis:
impl: RunInfo) -> FeatureAnalysis:
return FeatureAnalysis(
feature_type = "dot_meta",
data = dot_meta_record(
Expand Down
3 changes: 3 additions & 0 deletions antlir/antlir2/bzl/feature/extract.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ extract_record = record(

def extract_analyze(
ctx: AnalyzeFeatureContext,
impl: RunInfo,
source: str,
deps: dict[str, Dependency],
binaries: list[str] | None = None,
Expand All @@ -115,6 +116,7 @@ def extract_analyze(
buck = None,
),
required_layers = [layer[LayerInfo]],
impl_run_info = impl,
)
elif source == "buck":
src = deps["src"]
Expand Down Expand Up @@ -145,6 +147,7 @@ def extract_analyze(
),
required_artifacts = [src],
required_run_infos = [src_runinfo],
impl_run_info = impl,
)
else:
fail("invalid extract source '{}'".format(source))
3 changes: 2 additions & 1 deletion antlir/antlir2/bzl/feature/feature.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,7 @@ def _impl(ctx: AnalysisContext) -> list[Provider]:
),
label = ctx.label,
)
analyze_kwargs["impl"] = ctx.attrs.inline_features_impls[key][RunInfo]

analysis = _analyze_feature[inline["feature_type"]](**analyze_kwargs)
for analysis in flatten.flatten(analysis):
Expand All @@ -168,7 +169,7 @@ def _impl(ctx: AnalysisContext) -> list[Provider]:
feature_type = analysis.feature_type,
label = ctx.label.raw_target(),
analysis = analysis,
run_info = analysis.impl_run_info or ctx.attrs.inline_features_impls[key][RunInfo],
run_info = analysis.impl_run_info,
)
inline_features.append(feat)

Expand Down
2 changes: 1 addition & 1 deletion antlir/antlir2/bzl/feature/feature_info.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ def data_only_feature_analysis_fn(
record_type,
feature_type: str,
build_phase: BuildPhase = BuildPhase("compile")):
def inner(impl: RunInfo | None = None, **kwargs) -> FeatureAnalysis:
def inner(impl: RunInfo, **kwargs) -> FeatureAnalysis:
return FeatureAnalysis(
feature_type = feature_type,
data = record_type(**kwargs),
Expand Down
4 changes: 3 additions & 1 deletion antlir/antlir2/bzl/feature/genrule.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,8 @@ def genrule_analyze(
boot: bool,
bind_repo_ro: bool,
mount_platform: bool,
args: dict[str, str | ResolvedStringWithMacros]) -> FeatureAnalysis:
args: dict[str, str | ResolvedStringWithMacros],
impl: RunInfo) -> FeatureAnalysis:
cmd = {int(key.removeprefix("cmd_")): val for key, val in args.items() if key.startswith("cmd_")}
cmd = [val for _key, val in sorted(cmd.items())]
return FeatureAnalysis(
Expand All @@ -56,4 +57,5 @@ def genrule_analyze(
mount_platform = mount_platform,
),
build_phase = BuildPhase("genrule"),
impl_run_info = impl,
)
6 changes: 3 additions & 3 deletions antlir/antlir2/bzl/feature/install.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ def get_feature_anaylsis_for_install(
user: str,
skip_debuginfo_split: bool,
text: str | None,
impl: RunInfo | None = None):
impl: RunInfo):
binary_info = None
required_run_infos = []
required_artifacts = []
Expand Down Expand Up @@ -182,13 +182,13 @@ def get_feature_anaylsis_for_install(

def install_analyze(
ctx: AnalyzeFeatureContext,
impl: RunInfo,
dst: str,
group: str,
mode: int | None,
user: str,
text: str | None,
deps_or_srcs: dict[str, Artifact | Dependency] | None = None,
impl: RunInfo | None = None) -> FeatureAnalysis:
deps_or_srcs: dict[str, Artifact | Dependency] | None = None) -> FeatureAnalysis:
src = None if not deps_or_srcs else deps_or_srcs["src"]
return get_feature_anaylsis_for_install(
ctx,
Expand Down
3 changes: 3 additions & 0 deletions antlir/antlir2/bzl/feature/metakv.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,8 @@ metakv_record = record(
)

def metakv_analyze(
*,
impl: RunInfo,
store: dict[str, typing.Any] | None = None,
remove: dict[str, typing.Any] | None = None) -> FeatureAnalysis:
return FeatureAnalysis(
Expand All @@ -87,4 +89,5 @@ def metakv_analyze(
store = metakv_store_record(**store) if store else None,
remove = metakv_remove_record(**remove) if remove else None,
),
impl_run_info = impl,
)
3 changes: 3 additions & 0 deletions antlir/antlir2/bzl/feature/mount.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,7 @@ def mount_analyze(
is_directory: bool | None,
host_source: str | None,
_implicit_from_antlir1: bool,
impl: RunInfo,
deps: dict[str, Dependency] = {}) -> FeatureAnalysis:
if source_kind == "layer":
source = deps.pop("source")
Expand All @@ -128,6 +129,7 @@ def mount_analyze(
host = None,
),
required_layers = [source[LayerInfo]],
impl_run_info = impl,
)
elif source_kind == "host":
return FeatureAnalysis(
Expand All @@ -140,6 +142,7 @@ def mount_analyze(
),
layer = None,
),
impl_run_info = impl,
)
else:
fail("invalid source_kind '{}'".format(source_kind))
5 changes: 3 additions & 2 deletions antlir/antlir2/bzl/feature/rpms.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -143,12 +143,13 @@ rpms_record = record(
)

def rpms_analyze(
*,
ctx: AnalyzeFeatureContext,
impl: RunInfo,
action: str,
subjects: list[str],
srcs: dict[str, Artifact] = {},
unnamed_deps_or_srcs: list[Dependency | Artifact] = [],
impl: RunInfo | None = None) -> FeatureAnalysis:
unnamed_deps_or_srcs: list[Dependency | Artifact] = []) -> FeatureAnalysis:
rpms = []
for rpm in subjects:
rpms.append(rpm_source_record(subject = rpm))
Expand Down
4 changes: 3 additions & 1 deletion antlir/antlir2/bzl/feature/tarball.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,8 @@ def tarball_analyze(
into_dir: str,
user: str,
group: str,
srcs: dict[str, Artifact]) -> FeatureAnalysis:
srcs: dict[str, Artifact],
impl: RunInfo) -> FeatureAnalysis:
tarball = srcs["source"]

if user != "root" or group != "root":
Expand All @@ -65,6 +66,7 @@ def tarball_analyze(
),
feature_type = "install",
required_artifacts = [extracted],
impl_run_info = impl,
)

def _extract_impl(ctx: AnalysisContext) -> list[Provider]:
Expand Down
1 change: 0 additions & 1 deletion antlir/antlir2/dnf/BUCK
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@ feature.new(
dst = "/__antlir2__/dnf/versionlock.json",
mode = "a+rx",
),
# @oss-disable
],
visibility = ["//antlir/antlir2/build_appliance:features"],
)
Expand Down

0 comments on commit 1f845e9

Please sign in to comment.