diff --git a/CHANGELOG.D/560.feature b/CHANGELOG.D/560.feature new file mode 100644 index 00000000..06e6babf --- /dev/null +++ b/CHANGELOG.D/560.feature @@ -0,0 +1 @@ +Renamed `inherits` yaml property to `mixins`. diff --git a/CHANGELOG.D/562.feature b/CHANGELOG.D/562.feature new file mode 100644 index 00000000..3bac5cfe --- /dev/null +++ b/CHANGELOG.D/562.feature @@ -0,0 +1,2 @@ +Added support project level `mixins`. Mixins defined in `project.yml` available both for live and batch configurations, +so they cannot define any live or batch specific properties. diff --git a/neuro_flow/ast.py b/neuro_flow/ast.py index 9cd55806..e4b6586a 100644 --- a/neuro_flow/ast.py +++ b/neuro_flow/ast.py @@ -64,6 +64,9 @@ class Project(Base): images: Optional[Mapping[str, "Image"]] = field(metadata={"allow_none": True}) volumes: Optional[Mapping[str, "Volume"]] = field(metadata={"allow_none": True}) defaults: Optional["BatchFlowDefaults"] = field(metadata={"allow_none": True}) + mixins: Optional[Mapping[str, "ExecUnitMixin"]] = field( + metadata={"allow_none": True} + ) # There are 'batch' for pipelined mode and 'live' for interactive one @@ -95,6 +98,26 @@ class Image(Base): force_rebuild: OptBoolExpr +@dataclass(frozen=True) +class ExecUnitMixin(WithSpecifiedFields, Base): + title: OptStrExpr # Autocalculated if not passed explicitly + name: OptStrExpr + image: OptStrExpr + preset: OptStrExpr + schedule_timeout: OptTimeDeltaExpr + entrypoint: OptStrExpr + cmd: OptStrExpr + workdir: OptRemotePathExpr + env: Optional[BaseExpr[MappingT]] = field(metadata={"allow_none": True}) + volumes: Optional[BaseExpr[SequenceT]] = field(metadata={"allow_none": True}) + tags: Optional[BaseExpr[SequenceT]] = field(metadata={"allow_none": True}) + life_span: OptTimeDeltaExpr + http_port: OptIntExpr + http_auth: OptBoolExpr + pass_config: OptBoolExpr + mixins: Optional[Sequence[StrExpr]] = field(metadata={"allow_none": True}) + + @dataclass(frozen=True) class ExecUnit(Base): title: OptStrExpr # Autocalculated if not passed explicitly @@ -170,7 +193,7 @@ class JobMixin(WithSpecifiedFields, Base): port_forward: Optional[BaseExpr[SequenceT]] = field(metadata={"allow_none": True}) multi: SimpleOptBoolExpr params: Optional[Mapping[str, Param]] = field(metadata={"allow_none": True}) - inherits: Optional[Sequence[StrExpr]] = field(metadata={"allow_none": True}) + mixins: Optional[Sequence[StrExpr]] = field(metadata={"allow_none": True}) @dataclass(frozen=True) @@ -181,7 +204,7 @@ class Job(ExecUnit, WithSpecifiedFields, JobBase): browse: OptBoolExpr port_forward: Optional[BaseExpr[SequenceT]] = field(metadata={"allow_none": True}) multi: SimpleOptBoolExpr - inherits: Optional[Sequence[StrExpr]] = field(metadata={"allow_none": True}) + mixins: Optional[Sequence[StrExpr]] = field(metadata={"allow_none": True}) class NeedsLevel(enum.Enum): @@ -209,7 +232,7 @@ class TaskBase(Base): @dataclass(frozen=True) class Task(ExecUnit, WithSpecifiedFields, TaskBase): - inherits: Optional[Sequence[StrExpr]] = field(metadata={"allow_none": True}) + mixins: Optional[Sequence[StrExpr]] = field(metadata={"allow_none": True}) @dataclass(frozen=True) @@ -233,7 +256,7 @@ class TaskMixin(WithSpecifiedFields, Base): strategy: Optional[Strategy] = field(metadata={"allow_none": True}) enable: EnableExpr = field(metadata={"default_expr": "${{ success() }}"}) cache: Optional[Cache] = field(metadata={"allow_none": True}) - inherits: Optional[Sequence[StrExpr]] = field(metadata={"allow_none": True}) + mixins: Optional[Sequence[StrExpr]] = field(metadata={"allow_none": True}) @dataclass(frozen=True) diff --git a/neuro_flow/context.py b/neuro_flow/context.py index b5dd4d74..252b23c8 100644 --- a/neuro_flow/context.py +++ b/neuro_flow/context.py @@ -1149,7 +1149,7 @@ def _specified_fields(self) -> AbstractSet[str]: async def merge_asts(child: _MergeTarget, parent: SupportsAstMerge) -> _MergeTarget: child_fields = {f.name for f in dataclasses.fields(child)} for field in parent._specified_fields: - if field == "inherits" or field not in child_fields: + if field == "mixins" or field not in child_fields: continue field_present = field in child._specified_fields child_value = getattr(child, field) @@ -1178,7 +1178,7 @@ async def merge_asts(child: _MergeTarget, parent: SupportsAstMerge) -> _MergeTar class MixinApplyTarget(Protocol): @property - def inherits(self) -> Optional[Sequence[StrExpr]]: + def mixins(self) -> Optional[Sequence[StrExpr]]: ... @property @@ -1192,9 +1192,9 @@ def _specified_fields(self) -> AbstractSet[str]: async def apply_mixins( base: _MixinApplyTarget, mixins: Mapping[str, SupportsAstMerge] ) -> _MixinApplyTarget: - if base.inherits is None: + if base.mixins is None: return base - for mixin_expr in reversed(base.inherits): + for mixin_expr in reversed(base.mixins): mixin_name = await mixin_expr.eval(EMPTY_ROOT) try: mixin = mixins[mixin_name] @@ -1215,10 +1215,8 @@ async def setup_mixins( return {} graph: Dict[str, Dict[str, int]] = {} for mixin_name, mixin in raw_mixins.items(): - inherits = mixin.inherits or [] - graph[mixin_name] = { - await dep_expr.eval(EMPTY_ROOT): 1 for dep_expr in inherits - } + mixins = mixin.mixins or [] + graph[mixin_name] = {await dep_expr.eval(EMPTY_ROOT): 1 for dep_expr in mixins} topo = ColoredTopoSorter(graph) result: Dict[str, _MixinApplyTarget] = {} while not topo.is_all_colored(1): @@ -1232,7 +1230,7 @@ class RunningLiveFlow: _ast_flow: ast.LiveFlow _ctx: LiveContext _cl: ConfigLoader - _mixins: Optional[Mapping[str, ast.JobMixin]] = None + _mixins: Mapping[str, SupportsAstMerge] def __init__( self, @@ -1240,11 +1238,13 @@ def __init__( ctx: LiveContext, config_loader: ConfigLoader, defaults: DefaultsConf, + mixins: Mapping[str, SupportsAstMerge], ): self._ast_flow = ast_flow self._ctx = ctx self._cl = config_loader self._defaults = defaults + self._mixins = mixins @property def job_ids(self) -> Iterable[str]: @@ -1274,18 +1274,13 @@ async def is_multi(self, job_id: str) -> bool: # Simple shortcut return (await self.get_meta(job_id)).multi - async def get_mixins(self) -> Mapping[str, ast.JobMixin]: - if self._mixins is None: - self._mixins = await setup_mixins(self._ast_flow.mixins) - return self._mixins - async def _get_job_ast( self, job_id: str ) -> Union[ast.Job, ast.JobActionCall, ast.JobModuleCall]: try: base = self._ast_flow.jobs[job_id] if isinstance(base, ast.Job): - base = await apply_mixins(base, await self.get_mixins()) + base = await apply_mixins(base, self._mixins) return base except KeyError: raise UnknownJob(job_id) @@ -1491,7 +1486,13 @@ async def create( images=images, ) - return cls(ast_flow, live_ctx, config_loader, defaults) + raw_mixins: Mapping[str, MixinApplyTarget] = { + **(ast_project.mixins or {}), + **(ast_flow.mixins or {}), + } + mixins = await setup_mixins(raw_mixins) + + return cls(ast_flow, live_ctx, config_loader, defaults, mixins) _T = TypeVar("_T", bound=BaseBatchContext, covariant=True) @@ -1514,7 +1515,7 @@ def graph(self) -> Mapping[str, Mapping[str, ast.NeedsLevel]]: @property @abstractmethod - def mixins(self) -> Optional[Mapping[str, ast.TaskMixin]]: + def mixins(self) -> Optional[Mapping[str, SupportsAstMerge]]: pass @property @@ -1647,7 +1648,7 @@ def __init__( config_loader: ConfigLoader, action: ast.BatchAction, parent_ctx_class: Type[RootABC], - mixins: Optional[Mapping[str, ast.TaskMixin]], + mixins: Optional[Mapping[str, SupportsAstMerge]], ): super().__init__(ctx, tasks, config_loader) self._action = action @@ -1660,7 +1661,7 @@ def early_images(self) -> Mapping[str, EarlyImageCtx]: return self._early_images @property - def mixins(self) -> Optional[Mapping[str, ast.TaskMixin]]: + def mixins(self) -> Optional[Mapping[str, SupportsAstMerge]]: return self._mixins def get_image_ast(self, image_id: str) -> ast.Image: @@ -1942,7 +1943,7 @@ def __init__( local_info: Optional[LocallyPreparedInfo], ast_flow: ast.BatchFlow, ast_project: ast.Project, - mixins: Optional[Mapping[str, ast.TaskMixin]], + mixins: Optional[Mapping[str, SupportsAstMerge]], ): super().__init__( ctx, @@ -1969,7 +1970,7 @@ def get_image_ast(self, image_id: str) -> ast.Image: raise @property - def mixins(self) -> Optional[Mapping[str, ast.TaskMixin]]: + def mixins(self) -> Optional[Mapping[str, SupportsAstMerge]]: return self._mixins @property @@ -2079,7 +2080,11 @@ async def create( ), ) - mixins = await setup_mixins(ast_flow.mixins) + raw_mixins: Mapping[str, MixinApplyTarget] = { + **(ast_project.mixins or {}), + **(ast_flow.mixins or {}), + } + mixins = await setup_mixins(raw_mixins) tasks = await TaskGraphBuilder( batch_ctx, config_loader, cache_conf, ast_flow.tasks, mixins ).build() @@ -2109,7 +2114,7 @@ def __init__( action: ast.BatchAction, bake_id: str, local_info: Optional[LocallyPreparedInfo], - mixins: Optional[Mapping[str, ast.TaskMixin]], + mixins: Optional[Mapping[str, SupportsAstMerge]], ): super().__init__( flow_ctx, @@ -2130,7 +2135,7 @@ def get_image_ast(self, image_id: str) -> ast.Image: return self._action.images[image_id] @property - def mixins(self) -> Optional[Mapping[str, ast.TaskMixin]]: + def mixins(self) -> Optional[Mapping[str, SupportsAstMerge]]: return self._mixins async def calc_outputs(self, task_results: NeedsCtx) -> DepCtx: @@ -2162,7 +2167,7 @@ async def create( bake_id: str, local_info: Optional[LocallyPreparedInfo], defaults: DefaultsConf = DefaultsConf(), - mixins: Optional[Mapping[str, ast.TaskMixin]] = None, + mixins: Optional[Mapping[str, SupportsAstMerge]] = None, ) -> "RunningBatchActionFlow": step_1_ctx = BatchActionContextStep1( inputs=inputs, @@ -2503,7 +2508,7 @@ def __init__( self, config_loader: ConfigLoader, ast_tasks: Sequence[Union[ast.Task, ast.TaskActionCall, ast.TaskModuleCall]], - mixins: Optional[Mapping[str, ast.TaskMixin]], + mixins: Optional[Mapping[str, SupportsAstMerge]], ): self._cl = config_loader self._ast_tasks = ast_tasks @@ -2696,7 +2701,7 @@ def __init__( config_loader: ConfigLoader, default_cache: CacheConf, ast_tasks: Sequence[Union[ast.Task, ast.TaskActionCall, ast.TaskModuleCall]], - mixins: Optional[Mapping[str, ast.TaskMixin]], + mixins: Optional[Mapping[str, SupportsAstMerge]], ): super().__init__(config_loader, ast_tasks, mixins) self._ctx = ctx diff --git a/neuro_flow/expr.py b/neuro_flow/expr.py index 685172f0..605f7dd8 100644 --- a/neuro_flow/expr.py +++ b/neuro_flow/expr.py @@ -475,11 +475,18 @@ async def eval(self, root: RootABC) -> LiteralT: return self.val -def literal(toktype: str) -> Parser: - def f(tok: Token) -> Any: +def make_toktype_predicate(toktype: str) -> Callable[[Token], bool]: + def _predicate(token: Token) -> bool: + return token.type == toktype + + return _predicate + + +def literal(toktype: str) -> "Parser[Token, Literal]": + def f(tok: Token) -> Literal: return Literal(tok.start, tok.end, literal_eval(tok.value)) - return some(lambda tok: tok.type == toktype) >> f + return some(make_toktype_predicate(toktype)) >> f class Getter(Entity): @@ -756,7 +763,7 @@ def make_list(args: Tuple[Item, List[Item]]) -> ListMaker: return ListMaker(lst[0].start, lst[-1].end, lst) -def make_empty_list(args: Tuple[Item, Item]) -> ListMaker: +def make_empty_list(args: Tuple[Token, Token]) -> ListMaker: return ListMaker(args[0].start, args[1].end, []) @@ -778,23 +785,29 @@ def child_items(self) -> Iterable["Item"]: yield entry[1] -def make_dict(args: Tuple[Item, Item, List[Tuple[Item, Item]]]) -> DictMaker: +def make_dict( + args: Union[Tuple[Item, Item, List[Tuple[Item, Item]]], Tuple[Item, Item]] +) -> DictMaker: lst = [(args[0], args[1])] if len(args) > 2: - lst += args[2] + lst += args[2] # type: ignore return DictMaker(lst[0][0].start, lst[-1][1].end, lst) -def make_empty_dict(args: Tuple[Item, Item]) -> DictMaker: +def make_empty_dict(args: Tuple[Token, Token]) -> DictMaker: return DictMaker(args[0].start, args[0].end, []) -def a(value: str) -> Parser: +def a(value: str) -> "Parser[Token, Token]": """Eq(a) -> Parser(a, a) Returns a parser that parses a token that is equal to the value value. """ - return some(lambda t: t.value == value).named(f'(a "{value}")') + + def _is_value_eq(token: Token) -> bool: + return token.value == value + + return some(_is_value_eq).named(f'(a "{value}")') DOT: Final = skip(a(".")) @@ -837,7 +850,7 @@ def a(value: str) -> Parser: LITERAL: Final = NONE | BOOL | REAL | INT | STR -NAME: Final = some(lambda tok: tok.type == "NAME") +NAME: Final = some(make_toktype_predicate("NAME")) LIST_MAKER: Final = forward_decl() @@ -937,13 +950,13 @@ def a(value: str) -> Parser: + maybe(COMMA) + RBRACE ) - >> make_dict + >> make_dict # type: ignore | (a("{") + a("}")) >> make_empty_dict ) TMPL: Final = (OPEN_TMPL + EXPR + CLOSE_TMPL) | (OPEN_TMPL2 + EXPR + CLOSE_TMPL2) -TEXT: Final = some(lambda tok: tok.type == "TEXT") >> make_text +TEXT: Final = some(make_toktype_predicate("TEXT")) >> make_text PARSER: Final = oneplus(TMPL | TEXT) + skip(finished) diff --git a/neuro_flow/parser.py b/neuro_flow/parser.py index 5b0e54d5..3d556130 100644 --- a/neuro_flow/parser.py +++ b/neuro_flow/parser.py @@ -635,6 +635,11 @@ def parse_strategy(ctor: BaseConstructor, node: yaml.MappingNode) -> ast.Strateg "pass_config": OptBoolExpr, } +EXEC_UNIT_MIXIN: Dict[str, Any] = { + **EXEC_UNIT, + "mixins": SimpleSeq(StrExpr), +} + JOB_MIXIN = { **EXEC_UNIT, "detach": OptBoolExpr, @@ -642,7 +647,7 @@ def parse_strategy(ctor: BaseConstructor, node: yaml.MappingNode) -> ast.Strateg "port_forward": ExprOrSeq(PortPairExpr, port_pair_item), "multi": SimpleOptBoolExpr, "params": None, - "inherits": SimpleSeq(StrExpr), + "mixins": SimpleSeq(StrExpr), } JOB: Dict[str, Any] = { @@ -652,7 +657,7 @@ def parse_strategy(ctor: BaseConstructor, node: yaml.MappingNode) -> ast.Strateg "port_forward": ExprOrSeq(PortPairExpr, port_pair_item), "multi": SimpleOptBoolExpr, "params": None, - "inherits": SimpleSeq(StrExpr), + "mixins": SimpleSeq(StrExpr), } JOB_ACTION_CALL = { @@ -806,7 +811,7 @@ def parse_mixins( TASK: Mapping[str, Any] = { **TASK_BASE, **EXEC_UNIT, - "inherits": SimpleSeq(StrExpr), + "mixins": SimpleSeq(StrExpr), } TASK_MIXIN: Mapping[str, Any] = { @@ -815,7 +820,7 @@ def parse_mixins( "strategy": None, "enable": EnableExpr, "cache": None, - "inherits": SimpleSeq(StrExpr), + "mixins": SimpleSeq(StrExpr), } @@ -1475,6 +1480,7 @@ def __init__(self, stream: TextIO) -> None: "images": None, "volumes": None, "defaults": None, + "mixins": None, } @@ -1510,6 +1516,22 @@ def parse_project_defaults( ) +def parse_project_mixins( + ctor: FlowLoader, node: yaml.MappingNode +) -> Dict[str, ast.ExecUnitMixin]: + ret = {} + for k, v in node.value: + key = ctor.construct_id(k) + value = parse_dict( + ctor, + v, + EXEC_UNIT_MIXIN, + ast.ExecUnitMixin, + ) + ret[key] = value + return ret + + ProjectLoader.add_path_resolver("project:main", []) # type: ignore ProjectLoader.add_constructor("project:main", parse_project_main) # type: ignore @@ -1531,6 +1553,9 @@ def parse_project_defaults( ProjectLoader.add_path_resolver("project:volumes", [(dict, "volumes")]) # type: ignore ProjectLoader.add_constructor("project:volumes", parse_volumes) # type: ignore +ProjectLoader.add_path_resolver("project:mixins", [(dict, "mixins")]) # type: ignore +ProjectLoader.add_constructor("project:mixins", parse_project_mixins) # type: ignore + def parse_project_stream(stream: TextIO) -> ast.Project: ret: ast.Project @@ -1575,4 +1600,5 @@ def make_default_project(workspace_stem: str) -> ast.Project: defaults=None, images=None, volumes=None, + mixins=None, ) diff --git a/reference/mixins.md b/reference/mixins.md index 787f6a80..82059e51 100644 --- a/reference/mixins.md +++ b/reference/mixins.md @@ -13,13 +13,13 @@ mixins: ... ``` -Mixins can define the same properties as "job" and "task" do, except for the "id" field in batch mode. To apply a mixin, use the `inherits` property: +Mixins can define the same properties as "job" and "task" do, except for the "id" field in batch mode. To apply a mixin, use the `mixins` property: ```yaml ... jobs: test_a: - inherits: [ credentials ] + mixins: [ credentials ] ... ``` @@ -35,10 +35,10 @@ mixins: image: example_mix2 jobs: job1: - inherits: [mix1, mix2] + mixins: [mix1, mix2] image: example job2: - inherits: [mix1, mix2] + mixins: [mix1, mix2] ``` In this case, `job1` will use the `example` image and `job2` will use the `example_mix2` image. @@ -54,7 +54,7 @@ mixins: env: TEST: 1 mix2: - inherits: [ mix1 ] + mixins: [ mix1 ] image: example ``` ([#498](https://github.com/neuro-inc/neuro-flow/issues/498)) ``` \ No newline at end of file diff --git a/requirements/base.txt b/requirements/base.txt index aa9e474e..a48f7655 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -3,7 +3,7 @@ async_exit_stack==1.0.1; python_version<"3.7" backports-datetime-fromisoformat==1.0.0; python_version<"3.7" click==8.0.1 dataclasses==0.7; python_version<"3.7" -funcparserlib==0.3.6 +funcparserlib==1.0.0a0 graphviz==0.17 humanize==3.11.0 neuro-cli==21.9.2 diff --git a/setup.py b/setup.py index 761f0329..6e5310d9 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ install_requires=[ "neuro-cli>=21.8.27", "pyyaml>=5.4", - "funcparserlib>=0.3", + "funcparserlib>=1.0.0a0", 'dataclasses>=0.5; python_version<"3.7"', "humanize>=0.5.1", 'backports-datetime-fromisoformat>=1.0.0; python_version<"3.7"', diff --git a/tests/unit/batch-mixin.yml b/tests/unit/batch-mixin.yml index 2bbad2b2..b9745576 100644 --- a/tests/unit/batch-mixin.yml +++ b/tests/unit/batch-mixin.yml @@ -4,8 +4,8 @@ mixins: image: ubuntu preset: cpu-micro tasks: - - inherits: [basic] + - mixins: [basic] bash: echo abc - - inherits: [basic] + - mixins: [basic] bash: echo def diff --git a/tests/unit/batch-sub-mixin.yml b/tests/unit/batch-sub-mixin.yml index 3a6e3f23..bfef8424 100644 --- a/tests/unit/batch-sub-mixin.yml +++ b/tests/unit/batch-sub-mixin.yml @@ -5,7 +5,7 @@ mixins: env1: val-mixin1-1 env2: val-mixin1-2 envs2: - inherits: [envs1] + mixins: [envs1] env: env2: val-mixin2-2 env3: val-mixin2-3 @@ -13,8 +13,8 @@ mixins: image: ubuntu preset: cpu-micro tasks: - - inherits: [basic, envs1] + - mixins: [basic, envs1] bash: echo abc - - inherits: [basic, envs2] + - mixins: [basic, envs2] bash: echo def diff --git a/tests/unit/batch_mixins/batch-module-uses-mixin.yml b/tests/unit/batch_mixins/batch-module-uses-mixin.yml index d98a6fff..0d505052 100644 --- a/tests/unit/batch_mixins/batch-module-uses-mixin.yml +++ b/tests/unit/batch_mixins/batch-module-uses-mixin.yml @@ -1,5 +1,5 @@ kind: batch tasks: - id: task_1 - inherits: [basic] + mixins: [basic] bash: echo abc diff --git a/tests/unit/live-full.yml b/tests/unit/live-full.yml index 9a93bf40..2610ff39 100644 --- a/tests/unit/live-full.yml +++ b/tests/unit/live-full.yml @@ -42,7 +42,7 @@ mixins: jobs: test_a: name: job-name - inherits: [ envs ] + mixins: [ envs ] image: ${{ images.image_a.ref }} preset: cpu-micro schedule_timeout: 1d2h3m4s diff --git a/tests/unit/live-mixins.yml b/tests/unit/live-mixins.yml index c94a8a4b..e05f1a7c 100644 --- a/tests/unit/live-mixins.yml +++ b/tests/unit/live-mixins.yml @@ -27,15 +27,15 @@ mixins: jobs: test: name: job-name - inherits: [ envs1, envs2, preset, image ] + mixins: [ envs1, envs2, preset, image ] env: env1: val1 env2: val2 test2: name: job-name - inherits: [ image, image2 ] + mixins: [ image, image2 ] test3: name: job-name - inherits: [ image, volumes1, volumes2 ] + mixins: [ image, volumes1, volumes2 ] test4: - inherits: [ image, expression ] + mixins: [ image, expression ] diff --git a/tests/unit/live-sub-mixins.yml b/tests/unit/live-sub-mixins.yml index 7cbb199a..af81f2ed 100644 --- a/tests/unit/live-sub-mixins.yml +++ b/tests/unit/live-sub-mixins.yml @@ -5,7 +5,7 @@ mixins: env1: val-mixin1-1 env2: val-mixin1-2 envs2: - inherits: [envs1] + mixins: [envs1] env: env2: val-mixin2-2 env3: val-mixin2-3 @@ -13,4 +13,4 @@ jobs: test: name: job-name image: ubuntu - inherits: [ envs2] + mixins: [ envs2] diff --git a/tests/unit/test_action.py b/tests/unit/test_action.py index 29b81a07..68f61561 100644 --- a/tests/unit/test_action.py +++ b/tests/unit/test_action.py @@ -73,7 +73,7 @@ def test_parse_live_action(assets: LocalPath) -> None: Pos(11, 2, config_file), Pos(13, 0, config_file), _specified_fields={"cmd", "image"}, - inherits=None, + mixins=None, name=OptStrExpr(Pos(3, 4, config_file), Pos(5, 0, config_file), None), image=OptStrExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), "ubuntu"), preset=OptStrExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), None), @@ -263,7 +263,7 @@ def test_parse_batch_action(assets: LocalPath) -> None: Pos(36, 2, config_file), Pos(40, 0, config_file), _specified_fields={"needs", "image", "cmd", "id"}, - inherits=None, + mixins=None, title=OptStrExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), None), name=OptStrExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), None), image=OptStrExpr( @@ -313,7 +313,7 @@ def test_parse_batch_action(assets: LocalPath) -> None: Pos(40, 2, config_file), Pos(43, 0, config_file), _specified_fields={"image", "cmd", "id"}, - inherits=None, + mixins=None, title=OptStrExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), None), name=OptStrExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), None), image=OptStrExpr( diff --git a/tests/unit/test_batch_parser.py b/tests/unit/test_batch_parser.py index e4b01997..176fe598 100644 --- a/tests/unit/test_batch_parser.py +++ b/tests/unit/test_batch_parser.py @@ -197,7 +197,7 @@ def test_parse_minimal(assets: pathlib.Path) -> None: "id", "volumes", }, - inherits=None, + mixins=None, id=OptIdExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), "test_a"), title=OptStrExpr( Pos(0, 0, config_file), Pos(0, 0, config_file), "Batch title" @@ -325,7 +325,7 @@ def test_parse_seq(assets: pathlib.Path) -> None: _start=Pos(2, 4, config_file), _end=Pos(6, 2, config_file), _specified_fields={"preset", "cmd", "image"}, - inherits=None, + mixins=None, id=OptIdExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), None), title=OptStrExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), None), needs=None, @@ -375,7 +375,7 @@ def test_parse_seq(assets: pathlib.Path) -> None: _start=Pos(6, 4, config_file), _end=Pos(9, 0, config_file), _specified_fields={"preset", "cmd", "image"}, - inherits=None, + mixins=None, id=OptIdExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), None), title=OptStrExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), None), needs=None, @@ -456,7 +456,7 @@ def test_parse_needs(assets: pathlib.Path) -> None: _start=Pos(2, 4, config_file), _end=Pos(7, 2, config_file), _specified_fields={"cmd", "image", "id", "preset"}, - inherits=None, + mixins=None, id=OptIdExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), "task_a"), title=OptStrExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), None), needs=None, @@ -506,7 +506,7 @@ def test_parse_needs(assets: pathlib.Path) -> None: _start=Pos(7, 4, config_file), _end=Pos(11, 0, config_file), _specified_fields={"needs", "image", "cmd", "preset"}, - inherits=None, + mixins=None, id=OptIdExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), None), title=OptStrExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), None), needs={ @@ -591,7 +591,7 @@ def test_parse_needs_dict(assets: pathlib.Path) -> None: _start=Pos(2, 4, config_file), _end=Pos(7, 2, config_file), _specified_fields={"preset", "image", "cmd", "id"}, - inherits=None, + mixins=None, id=OptIdExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), "task_a"), title=OptStrExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), None), needs=None, @@ -641,7 +641,7 @@ def test_parse_needs_dict(assets: pathlib.Path) -> None: _start=Pos(7, 4, config_file), _end=Pos(12, 0, config_file), _specified_fields={"preset", "image", "cmd", "needs"}, - inherits=None, + mixins=None, id=OptIdExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), None), title=OptStrExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), None), needs={ @@ -728,7 +728,7 @@ def test_parse_matrix(assets: pathlib.Path) -> None: _start=Pos(2, 4, config_file), _end=Pos(14, 0, config_file), _specified_fields={"strategy", "image", "cmd"}, - inherits=None, + mixins=None, title=OptStrExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), None), name=OptStrExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), None), image=OptStrExpr( @@ -885,7 +885,7 @@ def test_parse_matrix_with_strategy(assets: pathlib.Path) -> None: _start=Pos(8, 4, config_file), _end=Pos(25, 2, config_file), _specified_fields={"image", "strategy", "cmd", "cache"}, - inherits=None, + mixins=None, title=OptStrExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), None), name=OptStrExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), None), image=OptStrExpr( @@ -991,7 +991,7 @@ def test_parse_matrix_with_strategy(assets: pathlib.Path) -> None: Pos(25, 4, config_file), Pos(28, 0, config_file), _specified_fields={"id", "image", "cmd"}, - inherits=None, + mixins=None, id=OptIdExpr( Pos(25, 8, config_file), Pos(25, 14, config_file), "simple" ), @@ -1154,7 +1154,7 @@ def test_parse_args(assets: pathlib.Path) -> None: config_file, ), _specified_fields={"image", "cmd"}, - inherits=None, + mixins=None, title=OptStrExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), None), name=OptStrExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), None), image=OptStrExpr( @@ -1233,7 +1233,7 @@ def test_parse_enable(assets: pathlib.Path) -> None: _start=Pos(2, 4, config_file), _end=Pos(6, 2, config_file), _specified_fields={"cmd", "id", "preset", "image"}, - inherits=None, + mixins=None, id=OptIdExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), "task_a"), title=OptStrExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), None), needs=None, @@ -1283,7 +1283,7 @@ def test_parse_enable(assets: pathlib.Path) -> None: _start=Pos(6, 4, config_file), _end=Pos(11, 0, config_file), _specified_fields={"enable", "image", "needs", "cmd", "preset"}, - inherits=None, + mixins=None, id=OptIdExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), None), title=OptStrExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), None), needs={ @@ -1367,7 +1367,7 @@ def test_parse_mixin(assets: pathlib.Path) -> None: Pos(3, 4, config_file), Pos(5, 0, config_file), _specified_fields={"image", "preset"}, - inherits=None, + mixins=None, name=OptStrExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), None), image=OptStrExpr( Pos(0, 0, config_file), @@ -1417,8 +1417,8 @@ def test_parse_mixin(assets: pathlib.Path) -> None: ast.Task( _start=Pos(6, 4, config_file), _end=Pos(9, 2, config_file), - _specified_fields={"inherits", "cmd"}, - inherits=[ + _specified_fields={"mixins", "cmd"}, + mixins=[ StrExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), "basic") ], id=OptIdExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), None), @@ -1465,8 +1465,8 @@ def test_parse_mixin(assets: pathlib.Path) -> None: ast.Task( _start=Pos(9, 4, config_file), _end=Pos(11, 0, config_file), - _specified_fields={"inherits", "cmd"}, - inherits=[ + _specified_fields={"mixins", "cmd"}, + mixins=[ StrExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), "basic") ], id=OptIdExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), None), diff --git a/tests/unit/test_context.py b/tests/unit/test_context.py index 10e6a11f..678a0062 100644 --- a/tests/unit/test_context.py +++ b/tests/unit/test_context.py @@ -165,6 +165,22 @@ async def test_project_level_defaults_live( await cl.close() +async def test_project_level_mixins_live(assets: pathlib.Path, client: Client) -> None: + ws = assets / "with_project_yaml" + config_dir = ConfigDir( + workspace=ws, + config_dir=ws, + ) + cl = LiveLocalCL(config_dir, client) + try: + flow = await RunningLiveFlow.create(cl, "live") + job = await flow.get_job("test_mixin", {}) + assert job.image == "mixin-image" + assert job.preset == "mixin-preset" + finally: + await cl.close() + + async def test_local_remote_path_images( client: Client, live_config_loader: ConfigLoader ) -> None: @@ -1172,3 +1188,20 @@ async def test_batch_with_project_globals(assets: pathlib.Path, client: Client) finally: await cl.close() + + +async def test_batch_with_project_mixins(assets: pathlib.Path, client: Client) -> None: + ws = assets / "with_project_yaml" + config_dir = ConfigDir( + workspace=ws, + config_dir=ws, + ) + cl = BatchLocalCL(config_dir, client) + try: + flow = await RunningBatchFlow.create(cl, "batch", "bake-id") + task = await flow.get_task((), "test_mixin", needs={}, state={}) + assert task.image == "mixin-image" + assert task.preset == "mixin-preset" + + finally: + await cl.close() diff --git a/tests/unit/test_live_parser.py b/tests/unit/test_live_parser.py index 63a97748..211dc9fa 100644 --- a/tests/unit/test_live_parser.py +++ b/tests/unit/test_live_parser.py @@ -59,7 +59,7 @@ def test_parse_minimal(assets: pathlib.Path) -> None: Pos(3, 4, config_file), Pos(5, 0, config_file), _specified_fields={"cmd", "image"}, - inherits=None, + mixins=None, name=OptStrExpr(Pos(3, 4, config_file), Pos(5, 0, config_file), None), image=OptStrExpr( Pos(0, 0, config_file), Pos(0, 0, config_file), "ubuntu" @@ -136,7 +136,7 @@ def test_parse_params(assets: pathlib.Path) -> None: Pos(3, 4, config_file), Pos(11, 0, config_file), _specified_fields={"cmd", "image", "params"}, - inherits=None, + mixins=None, name=OptStrExpr(Pos(3, 4, config_file), Pos(5, 0, config_file), None), image=OptStrExpr( Pos(0, 0, config_file), Pos(0, 0, config_file), "ubuntu" @@ -378,7 +378,7 @@ def test_parse_full(assets: pathlib.Path) -> None: Pos(38, 4, config_file), Pos(41, 0, config_file), _specified_fields={"env"}, - inherits=None, + mixins=None, name=OptStrExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), None), image=OptStrExpr( Pos(0, 0, config_file), @@ -458,12 +458,12 @@ def test_parse_full(assets: pathlib.Path) -> None: "name", "tags", "image", - "inherits", + "mixins", "browse", "volumes", "entrypoint", }, - inherits=[ + mixins=[ StrExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), "envs") ], name=OptStrExpr( @@ -729,7 +729,7 @@ def test_parse_full_exprs(assets: pathlib.Path) -> None: "volumes", "detach", }, - inherits=None, + mixins=None, name=OptStrExpr( Pos(0, 0, config_file), Pos(0, 0, config_file), "job-name" ), @@ -834,7 +834,7 @@ def test_parse_bash(assets: pathlib.Path) -> None: Pos(3, 4, config_file), Pos(7, 0, config_file), _specified_fields={"cmd", "image"}, - inherits=None, + mixins=None, name=OptStrExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), None), image=OptStrExpr( Pos(0, 0, config_file), Pos(0, 0, config_file), "ubuntu" @@ -913,7 +913,7 @@ def test_parse_python(assets: pathlib.Path) -> None: Pos(3, 4, config_file), Pos(7, 0, config_file), _specified_fields={"cmd", "image"}, - inherits=None, + mixins=None, name=OptStrExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), None), image=OptStrExpr( Pos(0, 0, config_file), Pos(0, 0, config_file), "ubuntu" @@ -1040,7 +1040,7 @@ def test_parse_multi(assets: pathlib.Path) -> None: Pos(3, 4, config_file), Pos(6, 0, config_file), _specified_fields={"cmd", "multi", "image"}, - inherits=None, + mixins=None, name=OptStrExpr(Pos(3, 4, config_file), Pos(5, 0, config_file), None), image=OptStrExpr( Pos(0, 0, config_file), Pos(0, 0, config_file), "ubuntu" @@ -1117,7 +1117,7 @@ def test_parse_explicit_flow_id(assets: pathlib.Path) -> None: Pos(4, 4, config_file), Pos(6, 0, config_file), _specified_fields={"cmd", "image"}, - inherits=None, + mixins=None, name=OptStrExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), None), image=OptStrExpr( Pos(0, 0, config_file), Pos(0, 0, config_file), "ubuntu" diff --git a/tests/unit/test_project_parser.py b/tests/unit/test_project_parser.py index d4654ef9..b33b2170 100644 --- a/tests/unit/test_project_parser.py +++ b/tests/unit/test_project_parser.py @@ -26,7 +26,7 @@ def test_parse_full(assets: pathlib.Path) -> None: project = parse_project_stream(stream) assert project == ast.Project( Pos(0, 0, config_file), - Pos(42, 0, config_file), + Pos(46, 0, config_file), id=SimpleIdExpr( Pos(0, 0, config_file), Pos(0, 0, config_file), @@ -191,4 +191,47 @@ def test_parse_full(assets: pathlib.Path) -> None: ), max_parallel=OptIntExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), 20), ), + mixins={ + "basic": ast.ExecUnitMixin( + Pos(44, 4, config_file), + Pos(46, 0, config_file), + _specified_fields={"image", "preset"}, + mixins=None, + name=OptStrExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), None), + image=OptStrExpr( + Pos(0, 0, config_file), + Pos(0, 0, config_file), + "mixin-image", + ), + preset=OptStrExpr( + Pos(0, 0, config_file), Pos(0, 0, config_file), "mixin-preset" + ), + schedule_timeout=OptTimeDeltaExpr( + Pos(0, 0, config_file), Pos(0, 0, config_file), None + ), + entrypoint=OptStrExpr( + Pos(0, 0, config_file), Pos(0, 0, config_file), None + ), + cmd=OptStrExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), None), + workdir=OptRemotePathExpr( + Pos(0, 0, config_file), Pos(0, 0, config_file), None + ), + env=None, + volumes=None, + tags=None, + life_span=OptTimeDeltaExpr( + Pos(0, 0, config_file), Pos(0, 0, config_file), None + ), + title=OptStrExpr(Pos(0, 0, config_file), Pos(0, 0, config_file), None), + http_port=OptIntExpr( + Pos(0, 0, config_file), Pos(0, 0, config_file), None + ), + http_auth=OptBoolExpr( + Pos(0, 0, config_file), Pos(0, 0, config_file), None + ), + pass_config=OptBoolExpr( + Pos(0, 0, config_file), Pos(0, 0, config_file), None + ), + ), + }, ) diff --git a/tests/unit/with_project_yaml/batch.yml b/tests/unit/with_project_yaml/batch.yml index de7e1106..1c20a482 100644 --- a/tests/unit/with_project_yaml/batch.yml +++ b/tests/unit/with_project_yaml/batch.yml @@ -10,3 +10,6 @@ tasks: bash: echo OK volumes: - ${{ volumes.volume_a.ref }} +- id: test_mixin + needs: [] + mixins: [basic] diff --git a/tests/unit/with_project_yaml/live.yml b/tests/unit/with_project_yaml/live.yml index 8c78e6c9..32004686 100644 --- a/tests/unit/with_project_yaml/live.yml +++ b/tests/unit/with_project_yaml/live.yml @@ -4,3 +4,5 @@ jobs: image: ${{ images.image_a.ref }} volumes: - ${{ volumes.volume_a.ref }} + test_mixin: + mixins: [basic] diff --git a/tests/unit/with_project_yaml/project.yml b/tests/unit/with_project_yaml/project.yml index 9a72e256..ed38c3e1 100644 --- a/tests/unit/with_project_yaml/project.yml +++ b/tests/unit/with_project_yaml/project.yml @@ -40,3 +40,7 @@ defaults: cache: strategy: none life_span: 2h30m +mixins: + basic: + image: mixin-image + preset: mixin-preset