Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: skip all migrators on schema v1 for now #3006

Merged
merged 5 commits into from
Sep 19, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 14 additions & 2 deletions conda_forge_tick/feedstock_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -230,8 +230,10 @@ def populate_feedstock_attributes(
if isinstance(feedstock_dir, str):
feedstock_dir = Path(feedstock_dir)

if meta_yaml is None and recipe_yaml is None:
raise ValueError("Either `meta_yaml` or `recipe_yaml` needs to be given.")
if (meta_yaml is None and recipe_yaml is None) or (
meta_yaml is not None and recipe_yaml is not None
):
raise ValueError("Either `meta_yaml` or `recipe_yaml` needs to be given.")

sub_graph.update({"feedstock_name": name, "parsing_error": False, "branch": "main"})

Expand All @@ -245,6 +247,8 @@ def populate_feedstock_attributes(

if isinstance(meta_yaml, str):
sub_graph["raw_meta_yaml"] = meta_yaml
elif isinstance(recipe_yaml, str):
sub_graph["raw_meta_yaml"] = recipe_yaml

# Get the conda-forge.yml
if isinstance(conda_forge_yaml, str):
Expand Down Expand Up @@ -301,6 +305,7 @@ def populate_feedstock_attributes(
),
),
)
variant_yamls[-1]["schema_version"] = 0
elif isinstance(recipe_yaml, str):
platform_arch = (
f"{plat}-{arch}"
Expand All @@ -314,6 +319,9 @@ def populate_feedstock_attributes(
cbc_path=cbc_path,
),
)
variant_yamls[-1]["schema_version"] = variant_yamls[-1].get(
"schema_version", 1
)

# sometimes the requirements come out to None or [None]
# and this ruins the aggregated meta_yaml / breaks stuff
Expand Down Expand Up @@ -358,6 +366,10 @@ def populate_feedstock_attributes(
parse_meta_yaml(meta_yaml, platform=plat, arch=arch)
for plat, arch in plat_archs
]
elif isinstance(recipe_yaml, str):
raise NotImplementedError(
"recipe_yaml generic parsing not implemented yet! Ensure the feedstock has .ci_support files."
)
except Exception as e:
import traceback

Expand Down
37 changes: 35 additions & 2 deletions conda_forge_tick/migrators/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,28 @@
logger = logging.getLogger(__name__)


def _skip_due_to_schema(
attrs: "AttrsTypedDict", allowed_schema_versions: List[int]
) -> bool:
__name = attrs.get("name", "")
schema_version = get_keys_default(
attrs,
["meta_yaml", "schema_version"],
{},
0,
)
if schema_version not in allowed_schema_versions:
logger.debug(
"%s: schema version not allowed - %r not in %r",
__name,
attrs["meta_yaml"].get("schema_version", 0),
allowed_schema_versions,
)
return True
else:
return False


def _make_effective_graph(graph, migrator):
"""Prune graph only to nodes that need rebuilds."""
gx2 = copy.deepcopy(graph)
Expand Down Expand Up @@ -142,6 +164,7 @@ def make_from_lazy_json_data(data):

class MiniMigrator:
post_migration = False
allowed_schema_versions = [0]

def __init__(self):
if not hasattr(self, "_init_args"):
Expand Down Expand Up @@ -206,6 +229,8 @@ class Migrator:

allow_empty_commits = False

allowed_schema_versions = [0]

build_patterns = (
(re.compile(r"(\s*?)number:\s*([0-9]+)"), "number: {}"),
(
Expand Down Expand Up @@ -369,7 +394,12 @@ def parse_already_pred() -> bool:
if bad_attr:
logger.debug("%s: bad attr - %s", __name, bad_attr)

return attrs.get("archived", False) or parse_already_pred() or bad_attr
return (
attrs.get("archived", False)
or parse_already_pred()
or bad_attr
or _skip_due_to_schema(attrs, self.allowed_schema_versions)
)

def get_possible_feedstock_branches(self, attrs: "AttrsTypedDict") -> List[str]:
"""Return the valid possible branches to which to apply this migration to
Expand Down Expand Up @@ -764,7 +794,10 @@ def filter(self, attrs: "AttrsTypedDict", not_bad_str_start: str = "") -> bool:
name = attrs.get("name", "")

if super().filter(attrs, "Upstream:"):
logger.debug("filter %s: archived or done or bad attr", name)
logger.debug(
"filter %s: archived or done or bad attr or schema_version not allowed",
name,
)
return True

if attrs.get("feedstock_name", None) not in self.graph:
Expand Down
26 changes: 19 additions & 7 deletions conda_forge_tick/migrators/cross_compile.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import typing
from typing import Any

from conda_forge_tick.migrators.core import MiniMigrator
from conda_forge_tick.migrators.core import MiniMigrator, _skip_due_to_schema
from conda_forge_tick.os_utils import pushd
from conda_forge_tick.provide_source_code import provide_source_code
from conda_forge_tick.utils import yaml_safe_dump, yaml_safe_load
Expand All @@ -28,7 +28,7 @@ def filter(self, attrs: "AttrsTypedDict", not_bad_str_start: str = "") -> bool:
if compiler in build_reqs:
needed = True
break
return not needed
return (not needed) or _skip_due_to_schema(attrs, self.allowed_schema_versions)


class UpdateConfigSubGuessMigrator(CrossCompilationMigratorBase):
Expand Down Expand Up @@ -150,7 +150,11 @@ class CrossPythonMigrator(CrossCompilationMigratorBase):
def filter(self, attrs: "AttrsTypedDict", not_bad_str_start: str = "") -> bool:
host_reqs = attrs.get("requirements", {}).get("host", set())
build_reqs = attrs.get("requirements", {}).get("build", set())
return "python" not in host_reqs or "python" in build_reqs
return (
"python" not in host_reqs
or "python" in build_reqs
or _skip_due_to_schema(attrs, self.allowed_schema_versions)
)

def migrate(self, recipe_dir: str, attrs: "AttrsTypedDict", **kwargs: Any) -> None:
host_reqs = attrs.get("requirements", {}).get("host", set())
Expand Down Expand Up @@ -207,7 +211,9 @@ def migrate(self, recipe_dir: str, attrs: "AttrsTypedDict", **kwargs: Any) -> No
class UpdateCMakeArgsMigrator(CrossCompilationMigratorBase):
def filter(self, attrs: "AttrsTypedDict", not_bad_str_start: str = "") -> bool:
build_reqs = attrs.get("requirements", {}).get("build", set())
return "cmake" not in build_reqs
return "cmake" not in build_reqs or _skip_due_to_schema(
attrs, self.allowed_schema_versions
)

def migrate(self, recipe_dir: str, attrs: "AttrsTypedDict", **kwargs: Any) -> None:
with pushd(recipe_dir):
Expand Down Expand Up @@ -238,6 +244,7 @@ def filter(self, attrs: "AttrsTypedDict", not_bad_str_start: str = "") -> bool:
build_reqs = attrs.get("requirements", {}).get("build", set())
host_reqs = attrs.get("requirements", {}).get("host", set())
run_reqs = attrs.get("requirements", {}).get("run", set())
skip_schema = _skip_due_to_schema(attrs, self.allowed_schema_versions)
if (
len(attrs.get("outputs_names", [])) <= 1
and "python" in build_reqs
Expand All @@ -252,6 +259,7 @@ def filter(self, attrs: "AttrsTypedDict", not_bad_str_start: str = "") -> bool:
"cxx_compiler_stub",
]
)
and not skip_schema
):
return False
else:
Expand Down Expand Up @@ -281,7 +289,8 @@ class NoCondaInspectMigrator(MiniMigrator):
post_migration = True

def filter(self, attrs: "AttrsTypedDict", not_bad_str_start: str = "") -> bool:
if "conda inspect" in attrs.get("raw_meta_yaml", ""):
skip_schema = _skip_due_to_schema(attrs, self.allowed_schema_versions)
if "conda inspect" in attrs.get("raw_meta_yaml", "") and not skip_schema:
return False
else:
return True
Expand Down Expand Up @@ -314,7 +323,10 @@ def migrate(self, recipe_dir: str, attrs: "AttrsTypedDict", **kwargs: Any) -> No
class CrossRBaseMigrator(CrossCompilationMigratorBase):
def filter(self, attrs: "AttrsTypedDict", not_bad_str_start: str = "") -> bool:
host_reqs = attrs.get("requirements", {}).get("host", set())
if "r-base" in host_reqs or attrs.get("name", "").startswith("r-"):
skip_schema = _skip_due_to_schema(attrs, self.allowed_schema_versions)
if (
"r-base" in host_reqs or attrs.get("name", "").startswith("r-")
) and not skip_schema:
return False
else:
return True
Expand Down Expand Up @@ -371,7 +383,7 @@ class CrossCompilationForARMAndPower(MiniMigrator):
post_migration = True

def filter(self, attrs: "AttrsTypedDict", not_bad_str_start: str = "") -> bool:
return False
return _skip_due_to_schema(attrs, self.allowed_schema_versions)

def migrate(self, recipe_dir: str, attrs: "AttrsTypedDict", **kwargs: Any) -> None:
with pushd(recipe_dir):
Expand Down
8 changes: 6 additions & 2 deletions conda_forge_tick/migrators/cstdlib.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import os
import re

from conda_forge_tick.migrators.core import MiniMigrator
from conda_forge_tick.migrators.core import MiniMigrator, _skip_due_to_schema
from conda_forge_tick.migrators.libboost import _replacer, _slice_into_output_sections

pat_stub = re.compile(r"(c|cxx|fortran)_compiler_stub")
Expand Down Expand Up @@ -200,7 +200,11 @@ def filter(self, attrs, not_bad_str_start=""):
has_compiler = any(pat_compiler.search(line) for line in lines)
has_sysroot = any(pat_sysroot_217.search(line) for line in lines)
# filter() returns True if we _don't_ want to migrate
return already_migrated or not (has_compiler or has_sysroot)
return (
_skip_due_to_schema(attrs, self.allowed_schema_versions)
or already_migrated
or not (has_compiler or has_sysroot)
)

def migrate(self, recipe_dir, attrs, **kwargs):
new_lines = []
Expand Down
4 changes: 2 additions & 2 deletions conda_forge_tick/migrators/dep_updates.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import typing
from typing import Any

from conda_forge_tick.migrators.core import MiniMigrator
from conda_forge_tick.migrators.core import MiniMigrator, _skip_due_to_schema
from conda_forge_tick.update_deps import apply_dep_update, get_dep_updates_and_hints
from conda_forge_tick.utils import get_keys_default

Expand Down Expand Up @@ -32,7 +32,7 @@ def filter(self, attrs: "AttrsTypedDict", not_bad_str_start: str = "") -> bool:
"hint",
)
if update_deps in ["update-all", "update-source", "update-grayskull"]:
return False
return False or _skip_due_to_schema(attrs, self.allowed_schema_versions)

return True

Expand Down
4 changes: 2 additions & 2 deletions conda_forge_tick/migrators/duplicate_lines.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import typing
from typing import Any

from conda_forge_tick.migrators.core import MiniMigrator
from conda_forge_tick.migrators.core import MiniMigrator, _skip_due_to_schema
from conda_forge_tick.os_utils import pushd

if typing.TYPE_CHECKING:
Expand All @@ -23,7 +23,7 @@ def filter(self, attrs: "AttrsTypedDict", not_bad_str_start: str = "") -> bool:
raw_yaml = attrs.get("raw_meta_yaml", "")
for line in raw_yaml.splitlines():
if any(r.match(line) for r in self.regex_to_check.values()):
return False
return False or _skip_due_to_schema(attrs, self.allowed_schema_versions)

return True

Expand Down
4 changes: 2 additions & 2 deletions conda_forge_tick/migrators/extra_jinj2a_keys_cleanup.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import typing
from typing import Any

from conda_forge_tick.migrators.core import MiniMigrator
from conda_forge_tick.migrators.core import MiniMigrator, _skip_due_to_schema
from conda_forge_tick.os_utils import pushd

if typing.TYPE_CHECKING:
Expand Down Expand Up @@ -34,7 +34,7 @@ def filter(self, attrs: "AttrsTypedDict", not_bad_str_start: str = "") -> bool:
raw_yaml = attrs["raw_meta_yaml"]
for var_name in self.vars_to_remove:
if f"{{% set {var_name}" in raw_yaml:
return False
return False or _skip_due_to_schema(attrs, self.allowed_schema_versions)
return True

def _replace_jinja_key(self, key_name, lines):
Expand Down
6 changes: 4 additions & 2 deletions conda_forge_tick/migrators/jinja2_vars_cleanup.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import typing
from typing import Any

from conda_forge_tick.migrators.core import MiniMigrator
from conda_forge_tick.migrators.core import MiniMigrator, _skip_due_to_schema
from conda_forge_tick.os_utils import pushd

if typing.TYPE_CHECKING:
Expand Down Expand Up @@ -34,7 +34,9 @@ class Jinja2VarsCleanup(MiniMigrator):
"""Cleanup the jinja2 vars by replacing {{name}} with {{ name }} etc."""

def filter(self, attrs: "AttrsTypedDict", not_bad_str_start: str = "") -> bool:
return _should_filter(attrs.get("raw_meta_yaml", ""))
return _should_filter(attrs.get("raw_meta_yaml", "")) or _skip_due_to_schema(
attrs, self.allowed_schema_versions
)

def migrate(self, recipe_dir: str, attrs: "AttrsTypedDict", **kwargs: Any) -> None:
with pushd(recipe_dir):
Expand Down
6 changes: 4 additions & 2 deletions conda_forge_tick/migrators/jpegturbo.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import os

from conda_forge_tick.migrators.core import MiniMigrator
from conda_forge_tick.migrators.core import MiniMigrator, _skip_due_to_schema


def _parse_jpeg(lines):
Expand All @@ -15,7 +15,9 @@ def _parse_jpeg(lines):
class JpegTurboMigrator(MiniMigrator):
def filter(self, attrs, not_bad_str_start=""):
host_req = (attrs.get("requirements", {}) or {}).get("host", set()) or set()
return "jpeg" not in host_req
return "jpeg" not in host_req or _skip_due_to_schema(
attrs, self.allowed_schema_versions
)

def migrate(self, recipe_dir, attrs, **kwargs):
fname = os.path.join(recipe_dir, "meta.yaml")
Expand Down
6 changes: 4 additions & 2 deletions conda_forge_tick/migrators/libboost.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import os
import re

from conda_forge_tick.migrators.core import MiniMigrator
from conda_forge_tick.migrators.core import MiniMigrator, _skip_due_to_schema


def _slice_into_output_sections(meta_yaml_lines, attrs):
Expand Down Expand Up @@ -214,7 +214,9 @@ def filter(self, attrs, not_bad_str_start=""):
run_req = (attrs.get("requirements", {}) or {}).get("run", set()) or set()
all_req = set(host_req) | set(run_req)
# filter() returns True if we _don't_ want to migrate
return not bool({"boost", "boost-cpp"} & all_req)
return (not bool({"boost", "boost-cpp"} & all_req)) or _skip_due_to_schema(
attrs, self.allowed_schema_versions
)

def migrate(self, recipe_dir, attrs, **kwargs):
fname = os.path.join(recipe_dir, "meta.yaml")
Expand Down
4 changes: 2 additions & 2 deletions conda_forge_tick/migrators/license.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import typing
from typing import Any

from conda_forge_tick.migrators.core import MiniMigrator
from conda_forge_tick.migrators.core import MiniMigrator, _skip_due_to_schema
from conda_forge_tick.os_utils import pushd
from conda_forge_tick.provide_source_code import provide_source_code
from conda_forge_tick.recipe_parser import CondaMetaYAML
Expand Down Expand Up @@ -293,7 +293,7 @@ def filter(self, attrs: "AttrsTypedDict", not_bad_str_start: str = "") -> bool:
or any(n in license_fam for n in NEEDED_FAMILIES)
or _is_r(attrs)
) and "license_file" not in attrs.get("meta_yaml", {}).get("about", {}):
return False
return False or _skip_due_to_schema(attrs, self.allowed_schema_versions)
return True

def migrate(self, recipe_dir: str, attrs: "AttrsTypedDict", **kwargs: Any) -> None:
Expand Down
10 changes: 8 additions & 2 deletions conda_forge_tick/migrators/matplotlib_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import typing
from typing import Any

from conda_forge_tick.migrators.core import _parse_bad_attr
from conda_forge_tick.migrators.core import _parse_bad_attr, _skip_due_to_schema
from conda_forge_tick.migrators.replacement import Replacement
from conda_forge_tick.utils import frozen_to_json_friendly

Expand Down Expand Up @@ -49,7 +49,13 @@ def parse_already_pred() -> bool:
)
_no_dep = len(rq & self.packages) == 0

return _is_archived or _is_pred or _is_bad or _no_dep
return (
_is_archived
or _is_pred
or _is_bad
or _no_dep
or _skip_due_to_schema(attrs, self.allowed_schema_versions)
)

def migrate(
self, recipe_dir: str, attrs: "AttrsTypedDict", **kwargs: Any
Expand Down
Loading