diff --git a/airbyte-ci/connectors/connector_ops/connector_ops/utils.py b/airbyte-ci/connectors/connector_ops/connector_ops/utils.py index 2af725f8a4f2..666f2fd96e02 100644 --- a/airbyte-ci/connectors/connector_ops/connector_ops/utils.py +++ b/airbyte-ci/connectors/connector_ops/connector_ops/utils.py @@ -127,27 +127,6 @@ def get_changed_file(file_name: str, diff_regex: Optional[str] = None) -> Set[st } return {Connector(get_connector_name_from_path(changed_file)) for changed_file in changed_acceptance_test_config_paths} - -def has_local_cdk_ref(build_file: Path) -> bool: - """Return true if the build file uses the local CDK. - - Args: - build_file (Path): Path to the build.gradle file of the project. - - Returns: - bool: True if using local CDK. - """ - contents = "\n".join( - [ - # Return contents without inline code comments - line.split("//")[0] - for line in build_file.read_text().split("\n") - ] - ) - contents = contents.replace(" ", "") - return "useLocalCdk=true" in contents - - def get_gradle_dependencies_block(build_file: Path) -> str: """Get the dependencies block of a Gradle file. @@ -362,6 +341,32 @@ def code_directory(self) -> Path: def has_dockerfile(self) -> bool: return (self.code_directory / "Dockerfile").is_file() + @property + def changelog_entry_files(self) -> Set[Path]: + changelog_entry_dir = (self.code_directory / ".changelog_entries") + return set(changelog_entry_dir.iterdir()) if changelog_entry_dir.is_dir() else set() + + @property + def uses_local_cdk(self) -> bool: + """Return true if the connector is in JAVA and uses the local CDK. + + Returns: + bool: True if using local CDK. + """ + if self.language != ConnectorLanguage.JAVA: + return False + + build_file = self.code_directory/"build.gradle" + contents = "\n".join( + [ + # Return contents without inline code comments + line.split("//")[0] + for line in build_file.read_text().split("\n") + ] + ) + contents = contents.replace(" ", "") + return "useLocalCdk=true" in contents + @property def metadata_file_path(self) -> Path: return self.code_directory / METADATA_FILE_NAME diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py index 4527a02c7536..1117f778ac44 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py @@ -25,13 +25,15 @@ def log_selected_connectors(selected_connectors_with_modified_files: List[Connec else: main_logger.info("No connectors to run.") - def get_selected_connectors_with_modified_files( selected_names: Tuple[str], selected_support_levels: Tuple[str], selected_languages: Tuple[str], modified: bool, metadata_changes_only: bool, + with_changelog_entry_files: bool, + with_use_local_cdk: bool, + without_use_local_cdk: bool, metadata_query: str, modified_files: Set[Path], enable_dependency_scanning: bool = False, @@ -44,6 +46,9 @@ def get_selected_connectors_with_modified_files( selected_languages (Tuple[str]): Selected connector languages. modified (bool): Whether to select the modified connectors. metadata_changes_only (bool): Whether to select only the connectors with metadata changes. + with_changelog_entry_files (bool): Whether to select the connectors with files in .changelog_entries + with_use_local_cdk (bool): Whether to select the connectors with useLocalCdk set to true + without_use_local_cdk (bool): Whether to unselect the connectors with useLocalCdk set to true modified_files (Set[Path]): The modified files. enable_dependency_scanning (bool): Whether to enable the dependency scanning. Returns: @@ -63,6 +68,9 @@ def get_selected_connectors_with_modified_files( selected_connectors_by_query = ( {connector for connector in ALL_CONNECTORS if connector.metadata_query_match(metadata_query)} if metadata_query else set() ) + selected_connectors_with_changelog_entry_files = {c for c in ALL_CONNECTORS if c.changelog_entry_files} if with_changelog_entry_files else set() + connectors_with_use_local_cdk = {c for c in ALL_CONNECTORS if c.uses_local_cdk} + selected_connectors_with_use_local_cdk = connectors_with_use_local_cdk if with_use_local_cdk else set() non_empty_connector_sets = [ connector_set @@ -72,11 +80,14 @@ def get_selected_connectors_with_modified_files( selected_connectors_by_language, selected_connectors_by_query, selected_modified_connectors, + selected_connectors_with_changelog_entry_files, + selected_connectors_with_use_local_cdk ] if connector_set ] # The selected connectors are the intersection of the selected connectors by name, support_level, language, simpleeval query and modified. selected_connectors = set.intersection(*non_empty_connector_sets) if non_empty_connector_sets else set() + selected_connectors = selected_connectors.discard(connectors_with_use_local_cdk) if without_use_local_cdk else selected_connectors selected_connectors_with_modified_files = [] for connector in selected_connectors: @@ -177,6 +188,9 @@ def should_use_remote_secrets(use_remote_secrets: Optional[bool]) -> bool: help="Filter connectors to test by support_level.", type=click.Choice(SupportLevelEnum), ) +@click.option("--with-changelog-entry-files", help="Select connectors that have changelog_entry files.", is_flag=True, default=False, type=bool) +@click.option("--with-use-local-cdk", help="Select connectors that have useLocalCdk set to true.", is_flag=True, default=False, type=bool) +@click.option("--without-use-local-cdk", help="Unselect connectors that have useLocalCdk set to true.", is_flag=True, default=False, type=bool) @click.option("--modified/--not-modified", help="Only test modified connectors in the current branch.", default=False, type=bool) @click.option( "--metadata-changes-only/--not-metadata-changes-only", @@ -258,6 +272,9 @@ async def connectors( ctx.obj["languages"], ctx.obj["modified"], ctx.obj["metadata_changes_only"], + ctx.obj["with_changelog_entry_files"], + ctx.obj["with_use_local_cdk"], + ctx.obj["without_use_local_cdk"], ctx.obj["metadata_query"], set(modified_files), ctx.obj["enable_dependency_scanning"],