add filters for connectors

better CDK dev workflow

airbyte-ci/connectors/connector_ops/connector_ops/utils.py

@@ -127,27 +127,6 @@ def get_changed_file(file_name: str, diff_regex: Optional[str] = None) -> Set[st
     }
     return {Connector(get_connector_name_from_path(changed_file)) for changed_file in changed_acceptance_test_config_paths}
 
-
-def has_local_cdk_ref(build_file: Path) -> bool:
-    """Return true if the build file uses the local CDK.
-
-    Args:
-        build_file (Path): Path to the build.gradle file of the project.
-
-    Returns:
-        bool: True if using local CDK.
-    """
-    contents = "\n".join(
-        [
-            # Return contents without inline code comments
-            line.split("//")[0]
-            for line in build_file.read_text().split("\n")
-        ]
-    )
-    contents = contents.replace(" ", "")
-    return "useLocalCdk=true" in contents
-
-
 def get_gradle_dependencies_block(build_file: Path) -> str:
     """Get the dependencies block of a Gradle file.
 
@@ -362,6 +341,32 @@ def code_directory(self) -> Path:
     def has_dockerfile(self) -> bool:
         return (self.code_directory / "Dockerfile").is_file()
 
+    @property
+    def changelog_entry_files(self) -> Set[Path]:
+        changelog_entry_dir = (self.code_directory / ".changelog_entries")
+        return set(changelog_entry_dir.iterdir()) if changelog_entry_dir.is_dir() else set()
+
+    @property
+    def uses_local_cdk(self) -> bool:
+        """Return true if the connector is in JAVA and uses the local CDK.
+
+        Returns:
+            bool: True if using local CDK.
+        """
+        if self.language != ConnectorLanguage.JAVA:
+            return False
+
+        build_file = self.code_directory/"build.gradle"
+        contents = "\n".join(
+            [
+                # Return contents without inline code comments
+                line.split("//")[0]
+                for line in build_file.read_text().split("\n")
+            ]
+        )
+        contents = contents.replace(" ", "")
+        return "useLocalCdk=true" in contents
+
     @property
     def metadata_file_path(self) -> Path:
         return self.code_directory / METADATA_FILE_NAME

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py

@@ -25,13 +25,15 @@ def log_selected_connectors(selected_connectors_with_modified_files: List[Connec
     else:
         main_logger.info("No connectors to run.")
 
-
 def get_selected_connectors_with_modified_files(
     selected_names: Tuple[str],
     selected_support_levels: Tuple[str],
     selected_languages: Tuple[str],
     modified: bool,
     metadata_changes_only: bool,
+    with_changelog_entry_files: bool,
+    with_use_local_cdk: bool,
+    without_use_local_cdk: bool,
     metadata_query: str,
     modified_files: Set[Path],
     enable_dependency_scanning: bool = False,
@@ -44,6 +46,9 @@ def get_selected_connectors_with_modified_files(
         selected_languages (Tuple[str]): Selected connector languages.
         modified (bool): Whether to select the modified connectors.
         metadata_changes_only (bool): Whether to select only the connectors with metadata changes.
+        with_changelog_entry_files (bool): Whether to select the connectors with files in .changelog_entries
+        with_use_local_cdk (bool): Whether to select the connectors with useLocalCdk set to true
+        without_use_local_cdk (bool): Whether to unselect the connectors with useLocalCdk set to true
         modified_files (Set[Path]): The modified files.
         enable_dependency_scanning (bool): Whether to enable the dependency scanning.
     Returns:
@@ -63,6 +68,9 @@ def get_selected_connectors_with_modified_files(
     selected_connectors_by_query = (
         {connector for connector in ALL_CONNECTORS if connector.metadata_query_match(metadata_query)} if metadata_query else set()
     )
+    selected_connectors_with_changelog_entry_files = {c for c in ALL_CONNECTORS if c.changelog_entry_files} if with_changelog_entry_files else set()
+    connectors_with_use_local_cdk = {c for c in ALL_CONNECTORS if c.uses_local_cdk}
+    selected_connectors_with_use_local_cdk = connectors_with_use_local_cdk if with_use_local_cdk else set()
 
     non_empty_connector_sets = [
         connector_set
@@ -72,11 +80,14 @@ def get_selected_connectors_with_modified_files(
             selected_connectors_by_language,
             selected_connectors_by_query,
             selected_modified_connectors,
+            selected_connectors_with_changelog_entry_files,
+            selected_connectors_with_use_local_cdk
         ]
         if connector_set
     ]
     # The selected connectors are the intersection of the selected connectors by name, support_level, language, simpleeval query and modified.
     selected_connectors = set.intersection(*non_empty_connector_sets) if non_empty_connector_sets else set()
+    selected_connectors = selected_connectors.discard(connectors_with_use_local_cdk) if without_use_local_cdk else selected_connectors
 
     selected_connectors_with_modified_files = []
     for connector in selected_connectors:
@@ -177,6 +188,9 @@ def should_use_remote_secrets(use_remote_secrets: Optional[bool]) -> bool:
     help="Filter connectors to test by support_level.",
     type=click.Choice(SupportLevelEnum),
 )
+@click.option("--with-changelog-entry-files", help="Select connectors that have changelog_entry files.", is_flag=True, default=False, type=bool)
+@click.option("--with-use-local-cdk", help="Select connectors that have useLocalCdk set to true.", is_flag=True, default=False, type=bool)
+@click.option("--without-use-local-cdk", help="Unselect connectors that have useLocalCdk set to true.", is_flag=True, default=False, type=bool)
 @click.option("--modified/--not-modified", help="Only test modified connectors in the current branch.", default=False, type=bool)
 @click.option(
     "--metadata-changes-only/--not-metadata-changes-only",
@@ -258,6 +272,9 @@ async def connectors(
         ctx.obj["languages"],
         ctx.obj["modified"],
         ctx.obj["metadata_changes_only"],
+        ctx.obj["with_changelog_entry_files"],
+        ctx.obj["with_use_local_cdk"],
+        ctx.obj["without_use_local_cdk"],
         ctx.obj["metadata_query"],
         set(modified_files),
         ctx.obj["enable_dependency_scanning"],