From 1a55fe9d11a86200aaabcdd50abe2e761fcba7db Mon Sep 17 00:00:00 2001 From: phlax Date: Mon, 2 Oct 2023 07:08:39 +0100 Subject: [PATCH] py/tools: Namespace repo packages to prevent conflict (#29787) Signed-off-by: Ryan Northey Signed-off-by: phlax --- BUILD | 7 ++ bazel/repositories.bzl | 6 ++ distribution/dockerhub/BUILD | 4 +- mobile/BUILD | 185 ----------------------------------- tools/BUILD | 3 + tools/base/envoy_python.bzl | 74 +++++++++++++- tools/code/BUILD | 4 +- tools/dependency/BUILD | 7 +- tools/distribution/BUILD | 6 +- tools/docs/BUILD | 25 ++--- tools/proto_format/BUILD | 9 +- tools/protodoc/BUILD | 13 +-- tools/protoprint/BUILD | 7 +- 13 files changed, 125 insertions(+), 225 deletions(-) delete mode 100644 mobile/BUILD diff --git a/BUILD b/BUILD index 34c8e7a4b633..28d92d8c704a 100644 --- a/BUILD +++ b/BUILD @@ -1,5 +1,12 @@ +load("//bazel:envoy_build_system.bzl", "envoy_package") +load("//tools/base:envoy_python.bzl", "envoy_py_namespace") + licenses(["notice"]) # Apache 2 +envoy_package() + +envoy_py_namespace() + exports_files([ "VERSION.txt", "API_VERSION.txt", diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index cce9b9165fa5..71667227f73c 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -108,6 +108,7 @@ envoy_entry_point( name = "get_project_json", pkg = "envoy.base.utils", script = "envoy.project_data", + init_data = [":__init__.py"], ) genrule( @@ -132,6 +133,7 @@ envoy_entry_point( ], pkg = "envoy.base.utils", script = "envoy.project", + init_data = [":__init__.py"], ) envoy_entry_point( @@ -142,6 +144,7 @@ envoy_entry_point( ], pkg = "envoy.base.utils", script = "envoy.project", + init_data = [":__init__.py"], ) envoy_entry_point( @@ -152,6 +155,7 @@ envoy_entry_point( ], pkg = "envoy.base.utils", script = "envoy.project", + init_data = [":__init__.py"], ) envoy_entry_point( @@ -162,6 +166,7 @@ envoy_entry_point( ], pkg = "envoy.base.utils", script = "envoy.project", + init_data = [":__init__.py"], ) envoy_entry_point( @@ -172,6 +177,7 @@ envoy_entry_point( ], pkg = "envoy.base.utils", script = "envoy.project", + init_data = [":__init__.py"], ) ''') diff --git a/distribution/dockerhub/BUILD b/distribution/dockerhub/BUILD index cb48d42a20fd..cd6321175ee6 100644 --- a/distribution/dockerhub/BUILD +++ b/distribution/dockerhub/BUILD @@ -1,10 +1,12 @@ load("//bazel:envoy_build_system.bzl", "envoy_package") -load("//tools/base:envoy_python.bzl", "envoy_gencontent") +load("//tools/base:envoy_python.bzl", "envoy_gencontent", "envoy_py_namespace") licenses(["notice"]) # Apache 2 envoy_package() +envoy_py_namespace() + envoy_gencontent( name = "readme", srcs = ["@envoy_repo//:project"], diff --git a/mobile/BUILD b/mobile/BUILD deleted file mode 100644 index 3972e1af9e43..000000000000 --- a/mobile/BUILD +++ /dev/null @@ -1,185 +0,0 @@ -load("@build_bazel_rules_android//android:rules.bzl", "aar_import") -load("@build_bazel_rules_apple//apple:apple.bzl", "apple_static_framework_import") -load("@io_bazel_rules_kotlin//kotlin/internal:toolchains.bzl", "define_kt_toolchain") -load( - "@com_github_buildbuddy_io_rules_xcodeproj//xcodeproj:defs.bzl", - "project_options", - "top_level_targets", - "xcode_schemes", - "xcodeproj", -) -load("//bazel:framework_imports_extractor.bzl", "framework_imports_extractor") - -licenses(["notice"]) # Apache 2 - -alias( - name = "ios_xcframework", - actual = "//library/swift:Envoy", - visibility = ["//visibility:public"], -) - -alias( - name = "ios_dist", - actual = "//library/swift:ios_framework", -) - -framework_imports_extractor( - name = "framework_imports", - framework = "//library/swift:ios_framework", -) - -apple_static_framework_import( - name = "envoy_mobile_ios", - framework_imports = [":framework_imports"], - sdk_dylibs = [ - "resolv.9", - "c++", - ], - sdk_frameworks = [ - "Network", - "SystemConfiguration", - "UIKit", - ], - visibility = ["//visibility:public"], -) - -alias( - name = "android_aar", - actual = "//library/kotlin/io/envoyproxy/envoymobile:envoy_aar", - visibility = ["//visibility:public"], -) - -aar_import( - name = "envoy_mobile_android", - aar = "//library/kotlin/io/envoyproxy/envoymobile:envoy_aar", - visibility = ["//visibility:public"], -) - -alias( - name = "android_dist", - actual = "//library/kotlin/io/envoyproxy/envoymobile:envoy_aar_with_artifacts", -) - -define_kt_toolchain( - name = "kotlin_toolchain", - jvm_target = "1.8", -) - -filegroup( - name = "kotlin_lint_config", - srcs = [".kotlinlint.yml"], - visibility = ["//visibility:public"], -) - -filegroup( - name = "editor_config", - srcs = [".editorconfig"], - visibility = ["//visibility:public"], -) - -genrule( - name = "kotlin_format", - srcs = ["//:editor_config"], - outs = ["kotlin_format.txt"], - cmd = """ - $(location @kotlin_formatter//file) --android "**/*.kt" \ - --reporter=plain --reporter=checkstyle,output=$@ \ - --editorconfig=$(location //:editor_config) - """, - tools = ["@kotlin_formatter//file"], -) - -genrule( - name = "kotlin_format_fix", - srcs = ["//:editor_config"], - outs = ["kotlin_format_fix.txt"], - cmd = """ - $(location @kotlin_formatter//file) -F --android "**/*.kt" \ - --reporter=plain --reporter=checkstyle,output=$@ \ - --editorconfig=$(location //:editor_config) - """, - tools = ["@kotlin_formatter//file"], -) - -xcodeproj( - name = "xcodeproj", - bazel_path = "./bazelw", - build_mode = "bazel", - project_name = "Envoy", - project_options = project_options( - indent_width = 2, - tab_width = 2, - ), - scheme_autogeneration_mode = "auto", # Switch to "all" to generate schemes for all deps - schemes = [ - xcode_schemes.scheme( - name = "Async Await App", - launch_action = xcode_schemes.launch_action("//examples/swift/async_await:app"), - ), - xcode_schemes.scheme( - name = "Hello World App", - launch_action = xcode_schemes.launch_action("//examples/swift/hello_world:app"), - ), - xcode_schemes.scheme( - name = "Hello World App (ObjC)", - launch_action = xcode_schemes.launch_action("//examples/objective-c/hello_world:app"), - ), - xcode_schemes.scheme( - name = "Baseline App", - launch_action = xcode_schemes.launch_action("//test/swift/apps/baseline:app"), - ), - xcode_schemes.scheme( - name = "Experimental App", - launch_action = xcode_schemes.launch_action("//test/swift/apps/experimental:app"), - ), - xcode_schemes.scheme( - name = "Swift Library", - build_action = xcode_schemes.build_action(["//library/swift:ios_lib"]), - ), - xcode_schemes.scheme( - name = "iOS Tests", - test_action = xcode_schemes.test_action([ - "//experimental/swift:quic_stream_test", - "//test/objective-c:envoy_bridge_utility_test", - "//test/swift/integration:test", - "//test/swift/stats:test", - "//test/swift:test", - ]), - ), - xcode_schemes.scheme( - name = "Swift C++ Interop Tests", - test_action = xcode_schemes.test_action([ - "//test/swift/cxx:test", - ]), - ), - xcode_schemes.scheme( - name = "Objective-C Library", - build_action = xcode_schemes.build_action(["//library/objective-c:envoy_engine_objc_lib"]), - test_action = xcode_schemes.test_action(["//test/objective-c:envoy_bridge_utility_test"]), - ), - ], - tags = ["manual"], - top_level_targets = [ - # Apps - top_level_targets( - labels = [ - "//examples/objective-c/hello_world:app", - "//examples/swift/async_await:app", - "//examples/swift/hello_world:app", - "//test/swift/apps/baseline:app", - "//test/swift/apps/experimental:app", - ], - target_environments = [ - "device", - "simulator", - ], - ), - # Tests - "//experimental/swift:quic_stream_test", - "//test/objective-c:envoy_bridge_utility_test", - "//test/swift/cxx:test", - "//test/swift/integration:test", - "//test/swift/stats:test", - "//test/swift:test", - ], -) diff --git a/tools/BUILD b/tools/BUILD index 5e578a3c82ca..ec8b6d14f8e2 100644 --- a/tools/BUILD +++ b/tools/BUILD @@ -5,11 +5,14 @@ load( "envoy_package", "envoy_py_test_binary", ) +load("//tools/base:envoy_python.bzl", "envoy_py_namespace") licenses(["notice"]) # Apache 2 envoy_package() +envoy_py_namespace() + exports_files([ "gen_git_sha.sh", "check_repositories.sh", diff --git a/tools/base/envoy_python.bzl b/tools/base/envoy_python.bzl index 1d94771708a6..ec852e511b44 100644 --- a/tools/base/envoy_python.bzl +++ b/tools/base/envoy_python.bzl @@ -4,6 +4,56 @@ load("@base_pip3//:requirements.bzl", "requirement", base_entry_point = "entry_p load("@envoy_toolshed//py:macros.bzl", "entry_point") load("@rules_python//python:defs.bzl", "py_binary", "py_library") +ENVOY_PYTOOL_NAMESPACE = [ + ":py-init", + "//:py-init", + "//tools:py-init", +] + +def envoy_py_namespace(): + """Adding this to a build, injects a namespaced __init__.py, this allows namespaced + packages - eg envoy.base.utils to co-exist with packages created from the repo.""" + native.genrule( + name = "py-init-file", + outs = ["__init__.py"], + cmd = """ + echo "__path__ = __import__('pkgutil').extend_path(__path__, __name__)" > $@ + """, + ) + py_library( + name = "py-init", + srcs = [":py-init-file"], + visibility = ["//visibility:public"], + ) + +def envoy_pytool_binary( + name, + data = None, + init_data = ENVOY_PYTOOL_NAMESPACE, + **kwargs): + """Wraps py_binary with envoy namespaced __init__.py files. + + If used outside of tools/${toolname}/BUILD you must specify the init_data.""" + py_binary( + name = name, + data = init_data + (data or []), + **kwargs + ) + +def envoy_pytool_library( + name, + data = None, + init_data = ENVOY_PYTOOL_NAMESPACE, + **kwargs): + """Wraps py_library with envoy namespaced __init__.py files. + + If used outside of tools/${toolname}/BUILD you must specify the init_data.""" + py_library( + name = name, + data = init_data + (data or []), + **kwargs + ) + def envoy_entry_point( name, pkg, @@ -11,6 +61,7 @@ def envoy_entry_point( entry_point_alias = base_entry_point, script = None, data = None, + init_data = ENVOY_PYTOOL_NAMESPACE, deps = None, args = None, visibility = ["//visibility:public"]): @@ -20,7 +71,7 @@ def envoy_entry_point( script = script, entry_point_script = entry_point_script, entry_point_alias = entry_point_alias, - data = data, + data = (data or []) + init_data, deps = deps, args = args, visibility = visibility, @@ -31,6 +82,8 @@ def envoy_jinja_env( templates, filters = {}, env_kwargs = {}, + init_data = ENVOY_PYTOOL_NAMESPACE, + data = [], deps = [], entry_point_alias = base_entry_point): """This provides a prebuilt jinja environment that can be imported as a module. @@ -152,9 +205,10 @@ def envoy_jinja_env( exec_tools = [name_templates], ) - py_library( + envoy_pytool_library( name = name, srcs = [name_env_py], + init_data = init_data, data = [name_templates], deps = [name_entry_point], ) @@ -212,7 +266,12 @@ def envoy_genjson(name, srcs = [], yaml_srcs = [], filter = None, args = None): filter = filter, ) -def envoy_py_data(name, src, format = None, entry_point_alias = base_entry_point): +def envoy_py_data( + name, + src, + init_data = ENVOY_PYTOOL_NAMESPACE, + format = None, + entry_point_alias = base_entry_point): """Preload JSON/YAML data as a python lib. Data is loaded to python and then dumped to a pickle file. @@ -293,9 +352,10 @@ def envoy_py_data(name, src, format = None, entry_point_alias = base_entry_point tools = [name_pickle], ) - py_library( + envoy_pytool_library( name = name, srcs = [name_env_py], + init_data = init_data, data = [name_pickle], deps = [name_entry_point, requirement("envoy.base.utils")], ) @@ -306,6 +366,7 @@ def envoy_gencontent( output, srcs = [], yaml_srcs = [], + init_data = ENVOY_PYTOOL_NAMESPACE, json_kwargs = {}, template_name = None, template_filters = {}, @@ -353,10 +414,12 @@ def envoy_gencontent( envoy_py_data( name = "%s_data" % name, src = ":%s_json" % name, + init_data = init_data, entry_point_alias = entry_point_alias, ) envoy_jinja_env( name = name_tpl, + init_data = init_data, env_kwargs = template_kwargs, templates = [template], filters = template_filters, @@ -377,10 +440,11 @@ def envoy_gencontent( outs = ["%s_generate_content.py" % name], tools = [":%s" % name_data, name_tpl, template], ) - py_binary( + envoy_pytool_binary( name = "%s_generate_content" % name, main = ":%s_generate_content.py" % name, srcs = [":%s_generate_content.py" % name], + init_data = init_data, deps = [ ":%s" % name_data, name_tpl, diff --git a/tools/code/BUILD b/tools/code/BUILD index 77466a7b4905..4de0a77a4b2e 100644 --- a/tools/code/BUILD +++ b/tools/code/BUILD @@ -6,12 +6,14 @@ load( "READFILTER_FUZZ_FILTERS", "READFILTER_NOFUZZ_FILTERS", ) -load("//tools/base:envoy_python.bzl", "envoy_entry_point") +load("//tools/base:envoy_python.bzl", "envoy_entry_point", "envoy_py_namespace") licenses(["notice"]) # Apache 2 envoy_package() +envoy_py_namespace() + FUZZ_FILTER_COUNT = ( len(READFILTER_FUZZ_FILTERS) + len(READFILTER_NOFUZZ_FILTERS) diff --git a/tools/dependency/BUILD b/tools/dependency/BUILD index 3db2d1dba578..855ed61dc386 100644 --- a/tools/dependency/BUILD +++ b/tools/dependency/BUILD @@ -1,13 +1,14 @@ load("@base_pip3//:requirements.bzl", "requirement") load("@envoy_repo//:path.bzl", "PATH") -load("@rules_python//python:defs.bzl", "py_binary") load("//bazel:envoy_build_system.bzl", "envoy_package") -load("//tools/base:envoy_python.bzl", "envoy_entry_point") +load("//tools/base:envoy_python.bzl", "envoy_entry_point", "envoy_py_namespace", "envoy_pytool_binary") licenses(["notice"]) # Apache 2 envoy_package() +envoy_py_namespace() + envoy_entry_point( name = "check", args = [ @@ -30,7 +31,7 @@ envoy_entry_point( pkg = "dependatool", ) -py_binary( +envoy_pytool_binary( name = "validate", srcs = ["validate.py"], args = [ diff --git a/tools/distribution/BUILD b/tools/distribution/BUILD index 44e4006cdd98..f809f3637d45 100644 --- a/tools/distribution/BUILD +++ b/tools/distribution/BUILD @@ -1,11 +1,13 @@ load("@base_pip3//:requirements.bzl", "requirement") load("//bazel:envoy_build_system.bzl", "envoy_package") -load("//tools/base:envoy_python.bzl", "envoy_entry_point") +load("//tools/base:envoy_python.bzl", "envoy_entry_point", "envoy_py_namespace", "envoy_pytool_binary") licenses(["notice"]) # Apache 2 envoy_package() +envoy_py_namespace() + envoy_entry_point( name = "release", pkg = "envoy.distribution.release", @@ -21,7 +23,7 @@ envoy_entry_point( pkg = "envoy.distribution.verify", ) -py_binary( +envoy_pytool_binary( name = "update_dockerhub_repository", srcs = ["update_dockerhub_repository.py"], data = ["//distribution/dockerhub:readme.md"], diff --git a/tools/docs/BUILD b/tools/docs/BUILD index 672ba2a42b93..7d2870c162ee 100644 --- a/tools/docs/BUILD +++ b/tools/docs/BUILD @@ -1,13 +1,14 @@ load("@base_pip3//:requirements.bzl", "requirement") -load("@rules_python//python:defs.bzl", "py_binary") load("//bazel:envoy_build_system.bzl", "envoy_package") -load("//tools/base:envoy_python.bzl", "envoy_entry_point") +load("//tools/base:envoy_python.bzl", "envoy_entry_point", "envoy_py_namespace", "envoy_pytool_binary") licenses(["notice"]) # Apache 2 envoy_package() -py_binary( +envoy_py_namespace() + +envoy_pytool_binary( name = "generate_extensions_security_rst", srcs = ["generate_extensions_security_rst.py"], deps = [ @@ -15,20 +16,16 @@ py_binary( ], ) -py_binary( +envoy_pytool_binary( name = "generate_external_deps_rst", - srcs = [ - "generate_external_deps_rst.py", - ], + srcs = ["generate_external_deps_rst.py"], args = ["$(location //bazel:all_repository_locations)"], data = ["//bazel:all_repository_locations"], ) -py_binary( +envoy_pytool_binary( name = "generate_api_rst", - srcs = [ - "generate_api_rst.py", - ], + srcs = ["generate_api_rst.py"], ) # The upstream lib is maintained here: @@ -45,11 +42,9 @@ envoy_entry_point( visibility = ["//visibility:public"], ) -py_binary( +envoy_pytool_binary( name = "generate_version_histories", - srcs = [ - "generate_version_histories.py", - ], + srcs = ["generate_version_histories.py"], deps = [ requirement("aio.run.runner"), requirement("envoy.base.utils"), diff --git a/tools/proto_format/BUILD b/tools/proto_format/BUILD index 89eccc31bb42..4a9bfc24e2f0 100644 --- a/tools/proto_format/BUILD +++ b/tools/proto_format/BUILD @@ -2,14 +2,15 @@ load("@aspect_bazel_lib//lib:jq.bzl", "jq") load("@envoy_repo//:path.bzl", "PATH") load("@rules_pkg//pkg:mappings.bzl", "pkg_files", "strip_prefix") load("@rules_pkg//pkg:pkg.bzl", "pkg_tar") -load("@rules_python//python:defs.bzl", "py_binary") load("//bazel:envoy_build_system.bzl", "envoy_package") -load("//tools/base:envoy_python.bzl", "envoy_genjson", "envoy_py_data") +load("//tools/base:envoy_python.bzl", "envoy_genjson", "envoy_py_data", "envoy_py_namespace", "envoy_pytool_binary") licenses(["notice"]) # Apache 2 envoy_package() +envoy_py_namespace() + # Files to include when building or comparing the normalized API API_FILES = [ "BUILD", @@ -130,7 +131,7 @@ genrule( tools = [":formatted_api"], ) -py_binary( +envoy_pytool_binary( name = "fetch_normalized_changes", srcs = ["fetch_normalized_changes.py"], args = [ @@ -165,7 +166,7 @@ genrule( ], ) -py_binary( +envoy_pytool_binary( name = "proto_sync", srcs = ["proto_sync.py"], args = [ diff --git a/tools/protodoc/BUILD b/tools/protodoc/BUILD index 01ce510978c0..f562c431ed3e 100644 --- a/tools/protodoc/BUILD +++ b/tools/protodoc/BUILD @@ -1,15 +1,16 @@ load("@base_pip3//:requirements.bzl", "requirement") load("@com_google_protobuf//:protobuf.bzl", "py_proto_library") -load("@rules_python//python:defs.bzl", "py_binary", "py_library") load("//bazel:envoy_build_system.bzl", "envoy_package") -load("//tools/base:envoy_python.bzl", "envoy_genjson", "envoy_jinja_env", "envoy_py_data") +load("//tools/base:envoy_python.bzl", "envoy_genjson", "envoy_jinja_env", "envoy_py_data", "envoy_py_namespace", "envoy_pytool_binary", "envoy_pytool_library") load("//tools/protodoc:protodoc.bzl", "protodoc_rule") licenses(["notice"]) # Apache 2 envoy_package() -py_binary( +envoy_py_namespace() + +envoy_pytool_binary( name = "generate_empty", srcs = ["generate_empty.py"], visibility = ["//visibility:public"], @@ -30,7 +31,7 @@ envoy_py_data( src = "//docs:protodoc_manifest.yaml", ) -py_binary( +envoy_pytool_binary( name = "manifest_to_json", srcs = ["manifest_to_json.py"], args = ["$(location @envoy_api//:v3_proto_set)"], @@ -101,7 +102,7 @@ envoy_py_data( src = ":data_srcs", ) -py_binary( +envoy_pytool_binary( name = "protodoc", srcs = ["protodoc.py"], visibility = ["//visibility:public"], @@ -124,7 +125,7 @@ protodoc_rule( ], ) -py_library( +envoy_pytool_library( name = "rst_filters", srcs = ["rst_filters.py"], ) diff --git a/tools/protoprint/BUILD b/tools/protoprint/BUILD index 4a8cac10d797..720e41c1a7ec 100644 --- a/tools/protoprint/BUILD +++ b/tools/protoprint/BUILD @@ -1,16 +1,17 @@ load("@base_pip3//:requirements.bzl", "requirement") load("@rules_pkg//pkg:mappings.bzl", "pkg_files", "strip_prefix") load("@rules_pkg//pkg:pkg.bzl", "pkg_tar") -load("@rules_python//python:defs.bzl", "py_binary") load("//bazel:envoy_build_system.bzl", "envoy_package") -load("//tools/base:envoy_python.bzl", "envoy_py_data") +load("//tools/base:envoy_python.bzl", "envoy_py_data", "envoy_py_namespace", "envoy_pytool_binary") load("//tools/protoprint:protoprint.bzl", "protoprint_rule") licenses(["notice"]) # Apache 2 envoy_package() -py_binary( +envoy_py_namespace() + +envoy_pytool_binary( name = "protoprint", srcs = ["protoprint.py"], data = [