Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Add discogapic support for GAPICBazel generation #459

Merged
merged 5 commits into from
Apr 6, 2020
Merged

feat: Add discogapic support for GAPICBazel generation #459

Changes from 2 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
f4aed26
feat: Add discogapic support for GAPICBazel generation
vam-google Mar 28, 2020
32d7d2b
reformat with black
vam-google Mar 30, 2020
aebdd3e
Merge branch 'master' into master
SurferJeffAtGoogle Mar 31, 2020
bbe520c
Rename source repository variable
vam-google Apr 3, 2020
8587b63
Merge remote-tracking branch 'upstream/master'
vam-google Apr 3, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 54 additions & 33 deletions synthtool/gcp/gapic_bazel.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@

GOOGLEAPIS_URL: str = git.make_repo_clone_url("googleapis/googleapis")
GOOGLEAPIS_PRIVATE_URL: str = git.make_repo_clone_url("googleapis/googleapis-private")
DISCOVERY_ARTIFACT_MANAGER_URL: str = git.make_repo_clone_url(
"googleapis/discovery-artifact-manager"
)
LOCAL_GOOGLEAPIS: Optional[str] = os.environ.get("SYNTHTOOL_GOOGLEAPIS")


Expand All @@ -36,6 +39,7 @@ def __init__(self):
self._ensure_dependencies_installed()
self._googleapis = None
self._googleapis_private = None
self._discovery_artifact_manager = None

def py_library(self, service: str, version: str, **kwargs) -> Path:
return self._generate_code(service, version, "python", **kwargs)
Expand Down Expand Up @@ -65,34 +69,32 @@ def _generate_code(
language: str,
*,
private: bool = False,
discogapic: bool = False,
proto_path: Union[str, Path] = None,
output_dir: Union[str, Path] = None,
bazel_target: str = None,
):
# Determine which googleapis repo to use
if not private:
googleapis = self._clone_googleapis()
source_repo = None
source_repo_name = None
if discogapic:
source_repo = self._clone_discovery_artifact_manager()
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

"source" is still kinda ambiguous. Could rename these variables:
source_repo_with_protos
name_of_source_repo_with_protos

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm happy to rename it, but _proto suffix would not work for the discovery-artifact-manager repo, because it is a repo with discovery docs (jsons). Maybe something like api_definitions_repo?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I like api_definitions_repo.

source_repo_name = "discovery-artifact-manager"
elif private:
source_repo = self._clone_googleapis_private()
source_repo_name = "googleapis_private"
else:
googleapis = self._clone_googleapis_private()
source_repo = self._clone_googleapis()
source_repo_name = "googleapis"

# Sanity check: We should have a googleapis repo; if we do not,
# something went wrong, and we should abort.
if googleapis is None:
if not source_repo:
raise RuntimeError(
f"Unable to generate {service}, the googleapis repository"
f"Unable to generate {service}, the sources repository repository"
"is unavailable."
)

# Determine where the protos we are generating actually live.
# We can sometimes (but not always) determine this from the service
# and version; in other cases, the user must provide it outright.
if proto_path:
proto_path = Path(proto_path)
if proto_path.is_absolute():
proto_path = proto_path.relative_to("/")
else:
proto_path = Path("google/cloud") / service / version

# Determine bazel target based on per-language patterns
# Java: google-cloud-{{assembly_name}}-{{version}}-java
# Go: gapi-cloud-{{assembly_name}}-{{version}}-go
Expand All @@ -102,6 +104,16 @@ def _generate_code(
# Ruby: google-cloud-{{assembly_name}}-{{version}}-ruby
# C#: google-cloud-{{assembly_name}}-{{version}}-csharp
if bazel_target is None:
# Determine where the protos we are generating actually live.
# We can sometimes (but not always) determine this from the service
# and version; in other cases, the user must provide it outright.
if proto_path:
proto_path = Path(proto_path)
if proto_path.is_absolute():
proto_path = proto_path.relative_to("/")
else:
proto_path = Path("google/cloud") / service / version

parts = list(proto_path.parts)
while len(parts) > 0 and parts[0] != "google":
parts.pop(0)
Expand All @@ -120,19 +132,19 @@ def _generate_code(
suffix = f"{'-'.join(parts)}-{language}"
bazel_target = f"//{os.path.sep.join(parts)}:{suffix}"

# Sanity check: Do we have protos where we think we should?
if not (googleapis / proto_path).exists():
raise FileNotFoundError(
f"Unable to find directory for protos: {(googleapis / proto_path)}."
)
if not tuple((googleapis / proto_path).glob("*.proto")):
raise FileNotFoundError(
f"Directory {(googleapis / proto_path)} exists, but no protos found."
)
if not (googleapis / proto_path / "BUILD.bazel"):
raise FileNotFoundError(
f"File {(googleapis / proto_path / 'BUILD.bazel')} does not exist."
)
# Sanity check: Do we have protos where we think we should?
if not (source_repo / proto_path).exists():
raise FileNotFoundError(
f"Unable to find directory for protos: {(source_repo / proto_path)}."
)
if not tuple((source_repo / proto_path).glob("*.proto")):
raise FileNotFoundError(
f"Directory {(source_repo / proto_path)} exists, but no protos found."
)
if not (source_repo / proto_path / "BUILD.bazel"):
raise FileNotFoundError(
f"File {(source_repo / proto_path / 'BUILD.bazel')} does not exist."
)

# Ensure the desired output directory exists.
# If none was provided, create a temporary directory.
Expand All @@ -142,11 +154,11 @@ def _generate_code(

# Let's build some stuff now.
cwd = os.getcwd()
os.chdir(str(googleapis))
os.chdir(str(source_repo))

bazel_run_args = ["bazel", "build", bazel_target]

log.debug(f"Generating code for: {proto_path}.")
log.debug(f"Generating code for: {bazel_target}.")
shell.run(bazel_run_args)

# We've got tar file!
Expand Down Expand Up @@ -180,7 +192,7 @@ def _generate_code(

# Record this in the synthtool metadata.
metadata.add_client_destination(
source="googleapis" if not private else "googleapis-private",
source=source_repo_name,
api_name=service,
api_version=version,
language=language,
Expand All @@ -191,7 +203,7 @@ def _generate_code(
return output_dir

def _clone_googleapis(self):
if self._googleapis is not None:
if self._googleapis:
return self._googleapis

if LOCAL_GOOGLEAPIS:
Expand All @@ -205,7 +217,7 @@ def _clone_googleapis(self):
return self._googleapis

def _clone_googleapis_private(self):
if self._googleapis_private is not None:
if self._googleapis_private:
return self._googleapis_private

if LOCAL_GOOGLEAPIS:
Expand All @@ -220,6 +232,15 @@ def _clone_googleapis_private(self):

return self._googleapis_private

def _clone_discovery_artifact_manager(self):
if self._discovery_artifact_manager:
return self._discovery_artifact_manager

log.debug("Cloning discovery-artifact-manager.")
self._discovery_artifact_manager = git.clone(DISCOVERY_ARTIFACT_MANAGER_URL)

return self._discovery_artifact_manager

def _ensure_dependencies_installed(self):
log.debug("Ensuring dependencies.")

Expand Down