Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: regenerate the client lib to pick new mtls env #197

Merged
merged 3 commits into from
Sep 18, 2020
Merged

feat: regenerate the client lib to pick new mtls env #197

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1474649
feat: regenerate the client lib to pick new mtls env
arithmetic1728 Sep 16, 2020
0a9304d
update google-api-core version
arithmetic1728 Sep 16, 2020
9b10e81
revert some files in samples/ and scripts/
arithmetic1728 Sep 16, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions .kokoro/populate-secrets.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
#!/bin/bash
# Copyright 2020 Google LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -eo pipefail

function now { date +"%Y-%m-%d %H:%M:%S" | tr -d '\n' ;}
function msg { println "$*" >&2 ;}
function println { printf '%s\n' "$(now) $*" ;}


# Populates requested secrets set in SECRET_MANAGER_KEYS from service account:
# kokoro-trampoline@cloud-devrel-kokoro-resources.iam.gserviceaccount.com
SECRET_LOCATION="${KOKORO_GFILE_DIR}/secret_manager"
msg "Creating folder on disk for secrets: ${SECRET_LOCATION}"
mkdir -p ${SECRET_LOCATION}
for key in $(echo ${SECRET_MANAGER_KEYS} | sed "s/,/ /g")
do
msg "Retrieving secret ${key}"
docker run --entrypoint=gcloud \
--volume=${KOKORO_GFILE_DIR}:${KOKORO_GFILE_DIR} \
gcr.io/google.com/cloudsdktool/cloud-sdk \
secrets versions access latest \
--project cloud-devrel-kokoro-resources \
--secret ${key} > \
"${SECRET_LOCATION}/${key}"
if [[ $? == 0 ]]; then
msg "Secret written to ${SECRET_LOCATION}/${key}"
else
msg "Error retrieving secret ${key}"
fi
done
50 changes: 13 additions & 37 deletions .kokoro/release/common.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,42 +23,18 @@ env_vars: {
value: "github/python-pubsub/.kokoro/release.sh"
}

# Fetch the token needed for reporting release status to GitHub
before_action {
fetch_keystore {
keystore_resource {
keystore_config_id: 73713
keyname: "yoshi-automation-github-key"
}
}
}

# Fetch PyPI password
before_action {
fetch_keystore {
keystore_resource {
keystore_config_id: 73713
keyname: "google_cloud_pypi_password"
}
}
}

# Fetch magictoken to use with Magic Github Proxy
before_action {
fetch_keystore {
keystore_resource {
keystore_config_id: 73713
keyname: "releasetool-magictoken"
}
}
# Fetch PyPI password
before_action {
fetch_keystore {
keystore_resource {
keystore_config_id: 73713
keyname: "google_cloud_pypi_password"
}
}
}

# Fetch api key to use with Magic Github Proxy
before_action {
fetch_keystore {
keystore_resource {
keystore_config_id: 73713
keyname: "magic-github-proxy-api-key"
}
}
}
# Tokens needed to report release status back to GitHub
env_vars: {
key: "SECRET_MANAGER_KEYS"
value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem"
}
15 changes: 10 additions & 5 deletions .kokoro/trampoline.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,14 @@

set -eo pipefail

python3 "${KOKORO_GFILE_DIR}/trampoline_v1.py" || ret_code=$?
# Always run the cleanup script, regardless of the success of bouncing into
# the container.
function cleanup() {
chmod +x ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh
${KOKORO_GFILE_DIR}/trampoline_cleanup.sh
echo "cleanup";
}
trap cleanup EXIT

chmod +x ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh
${KOKORO_GFILE_DIR}/trampoline_cleanup.sh || true

exit ${ret_code}
$(dirname $0)/populate-secrets.sh # Secret Manager secrets.
python3 "${KOKORO_GFILE_DIR}/trampoline_v1.py"
2 changes: 1 addition & 1 deletion docs/conf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
needs_sphinx = "1.6.3"
needs_sphinx = "1.5.5"

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
Expand Down
32 changes: 18 additions & 14 deletions google/pubsub_v1/services/publisher/async_client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ class PublisherAsyncClient:
DEFAULT_MTLS_ENDPOINT = PublisherClient.DEFAULT_MTLS_ENDPOINT

topic_path = staticmethod(PublisherClient.topic_path)
parse_topic_path = staticmethod(PublisherClient.parse_topic_path)

from_service_account_file = PublisherClient.from_service_account_file
from_service_account_json = from_service_account_file
Expand Down Expand Up @@ -79,16 +80,19 @@ def __init__(
client_options (ClientOptions): Custom options for the client. It
won't take effect if a ``transport`` instance is provided.
(1) The ``api_endpoint`` property can be used to override the
default endpoint provided by the client. GOOGLE_API_USE_MTLS
default endpoint provided by the client. GOOGLE_API_USE_MTLS_ENDPOINT
environment variable can also be used to override the endpoint:
"always" (always use the default mTLS endpoint), "never" (always
use the default regular endpoint, this is the default value for
the environment variable) and "auto" (auto switch to the default
mTLS endpoint if client SSL credentials is present). However,
the ``api_endpoint`` property takes precedence if provided.
(2) The ``client_cert_source`` property is used to provide client
SSL credentials for mutual TLS transport. If not provided, the
default SSL credentials will be used if present.
use the default regular endpoint) and "auto" (auto switch to the
default mTLS endpoint if client certificate is present, this is
the default value). However, the ``api_endpoint`` property takes
precedence if provided.
(2) If GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable
is "true", then the ``client_cert_source`` property can be used
to provide client certificate for mutual TLS transport. If
not provided, the default SSL client certificate will be used if
present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not
set, no client certificate will be used.

Raises:
google.auth.exceptions.MutualTlsChannelError: If mutual TLS transport
Expand Down Expand Up @@ -308,13 +312,13 @@ async def publish(
maximum=60.0,
multiplier=1.3,
predicate=retries.if_exception_type(
exceptions.Aborted,
exceptions.Cancelled,
exceptions.DeadlineExceeded,
exceptions.InternalServerError,
exceptions.ResourceExhausted,
exceptions.ServiceUnavailable,
exceptions.Unknown,
exceptions.InternalServerError,
exceptions.Aborted,
),
),
default_timeout=60.0,
Expand Down Expand Up @@ -391,8 +395,8 @@ async def get_topic(
multiplier=1.3,
predicate=retries.if_exception_type(
exceptions.Aborted,
exceptions.ServiceUnavailable,
exceptions.Unknown,
exceptions.ServiceUnavailable,
),
),
default_timeout=60.0,
Expand Down Expand Up @@ -473,8 +477,8 @@ async def list_topics(
multiplier=1.3,
predicate=retries.if_exception_type(
exceptions.Aborted,
exceptions.ServiceUnavailable,
exceptions.Unknown,
exceptions.ServiceUnavailable,
),
),
default_timeout=60.0,
Expand Down Expand Up @@ -564,8 +568,8 @@ async def list_topic_subscriptions(
multiplier=1.3,
predicate=retries.if_exception_type(
exceptions.Aborted,
exceptions.ServiceUnavailable,
exceptions.Unknown,
exceptions.ServiceUnavailable,
),
),
default_timeout=60.0,
Expand Down Expand Up @@ -659,8 +663,8 @@ async def list_topic_snapshots(
multiplier=1.3,
predicate=retries.if_exception_type(
exceptions.Aborted,
exceptions.ServiceUnavailable,
exceptions.Unknown,
exceptions.ServiceUnavailable,
),
),
default_timeout=60.0,
Expand Down
70 changes: 46 additions & 24 deletions google/pubsub_v1/services/publisher/client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
#

from collections import OrderedDict
from distutils import util
import os
import re
from typing import Callable, Dict, Sequence, Tuple, Type, Union
Expand All @@ -27,6 +28,7 @@
from google.api_core import retry as retries # type: ignore
from google.auth import credentials # type: ignore
from google.auth.transport import mtls # type: ignore
from google.auth.transport.grpc import SslCredentials # type: ignore
from google.auth.exceptions import MutualTLSChannelError # type: ignore
from google.oauth2 import service_account # type: ignore

Expand Down Expand Up @@ -177,16 +179,19 @@ def __init__(
client_options (ClientOptions): Custom options for the client. It
won't take effect if a ``transport`` instance is provided.
(1) The ``api_endpoint`` property can be used to override the
default endpoint provided by the client. GOOGLE_API_USE_MTLS
default endpoint provided by the client. GOOGLE_API_USE_MTLS_ENDPOINT
environment variable can also be used to override the endpoint:
"always" (always use the default mTLS endpoint), "never" (always
use the default regular endpoint, this is the default value for
the environment variable) and "auto" (auto switch to the default
mTLS endpoint if client SSL credentials is present). However,
the ``api_endpoint`` property takes precedence if provided.
(2) The ``client_cert_source`` property is used to provide client
SSL credentials for mutual TLS transport. If not provided, the
default SSL credentials will be used if present.
use the default regular endpoint) and "auto" (auto switch to the
default mTLS endpoint if client certificate is present, this is
the default value). However, the ``api_endpoint`` property takes
precedence if provided.
(2) If GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable
is "true", then the ``client_cert_source`` property can be used
to provide client certificate for mutual TLS transport. If
not provided, the default SSL client certificate will be used if
present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not
set, no client certificate will be used.
client_info (google.api_core.gapic_v1.client_info.ClientInfo):
The client info used to send a user-agent string along with
API requests. If ``None``, then default info will be used.
Expand All @@ -202,25 +207,43 @@ def __init__(
if client_options is None:
client_options = ClientOptions.ClientOptions()

if client_options.api_endpoint is None:
use_mtls_env = os.getenv("GOOGLE_API_USE_MTLS", "never")
# Create SSL credentials for mutual TLS if needed.
use_client_cert = bool(
util.strtobool(os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false"))
)

ssl_credentials = None
is_mtls = False
if use_client_cert:
if client_options.client_cert_source:
import grpc # type: ignore

cert, key = client_options.client_cert_source()
ssl_credentials = grpc.ssl_channel_credentials(
certificate_chain=cert, private_key=key
)
is_mtls = True
else:
creds = SslCredentials()
is_mtls = creds.is_mtls
ssl_credentials = creds.ssl_credentials if is_mtls else None

# Figure out which api endpoint to use.
if client_options.api_endpoint is not None:
api_endpoint = client_options.api_endpoint
else:
use_mtls_env = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto")
if use_mtls_env == "never":
client_options.api_endpoint = self.DEFAULT_ENDPOINT
api_endpoint = self.DEFAULT_ENDPOINT
elif use_mtls_env == "always":
client_options.api_endpoint = self.DEFAULT_MTLS_ENDPOINT
api_endpoint = self.DEFAULT_MTLS_ENDPOINT
elif use_mtls_env == "auto":
has_client_cert_source = (
client_options.client_cert_source is not None
or mtls.has_default_client_cert_source()
)
client_options.api_endpoint = (
self.DEFAULT_MTLS_ENDPOINT
if has_client_cert_source
else self.DEFAULT_ENDPOINT
api_endpoint = (
self.DEFAULT_MTLS_ENDPOINT if is_mtls else self.DEFAULT_ENDPOINT
)
else:
raise MutualTLSChannelError(
"Unsupported GOOGLE_API_USE_MTLS value. Accepted values: never, auto, always"
"Unsupported GOOGLE_API_USE_MTLS_ENDPOINT value. Accepted values: never, auto, always"
)

# Save or instantiate the transport.
Expand All @@ -244,10 +267,9 @@ def __init__(
self._transport = Transport(
credentials=credentials,
credentials_file=client_options.credentials_file,
host=client_options.api_endpoint,
host=api_endpoint,
scopes=client_options.scopes,
api_mtls_endpoint=client_options.api_endpoint,
client_cert_source=client_options.client_cert_source,
ssl_channel_credentials=ssl_credentials,
quota_project_id=client_options.quota_project_id,
client_info=client_info,
)
Expand Down
12 changes: 6 additions & 6 deletions google/pubsub_v1/services/publisher/transports/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,13 +139,13 @@ def _prep_wrapped_messages(self, client_info):
maximum=60.0,
multiplier=1.3,
predicate=retries.if_exception_type(
exceptions.Aborted,
exceptions.Cancelled,
exceptions.DeadlineExceeded,
exceptions.InternalServerError,
exceptions.ResourceExhausted,
exceptions.ServiceUnavailable,
exceptions.Unknown,
exceptions.InternalServerError,
exceptions.Aborted,
),
),
default_timeout=60.0,
Expand All @@ -159,8 +159,8 @@ def _prep_wrapped_messages(self, client_info):
multiplier=1.3,
predicate=retries.if_exception_type(
exceptions.Aborted,
exceptions.ServiceUnavailable,
exceptions.Unknown,
exceptions.ServiceUnavailable,
),
),
default_timeout=60.0,
Expand All @@ -174,8 +174,8 @@ def _prep_wrapped_messages(self, client_info):
multiplier=1.3,
predicate=retries.if_exception_type(
exceptions.Aborted,
exceptions.ServiceUnavailable,
exceptions.Unknown,
exceptions.ServiceUnavailable,
),
),
default_timeout=60.0,
Expand All @@ -189,8 +189,8 @@ def _prep_wrapped_messages(self, client_info):
multiplier=1.3,
predicate=retries.if_exception_type(
exceptions.Aborted,
exceptions.ServiceUnavailable,
exceptions.Unknown,
exceptions.ServiceUnavailable,
),
),
default_timeout=60.0,
Expand All @@ -204,8 +204,8 @@ def _prep_wrapped_messages(self, client_info):
multiplier=1.3,
predicate=retries.if_exception_type(
exceptions.Aborted,
exceptions.ServiceUnavailable,
exceptions.Unknown,
exceptions.ServiceUnavailable,
),
),
default_timeout=60.0,
Expand Down
Loading