Skip to content

Commit

Permalink
Merge branch 'develop' into bugfix/2889-cleanup-dangling-gitpython
Browse files Browse the repository at this point in the history
  • Loading branch information
Panaetius authored May 25, 2022
2 parents 10e4021 + 5cc006c commit 153d4ac
Show file tree
Hide file tree
Showing 14 changed files with 344 additions and 155 deletions.
38 changes: 34 additions & 4 deletions DEVELOPING.rst
Original file line number Diff line number Diff line change
Expand Up @@ -231,8 +231,38 @@ Branching Model
We follow the git-flow_ model of branches for development, with ``master`` being
the release branch and ``develop`` being the development branch.

Release branches should be created off of master, have develop merged into them
and then should be merged (not squash merged) back into master. A Github Action
will then take care of merging master back into develop.

.. _git-flow: https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html

Making a Release
----------------
This section uses `v1.2.3` as an example, replace it with the actual version
number.

- Create a new release branch off of **master**. **Do not** call this branch
e.g. `v1.2.3`, it cannot have the same name as the release version.
- Merge changes from `develop` into the release branch (plain merge, **don't
squash**).
- Run `conventional-changelog -r 1 -p angular | pandoc --from markdown --to rst`
to get the changelog and update `CHANGES:rst` with it. Make sure to fill in
the version number in the header and to replace the `=` underline with `-`.
- Proof-read the changelog and adjust messages so they make sense to third
parties reading the changelog
- Update the version in `helm-chart/renku-core/Chart.yaml` and `helm-chart/renku-core/values.yaml`
(for `versions.latest.image.tag`).
- Commit the changes to the release branch, with a message like "chore: release
v1.2.3"
- Push the release branch and create a PR **against master**. Wait for it to be
approved. **Do not squash merge this PR!** Use a regular merge.
- Create a new `github release <https://github.com/SwissDataScienceCenter/renku-python/releases/new>`_.
Set to create a new tag like `v1.2.3` against master, title should be
`v1.2.3` as well. The description should be the output of
`conventional-changelog -r 1 -p angular` with the same adjustments done above
for the `CHANGES.rst` file.
- Once the release PR has been merged, publish the github release. This creates
the tag on master that kicks off the publishing CI.
- Keep an eye on CI, make sure that the `publish-pypi`, `build-images`,
`publish-chart` and `update-develop-branch` finish successfully.
* If any of them don't finish successfully, ask for help.
- Go to the `Renku` repository and approve/merge the automatically created PR
there.
- Announce that we have a new version through appropriate channels.
107 changes: 71 additions & 36 deletions renku/command/login.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,8 @@
"""Logging in to a Renku deployment."""

import os
import sys
import time
import urllib
import uuid
import webbrowser
from typing import TYPE_CHECKING

Expand All @@ -37,6 +36,8 @@


CONFIG_SECTION = "http"
KEYCLOAK_REALM = "Renku"
CLIENT_ID = "renku-cli"


def login_command():
Expand Down Expand Up @@ -67,48 +68,82 @@ def _login(endpoint, git_login, yes, client_dispatcher: IClientDispatcher):
else:
raise errors.ParameterError("Cannot find a unique remote URL for project.")

cli_nonce = str(uuid.uuid4())
auth_server_url = _get_url(
parsed_endpoint, path=f"auth/realms/{KEYCLOAK_REALM}/protocol/openid-connect/auth/device"
)

communication.echo(f"Please log in at {parsed_endpoint.geturl()} on your browser.")
try:
response = requests.post(auth_server_url, data={"client_id": CLIENT_ID})
except errors.RequestError as e:
raise errors.RequestError(f"Cannot connect to authorization server at {auth_server_url}.") from e

login_url = _get_url(parsed_endpoint, "/api/auth/login", cli_nonce=cli_nonce)
webbrowser.open_new_tab(login_url)
requests.check_response(response=response)
data = response.json()

server_nonce = communication.prompt("Once completed, enter the security code that you receive at the end")
cli_token_url = _get_url(parsed_endpoint, "/api/auth/cli-token", cli_nonce=cli_nonce, server_nonce=server_nonce)
verification_uri = data.get("verification_uri")
user_code = data.get("user_code")
verification_uri_complete = f"{verification_uri}?user_code={user_code}"

try:
response = requests.get(cli_token_url)
except errors.RequestError as e:
raise errors.OperationError("Cannot get access token from remote host.") from e
communication.echo(
f"Please grant access to '{CLIENT_ID}' in your browser.\n"
f"If a browser window does not open automatically, go to {verification_uri_complete}"
)

if response.status_code == 200:
access_token = response.json().get("access_token")
_store_token(parsed_endpoint.netloc, access_token)
webbrowser.open_new_tab(verification_uri_complete)

if git_login:
_set_git_credential_helper(repository=client.repository, hostname=parsed_endpoint.netloc)
backup_remote_name, backup_exists, remote = create_backup_remote(
repository=client.repository, remote_name=remote_name, url=remote_url # type:ignore
)
if backup_exists:
communication.echo(f"Backup remote '{backup_remote_name}' already exists. Ignoring '--git' flag.")
elif not remote:
communication.error(f"Cannot create backup remote '{backup_remote_name}' for '{remote_url}'")
polling_interval = min(data.get("interval", 5), 5)
token_url = _get_url(parsed_endpoint, path=f"auth/realms/{KEYCLOAK_REALM}/protocol/openid-connect/token")
device_code = data.get("device_code")

while True:
time.sleep(polling_interval)

response = requests.post(
token_url,
data={
"device_code": device_code,
"client_id": CLIENT_ID,
"grant_type": "urn:ietf:params:oauth:grant-type:device_code",
},
)
status_code = response.status_code
if status_code == 200:
break
elif status_code == 400:
error = response.json().get("error")

if error == "authorization_pending":
continue
elif error == "slow_down":
polling_interval += 1
elif error == "access_denied":
raise errors.AuthenticationError("Access denied")
elif error == "expired_token":
raise errors.AuthenticationError("Session expired, try again")
else:
_set_renku_url_for_remote(
repository=client.repository,
remote_name=remote_name, # type:ignore
remote_url=remote_url, # type:ignore
hostname=parsed_endpoint.netloc,
)
raise errors.AuthenticationError(f"Invalid error message from server: {response.json()}")
else:
raise errors.AuthenticationError(f"Invalid status code from server: {status_code} - {response.content}")

else:
communication.error(
f"Remote host did not return an access token: {parsed_endpoint.geturl()}, "
f"status code: {response.status_code}"
access_token = response.json().get("access_token")
_store_token(parsed_endpoint.netloc, access_token)

if git_login:
_set_git_credential_helper(repository=client.repository, hostname=parsed_endpoint.netloc)
backup_remote_name, backup_exists, remote = create_backup_remote(
repository=client.repository, remote_name=remote_name, url=remote_url # type:ignore
)
sys.exit(1)
if backup_exists:
communication.echo(f"Backup remote '{backup_remote_name}' already exists. Ignoring '--git' flag.")
elif not remote:
communication.error(f"Cannot create backup remote '{backup_remote_name}' for '{remote_url}'")
else:
_set_renku_url_for_remote(
repository=client.repository,
remote_name=remote_name, # type:ignore
remote_url=remote_url, # type:ignore
hostname=parsed_endpoint.netloc,
)


def _parse_endpoint(endpoint):
Expand All @@ -119,7 +154,7 @@ def _parse_endpoint(endpoint):
return parsed_endpoint


def _get_url(parsed_endpoint, path, **query_args):
def _get_url(parsed_endpoint, path, **query_args) -> str:
query = urllib.parse.urlencode(query_args)
return parsed_endpoint._replace(path=path, query=query).geturl()

Expand Down
5 changes: 4 additions & 1 deletion renku/command/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,10 @@ def parse_explicit_definition(entries, type):
)
except FileNotFoundError:
command = " ".join(factory.base_command)
raise errors.ParameterError(f"Cannot execute command '{command}'")
raise errors.ParameterError(
f"Cannot execute command '{command}': "
"This is likely because the executable doesn't exist or has the wrong permissions set."
)

ended_at_time = local_now()

Expand Down
19 changes: 16 additions & 3 deletions renku/core/dataset/dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -733,17 +733,30 @@ def update_datasets(
return imported_datasets_view_models, dataset_files_view_models


def show_dataset(name):
def show_dataset(name: str, tag: Optional[str] = None):
"""Show detailed dataset information.
Args:
name: Name of dataset to show details for.
name(str): Name of dataset to show details for.
tag(str, optional): Tags for which to get the metadata (Default value = None).
Returns:
dict: JSON dictionary of dataset details.
"""
dataset = DatasetsProvenance().get_by_name(name, strict=True)
datasets_provenance = DatasetsProvenance()
dataset = datasets_provenance.get_by_name(name, strict=True)

if tag is None:
return DatasetDetailsJson().dump(dataset)

tags = datasets_provenance.get_all_tags(dataset=cast(Dataset, dataset))

selected_tag = next((t for t in tags if t.name == tag), None)

if selected_tag is None:
raise errors.DatasetTagNotFound(tag)

dataset = datasets_provenance.get_by_id(selected_tag.dataset_id.value)
return DatasetDetailsJson().dump(dataset)


Expand Down
8 changes: 5 additions & 3 deletions renku/core/dataset/providers/dataverse.py
Original file line number Diff line number Diff line change
Expand Up @@ -546,9 +546,11 @@ def _post(self, url, json=None, data=None, files=None):

@staticmethod
def _check_response(response):
if response.status_code not in [200, 201, 202]:
if response.status_code == 401:
raise errors.AuthenticationError("Access unauthorized - update access token.")
from renku.core.util import requests

try:
requests.check_response(response=response)
except errors.RequestError:
json_res = response.json()
raise errors.ExportError(
"HTTP {} - Cannot export dataset: {}".format(
Expand Down
11 changes: 7 additions & 4 deletions renku/core/dataset/providers/olos.py
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,8 @@ def upload_file(self, full_path, path_in_dataset):

return response

def _make_url(self, server_url, api_path, **query_params):
@staticmethod
def _make_url(server_url, api_path, **query_params):
"""Create URL for creating a dataset."""
url_parts = urlparse.urlparse(server_url)

Expand Down Expand Up @@ -259,15 +260,17 @@ def _post(self, url, json=None, data=None, files=None):

@staticmethod
def _check_response(response):
from renku.core.util import requests

if len(response.history) > 0:
raise errors.ExportError(
f"Couldn't execute request to {response.request.url}, got redirected to {response.url}."
"Maybe you mixed up http and https in the server url?"
)

if response.status_code not in [200, 201, 202]:
if response.status_code == 401:
raise errors.AuthenticationError("Access unauthorized - update access token.")
try:
requests.check_response(response=response)
except errors.RequestError:
json_res = response.json()
raise errors.ExportError(
"HTTP {} - Cannot export dataset: {}".format(
Expand Down
29 changes: 25 additions & 4 deletions renku/core/dataset/providers/zenodo.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,14 @@
import attr
from tqdm import tqdm

from renku.core import errors
from renku.core.dataset.providers.api import ExporterApi, ProviderApi, ProviderRecordSerializerApi
from renku.core.util.file_size import bytes_to_unit

if TYPE_CHECKING:
from renku.core.dataset.providers.models import ProviderDataset


ZENODO_BASE_URL = "https://zenodo.org"
ZENODO_SANDBOX_URL = "https://sandbox.zenodo.org/"

Expand Down Expand Up @@ -358,7 +360,7 @@ def new_deposition(self):
response = requests.post(
url=self.new_deposit_url, params=self.exporter.default_params, json={}, headers=self.exporter.HEADERS
)
requests.check_response(response)
self._check_response(response)

return response

Expand All @@ -371,7 +373,7 @@ def upload_file(self, filepath, path_in_repo):
response = requests.post(
url=self.upload_file_url, params=self.exporter.default_params, data=request_payload, files=file
)
requests.check_response(response)
self._check_response(response)

return response

Expand Down Expand Up @@ -402,7 +404,7 @@ def attach_metadata(self, dataset, tag):
data=json.dumps(request_payload),
headers=self.exporter.HEADERS,
)
requests.check_response(response)
self._check_response(response)

return response

Expand All @@ -411,7 +413,7 @@ def publish_deposition(self, secret):
from renku.core.util import requests

response = requests.post(url=self.publish_url, params=self.exporter.default_params)
requests.check_response(response)
self._check_response(response)

return response

Expand All @@ -420,6 +422,25 @@ def __attrs_post_init__(self):
response = self.new_deposition()
self.id = response.json()["id"]

@staticmethod
def _check_response(response):
from renku.core.util import requests

try:
requests.check_response(response=response)
except errors.RequestError:
if response.status_code == 400:
err_response = response.json()
messages = [
'"{0}" failed with "{1}"'.format(err["field"], err["message"]) for err in err_response["errors"]
]

raise errors.ExportError(
"\n" + "\n".join(messages) + "\nSee `renku dataset edit -h` for details on how to edit" " metadata"
)
else:
raise errors.ExportError(response.content)


@attr.s
class ZenodoExporter(ExporterApi):
Expand Down
Loading

0 comments on commit 153d4ac

Please sign in to comment.