Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add hypermedia API to replace XML-RPC and simple #4078

Closed
wants to merge 1 commit into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions dev/environment
Original file line number Diff line number Diff line change
Expand Up @@ -33,3 +33,5 @@ TOKEN_EMAIL_SECRET="an insecure email verification secret key"
DATADOG_HOST=notdatadog

WAREHOUSE_LEGACY_DOMAIN=pypi.python.org

HYPERMEDIA_API=api.pypi.org
3 changes: 3 additions & 0 deletions requirements/main.in
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
alembic>=0.7.0
apispec
Automat
argon2-cffi
Babel
Expand All @@ -21,6 +22,7 @@ itsdangerous
Jinja2>=2.8
limits
lxml
marshmallow
mistune
msgpack
packaging>=15.2
Expand All @@ -36,6 +38,7 @@ pyramid_retry>=0.3
pyramid_rpc>=0.7
pyramid_services
pyramid_tm>=0.12
PyYaml
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If this is a dependency of apispec, it shouldn’t be in the in file.

raven
readme_renderer>=0.7.0
requests
Expand Down
6 changes: 6 additions & 0 deletions requirements/main.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ alembic==0.9.9 \
amqp==2.2.2 \
--hash=sha256:4e28d3ea61a64ae61830000c909662cb053642efddbe96503db0e7783a6ee85b \
--hash=sha256:cba1ace9d4ff6049b190d8b7991f9c1006b443a5238021aca96dd6ad2ac9da22
apispec==0.37.0 \
--hash=sha256:8c497b70f0095da521b41ea1e85cd171302b80e4bb6382cb0055cadf4980ced1
argon2-cffi==18.1.0 \
--hash=sha256:93f631fa567dbf948f26874476c9e9afb51e0a835372bf1a319df0c5aa071bfb \
--hash=sha256:131effd5eabbe08649bc672b5d602fd6e2772b03cfec2ddb2795f9d9babe3fba \
Expand Down Expand Up @@ -269,6 +271,8 @@ Mako==1.0.7 \
--hash=sha256:4e02fde57bd4abb5ec400181e4c314f56ac3e49ba4fb8b0d50bba18cb27d25ae
MarkupSafe==1.0 \
--hash=sha256:a6be69091dac236ea9c6bc7d012beab42010fa914c459791d627dad4910eb665
marshmallow==3.0.0b10 \
--hash=sha256:be2541dfd0fe7fdbb6ab83ab187e5190dfe2e169b68bb6ff982b06fad5bdb7e0
mistune==0.8.3 \
--hash=sha256:b4c512ce2fc99e5a62eb95a4aba4b73e5f90264115c40b70a21e1f7d4e0eac91 \
--hash=sha256:bc10c33bfdcaa4e749b779f62f60d6e12f8215c46a292d05e486b869ae306619
Expand Down Expand Up @@ -420,6 +424,8 @@ python-editor==1.0.3 \
pytz==2018.4 \
--hash=sha256:65ae0c8101309c45772196b21b74c46b2e5d11b6275c45d251b150d5da334555 \
--hash=sha256:c06425302f2cf668f1bba7a0a03f3c1d34d4ebeef2c72003da308b3947c7f749
PyYAML==3.12 \
--hash=sha256:592766c6303207a20efc445587778322d7f73b161bd994f227adaa341ba212ab
raven==6.8.0 \
--hash=sha256:1c641e5ebc2d4185560608e253970ca0d4b98475f4edf67735015a415f9e1d48 \
--hash=sha256:95aecf76c414facaddbb056f3e98c7936318123e467728f2e50b3a66b65a6ef7
Expand Down
2 changes: 1 addition & 1 deletion tests/unit/test_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -327,7 +327,7 @@ def __init__(self):
pretend.call(HostRewrite),
]
assert configurator_obj.include.calls == (
[pretend.call(".datadog"), pretend.call(".csrf")]
[pretend.call(".api"), pretend.call(".datadog"), pretend.call(".csrf")]
+ [
pretend.call(x)
for x in [
Expand Down
15 changes: 15 additions & 0 deletions warehouse/api/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


def includeme(config):
config.include(".routes")
92 changes: 92 additions & 0 deletions warehouse/api/routes.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


def includeme(config):
# Add a subdomain for the hypermedia api.
hypermedia = config.get_settings().get("hypermedia.domain")

config.add_route("api.spec", "/api/", read_only=True, domain=hypermedia)
config.add_route(
"api.views.projects",
"/api/projects/",
factory="warehouse.packaging.models:ProjectFactory",
read_only=True,
domain=hypermedia,
)
config.add_route(
"api.views.projects.detail",
"/api/projects/{name}/",
factory="warehouse.packaging.models:ProjectFactory",
traverse="/{name}",
read_only=True,
domain=hypermedia,
)
config.add_route(
"api.views.projects.detail.files",
"/api/projects/{name}/files/",
factory="warehouse.packaging.models:ProjectFactory",
traverse="/{name}",
read_only=True,
domain=hypermedia,
)
config.add_route(
"api.views.projects.releases",
"/api/projects/{name}/releases/",
factory="warehouse.packaging.models:ProjectFactory",
traverse="/{name}",
read_only=True,
domain=hypermedia,
)
config.add_route(
"api.views.projects.releases.detail",
"/api/projects/{name}/releases/{version}/",
factory="warehouse.packaging.models:ProjectFactory",
traverse="/{name}/{version}",
read_only=True,
domain=hypermedia,
)
config.add_route(
"api.views.projects.releases.files",
"/api/projects/{name}/releases/{version}/files/",
factory="warehouse.packaging.models:ProjectFactory",
traverse="/{name}/{version}",
read_only=True,
domain=hypermedia,
)
config.add_route(
"api.views.projects.detail.roles",
"/api/projects/{name}/roles/",
factory="warehouse.packaging.models:ProjectFactory",
traverse="/{name}",
read_only=True,
domain=hypermedia,
)
config.add_route(
"api.views.journals", "/api/journals/", read_only=True, domain=hypermedia
)
# This is the JSON API equivalent of changelog_last_serial()
config.add_route(
"api.views.journals.latest",
"/api/journals/latest/",
read_only=True,
domain=hypermedia,
)
# This is the JSON API equivalent of user_packages(user)
config.add_route(
"api.views.users.details.projects",
"/api/users/{user}/projects/",
factory="warehouse.accounts.models:UserFactory",
traverse="/{user}",
read_only=True,
domain=hypermedia,
)
195 changes: 195 additions & 0 deletions warehouse/api/schema.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,195 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import datetime

from marshmallow import Schema, fields


class File(Schema):
filename = fields.Str()
packagetype = fields.Str()
python_version = fields.Str()
has_sig = fields.Bool(attribute="has_signature")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why change the name?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

has_sig is what the JSON API uses, so I started there. I prefer to keep the names consistent with the db where possible, I'll change it to has_signature.

comment_text = fields.Str()
md5_digest = fields.Str()
digests = fields.Method("get_digests")
size = fields.Int()
upload_time = fields.Function(
lambda obj: obj.upload_time.strftime("%Y-%m-%dT%H:%M:%S")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No timezone?

Copy link

@sloria sloria Jun 1, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Drive-by comment: You can use fields.DateTime here. If you need a specific format, you can pass format:

import datetime as dt

import pytz
from marshmallow import Schema, fields

class File(Schema):
    upload_time = fields.DateTime(format="%Y-%m-%dT%H:%M:%S")


schema = File()
timezone_aware_dt = dt.datetime.utcnow()
timezone_aware_dt.replace(tzinfo=pytz.UTC)
print(schema.dump({'upload_time': timezone_aware_dt}))
# {'upload_time': '2018-06-01T13:29:25'}

If you don't pass a format, the default behavior is to format as ISO8601.

import datetime as dt

import pytz
from marshmallow import Schema, fields

class File(Schema):
    upload_time = fields.DateTime()


schema = File()
timezone_aware_dt = dt.datetime.utcnow()
timezone_aware_dt.replace(tzinfo=pytz.UTC)
print(schema.dump({'upload_time': timezone_aware_dt}))
# {'upload_time': '2018-06-01T13:31:08.732522+00:00'}

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@merwok, we left out the timezone because that's what the JSON api does. However, the XML-RPC api does include a tz. I'll add tz to this field on the next push.

)
url = fields.Method("get_detail_url")

def get_digests(self, obj):
return {"md5": obj.md5_digest, "sha256": obj.sha256_digest}

def get_detail_url(self, obj):
request = self.context.get("request")
return request.route_url("packaging.file", path=obj.path)


class Release(Schema):
bugtrack_url = fields.Str(attribute="project.bugtrack_url")
classifiers = fields.List(fields.Str())
docs_url = fields.Str(attribute="project.documentation_url")
downloads = fields.Method("get_downloads")
project_url = fields.Method("get_project_url")
url = fields.Method("get_release_url")
requires_dist = fields.List(fields.Str())
files_url = fields.Method("get_files_url")

def get_files_url(self, obj):
request = self.context.get("request")
return request.route_url(
"api.views.projects.releases.files",
name=obj.project.name,
version=obj.version,
)

def get_project_url(self, obj):
request = self.context.get("request")
return request.route_url("api.views.projects.detail", name=obj.project.name)

def get_release_url(self, obj):
request = self.context.get("request")
return request.route_url(
"api.views.projects.releases.detail",
name=obj.project.name,
version=obj.version,
)

def get_downloads(self, obj):
return {"last_day": -1, "last_week": -1, "last_month": -1}

class Meta:
fields = (
"author",
"author_email",
"bugtrack_url",
"classifiers",
"description",
"description_content_type",
"docs_url",
"downloads",
"download_url",
"home_page",
"keywords",
"license",
"maintainer",
"maintainer_email",
"name",
"project_url",
"url",
"platform",
"requires_dist",
"requires_python",
"summary",
"version",
"files_url",
)
ordered = True


class Project(Schema):
url = fields.Method("get_detail_url")
releases_url = fields.Method("get_releases_url")
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Drive-by comment: Since serializing URLs is a common usage, you might want to create a custom field. It could look something like

class Project(Schema):
    releases_url = RouteURL("api.views.projects.releases",
                            name="<normalized_name>")

For an example of how this might be implemented, check out flask-marshmallow's equivalent: https://github.com/marshmallow-code/flask-marshmallow/blob/7e594dc89ba853946ef1593e9ac19850a5ec9d2a/flask_marshmallow/fields.py#L39

latest_version_url = fields.Method("get_latest_version_url")
legacy_project_json = fields.Method("get_legacy_project_json")
roles_url = fields.Method("get_roles_url")
files_url = fields.Method("get_files_url")

def get_files_url(self, obj):
request = self.context.get("request")
return request.route_url("api.views.projects.detail.files", name=obj.name)

def get_roles_url(self, obj):
request = self.context.get("request")
return request.route_url(
"api.views.projects.detail.roles", name=obj.normalized_name
)

def get_legacy_project_json(self, obj):
request = self.context.get("request")
return request.route_url("legacy.api.json.project", name=obj.normalized_name)

def get_detail_url(self, obj):
request = self.context.get("request")
return request.route_url("api.views.projects.detail", name=obj.normalized_name)

def get_latest_version_url(self, obj):
request = self.context.get("request")
if not obj.latest_version:
return None
return request.route_url(
"api.views.projects.releases.detail",
name=obj.name,
version=obj.latest_version[0],
)

def get_releases_url(self, obj):
request = self.context.get("request")
return request.route_url(
"api.views.projects.releases", name=obj.normalized_name
)

class Meta:
fields = (
"name",
"normalized_name",
"latest_version_url",
"bugtrack_url",
"last_serial",
"url",
"releases_url",
"legacy_project_json",
"stable_version",
"created",
"roles_url",
"files_url",
)


class Journal(Schema):
project_name = fields.Str(attribute="name")
timestamp = fields.Method("get_timestamp")
release_url = fields.Method("get_release_url")

def get_release_url(self, obj):
request = self.context.get("request")
if not obj.version:
return None
return request.route_url(
"api.views.projects.releases.detail", name=obj.name, version=obj.version
)

def get_timestamp(self, obj):
return int(obj.submitted_date.replace(tzinfo=datetime.timezone.utc).timestamp())

class Meta:
fields = (
"project_name",
"release_url",
"version",
"timestamp",
"action",
"submitted_date",
)


class Role(Schema):
role = fields.Str(attribute="role_name")
name = fields.Str(attribute="user.username")


class UserProjects(Schema):
role = fields.Str(attribute="role_name")
project = fields.Nested(Project(only=("name", "url")))
Loading