Adding syncing github releases to pygame website releases.

pygameweb/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '0.0.5.dev'
+__version__ = '0.0.6'
 
 
 # So we can use environment variables to configure things.

pygameweb/alembic/versions/e22b4355e6fd_youtube_github_and_patreon_fields_added_.py

@@ -0,0 +1,34 @@
+"""Youtube, github, and patreon fields added to project.
+
+Revision ID: e22b4355e6fd
+Revises: 7bb2943bbcaf
+Create Date: 2019-01-27 09:18:12.312081
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'e22b4355e6fd'
+down_revision = '7bb2943bbcaf'
+branch_labels = None
+depends_on = None
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('project', sa.Column('github_repo', sa.Text(), nullable=True))
+    op.add_column('project', sa.Column('patreon', sa.Text(), nullable=True))
+    op.add_column('project', sa.Column('youtube_trailer', sa.Text(), nullable=True))
+    op.add_column('release', sa.Column('from_external', sa.String(length=255), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('release', 'from_external')
+    op.drop_column('project', 'youtube_trailer')
+    op.drop_column('project', 'patreon')
+    op.drop_column('project', 'github_repo')
+    # ### end Alembic commands ###

pygameweb/config.py

@@ -76,3 +76,8 @@ class Config(object):
     COMMENT_MODEL = os.getenv(CONFIG_PREFIX + 'COMMENT_MODEL', 'comment_spam_model.pkl')
     """For the comment spam classifier model file.
     """
+
+    GITHUB_RELEASES_OAUTH = os.getenv(CONFIG_PREFIX + 'GITHUB_RELEASES_OAUTH', None)
+    """ For syncing github releases to pygame_org.
+    """
+

pygameweb/project/forms.py

@@ -3,7 +3,7 @@
 
 from wtforms.fields import StringField, HiddenField
 from wtforms.fields.html5 import URLField
-from wtforms.validators import DataRequired, Required
+from wtforms.validators import DataRequired, Required, URL, Optional
 from wtforms.widgets import TextArea
 
 
@@ -12,20 +12,23 @@ class ProjectForm(FlaskForm):
     tags = StringField('Tags')
     summary = StringField('Summary', widget=TextArea(), validators=[Required()])
     description = StringField('Description', widget=TextArea())
-    uri = URLField('Home URL', validators=[Required()])
+    uri = URLField('Home URL', validators=[Required(), URL()])
 
     image = FileField('image', validators=[
         # FileRequired(),
         FileAllowed(['jpg', 'png'], 'Images only!')
     ])
+    github_repo = URLField('Github repository URL', validators=[Optional(), URL()])
+    youtube_trailer = URLField('Youtube trailer URL', validators=[Optional(), URL()])
+    patreon = URLField('Patreon URL', validators=[Optional(), URL()])
 
 
 class ReleaseForm(FlaskForm):
     version = StringField('version', validators=[Required()])
     description = StringField('description', widget=TextArea())
-    srcuri = URLField('Source URL')
-    winuri = URLField('Windows URL')
-    macuri = URLField('Mac URL')
+    srcuri = URLField('Source URL', validators=[Optional(), URL()])
+    winuri = URLField('Windows URL', validators=[Optional(), URL()])
+    macuri = URLField('Mac URL', validators=[Optional(), URL()])
 
 
 class FirstReleaseForm(ProjectForm, ReleaseForm):

pygameweb/project/gh_releases.py

@@ -0,0 +1,209 @@
+""" For syncing github releases to pygame releases.
+"""
+import urllib.parse
+import feedparser
+import requests
+import dateutil.parser
+
+from pygameweb.project.models import Project, Release
+from pygameweb.config import Config
+
+def sync_github_releases():
+    """ to the pygame website releases.
+    """
+    from pygameweb.config import Config
+    from sqlalchemy import create_engine
+    from sqlalchemy.orm import sessionmaker
+    engine = create_engine(Config.SQLALCHEMY_DATABASE_URI)
+
+    a_connection = engine.connect()
+    a_transaction = a_connection.begin()
+    session = sessionmaker(bind=a_connection)()
+
+    projects = (session
+        .query(Project)
+        .filter(Project.github_repo.isnot(None))
+    )
+    for project in projects:
+        sync_project(session, project)
+
+    session.commit()
+    a_transaction.commit()
+
+
+def sync_project(session, project):
+    if not project.github_repo:
+        return
+    if project.user is not None and project.user.disabled:
+        return
+    gh_releases = get_gh_releases_feed(project)
+    releases = project.releases
+
+    gh_add, gh_update, pg_delete = releases_to_sync(gh_releases, releases)
+
+    # only do the API call once if we need to add/update.
+    releases_gh_api = (
+        get_gh_releases_api(project)
+        if gh_add or gh_update else None
+    )
+
+    releases_added = []
+    for gh_release in gh_add:
+        gh_release_api = [
+            r for r in releases_gh_api
+            if r['name'] == gh_release['title']
+        ]
+        if not gh_release_api or gh_release_api[0]['draft']:
+            continue
+
+        release = release_from_gh(session, project, gh_release, gh_release_api[0])
+        releases_added.append(release)
+
+    for release in releases_added:
+        session.add(release)
+
+    for gh_release in gh_update:
+        releases = [
+            r for r in project.releases
+            if r.version == gh_release['title']
+        ]
+        if releases:
+            release = releases[0]
+
+            release.version = gh_release['title']
+            release.description = gh_release['body']
+            session.add(release)
+
+    for pg_release in pg_delete:
+        pg_release.delete()
+        session.add(pg_release)
+
+
+def release_from_gh(session, project, gh_release_atom, gh_release_api):
+    """ make a Release from a gh release.
+
+    :param gh_release_atom: from the atom feed.
+    :param gh_release_api: from the API.
+    """
+    winuri = ''
+    srcuri = ''
+    macuri = ''
+    for asset in gh_release_api['assets']:
+        if asset["browser_download_url"].endswith('msi'):
+            winuri = asset["browser_download_url"]
+        elif asset["browser_download_url"].endswith('tar.gz'):
+            srcuri = asset["browser_download_url"]
+        elif asset["browser_download_url"].endswith('dmg'):
+            macuri = asset["browser_download_url"]
+
+    published_at = dateutil.parser.parse(gh_release_api["published_at"])
+    # "2019-01-06T15:29:18Z",
+
+    release = Release(
+        datetimeon=published_at,
+        description=gh_release_atom['content'][0]["value"],
+        srcuri=srcuri,
+        winuri=winuri,
+        macuri=macuri,
+        version=gh_release_atom['title'],
+        project=project
+    )
+    return release
+
+
+def releases_to_sync(gh_releases, releases):
+    """
+    :param gh_releases: github release objects from atom.
+    :param releases: the db releases.
+    """
+    add, update, delete = versions_to_sync(gh_releases, releases)
+
+    gh_add = [r for r in gh_releases if r.title in add]
+    gh_update = [r for r in gh_releases if r.title in update]
+    pg_delete = [r for r in releases if r.version in delete]
+    return gh_add, gh_update, pg_delete
+
+def versions_to_sync(gh_releases, releases):
+    """
+    :param gh_releases: github release objects from atom.
+    :param releases: the db releases.
+    """
+    # Because many projects might have existing ones on pygame,
+    #  but not have them on github, we don't delete ones unless
+    #  they came originally from github.
+    return what_versions_sync(
+        {r.version for r in releases},
+        {r.title for r in gh_releases},
+        {r.version for r in releases if r.from_external == 'github'}
+    )
+
+def what_versions_sync(pg_versions, gh_versions, pg_versions_gh):
+    """ versions to add, update, delete.
+    """
+    to_add = gh_versions - pg_versions
+    to_update = pg_versions_gh & gh_versions
+    to_delete = pg_versions_gh - gh_versions
+    return to_add, to_update, to_delete
+
+def get_gh_releases_feed(project):
+    """ for a project.
+    """
+    repo = project.github_repo
+    if not repo.endswith('/'):
+        repo += '/'
+    feed_url = urllib.parse.urljoin(
+        repo,
+        "releases.atom"
+    )
+    data = feedparser.parse(feed_url)
+    if not data['feed']['title'].startswith('Release notes from'):
+        raise ValueError('does not appear to be a github release feed.')
+    return data.entries
+
+
+def get_repo_from_url(url):
+    """ get the github repo from the url
+    """
+    if not url.startswith('https://github.com/'):
+        return
+    repo = (
+        urllib.parse.urlparse(url).path
+        .lstrip('/')
+        .rstrip('/')
+    )
+    if len(repo.split('/')) != 2:
+        return
+    return repo
+
+
+def get_gh_releases_api(project, version=None):
+    """
+    """
+    # https://developer.github.com/v3/auth/
+    # self.headers = {'Authorization': 'token %s' % self.api_token}
+    # https://api.github.com/repos/pygame/stuntcat/releases/latest
+    repo = get_repo_from_url(project.github_repo)
+    if not repo:
+        return
+
+    url = f'https://api.github.com/repos/{repo}/releases'
+    if version is not None:
+        url += f'/{version}'
+
+    if Config.GITHUB_RELEASES_OAUTH is None:
+        headers = {}
+    else:
+        headers = {'Authorization': 'token %s' % Config.GITHUB_RELEASES_OAUTH}
+    resp = requests.get(
+        url,
+        headers = headers
+    )
+    if resp.status_code != 200:
+        raise ValueError('github api failed')
+
+    data = resp.json()
+    return data
+
+
+
+

pygameweb/project/models.py

@@ -3,10 +3,11 @@
 from math import sqrt
 from pathlib import Path
 from email.utils import formatdate
+from urllib.parse import urlparse, parse_qs, urlencode
 
 from sqlalchemy import (Column, DateTime, ForeignKey, Integer,
-                        String, Text, inspect, func, and_)
-from sqlalchemy.orm import relationship
+                        String, Text, inspect, func, and_, or_, CheckConstraint)
+from sqlalchemy.orm import relationship, validates
 from sqlalchemy.sql.functions import count
 
 from pyquery import PyQuery as pq
@@ -34,6 +35,44 @@ class Project(Base):
     datetimeon = Column(DateTime)
     image = Column(String(80))
 
+    github_repo = Column(Text)
+    """ URL to the github repo for this project.
+    """
+    _github_repo_constraint = CheckConstraint(
+        or_(
+            github_repo is None,
+            github_repo == '',
+            github_repo.startswith('https://github.com/')
+        ),
+        name="project_github_repo_constraint"
+    )
+
+
+    youtube_trailer = Column(Text)
+    """ URL to the youtube trailer for this project.
+    """
+    _youtube_trailer_constraint = CheckConstraint(
+        or_(
+            youtube_trailer is None,
+            youtube_trailer == '',
+            youtube_trailer.startswith('https://www.youtube.com/watch?v=')
+        ),
+        name="project_youtube_trailer_constraint"
+    )
+
+    patreon = Column(Text)
+    """ URL to the patreon.
+    """
+    _patreon_constraint = CheckConstraint(
+        or_(
+            patreon is None,
+            patreon == '',
+            patreon.startswith('https://www.patreon.com/')
+        ),
+        name="project_patreon_constraint"
+    )
+
+
     def __repr__(self):
         return "<Project with title=%r>" % self.title
 
@@ -75,6 +114,22 @@ def tag_counts(self):
         return [(tag, cnt, (int(10 + min(24, sqrt(cnt) * 24 / 5))))
                 for tag, cnt in tag_counts]
 
+    @property
+    def youtube_trailer_embed(self):
+        if not self.youtube_trailer:
+            return
+        video_key = parse_qs(urlparse(self.youtube_trailer).query).get('v')[0]
+        bad_chars = ['?', ';', '&', '..', '/']
+        if any(bad in video_key for bad in bad_chars):
+            raise ValueError('problem')
+        return f'http://www.youtube.com/embed/{video_key}'
+
+    __table_args__ = (
+        _github_repo_constraint,
+        _youtube_trailer_constraint,
+        _patreon_constraint,
+    )
+
 
 def top_tags(session, limit=30):
     """
@@ -156,6 +211,13 @@ class Release(Base):
     macuri = Column(String(255))
     version = Column(String(80))
 
+    from_external = Column(String(255))
+    """ is this release sucked in from an external source.
+
+        If it is 'github' then it comes from a github release.
+        If it is None, then it is user entered.
+    """
+
     project = relationship(Project, backref='releases')
 
     @property