Rip out READMEs after all

They're too much work to keep up with and keep secure, especially when
we want to add 30+ more package managers.

gratipay/sync_npm/cli.py

@@ -7,20 +7,18 @@
 import argparse
 
 from gratipay import wireup
-from gratipay.sync_npm import serialize, upsert, fetch_readmes, process_readmes
+from gratipay.sync_npm import serialize, upsert
 
 
 def parse_args(argv):
     p = argparse.ArgumentParser()
-    p.add_argument('command', choices=['serialize', 'upsert', 'fetch-readmes', 'process-readmes'])
+    p.add_argument('command', choices=['serialize', 'upsert'])
     p.add_argument('path', help='the path to the input file', nargs='?', default='/dev/stdin')
     return p.parse_args(argv)
 
 
 subcommands = { 'serialize': serialize.main
               , 'upsert': upsert.main
-              , 'fetch-readmes': fetch_readmes.main
-              , 'process-readmes': process_readmes.main
                }
 
 
@@ -30,10 +28,9 @@ def main(argv=sys.argv):
 
     Usage::
 
-      sync-npm {serialize,upsert,fetch-readmes,process-readmes} {<filepath>}
+      sync-npm {serialize,upsert} {<filepath>}
 
-    ``<filepath>`` defaults to stdin. It's necessary for ``serialize`` and
-    ``upsert``, and silently ignored for ``{fetch,process}-readmes``.
+    ``<filepath>`` defaults to stdin.
 
     .. note:: Sphinx is expanding ``sys.argv`` in the parameter list. Sorry. :-/
 

gratipay/testing/__init__.py

@@ -4,12 +4,8 @@
 
 from decimal import Decimal
 
-import pytest
-from aspen import log_dammit
-
 from ..models.participant import Participant
 from ..models.team import Team
-from ..utils import markdown
 
 D = Decimal                     #:
 P = Participant.from_username   #:
@@ -50,17 +46,3 @@ def debug_http():
     requests_log = logging.getLogger("requests.packages.urllib3")
     requests_log.setLevel(logging.DEBUG)
     requests_log.propagate = True
-
-
-# Provide a decorator to skip tests when marky-markdown is missing.
-
-try:
-    markdown.render_like_npm('test')
-except OSError as exc:
-    MISSING_MARKY_MARKDOWN = True
-    log_dammit('Will skip marky-markdown-related tests because:', exc.args[0])
-else:
-    MISSING_MARKY_MARKDOWN = False
-
-def skipif_missing_marky_markdown(func):
-    return pytest.mark.skipif(MISSING_MARKY_MARKDOWN, reason="missing marky-markdown")(func)

gratipay/utils/markdown.py

@@ -1,8 +1,6 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-from subprocess import Popen, PIPE
 
-import json
 import misaka as m  # http://misaka.61924.nl/
 from markupsafe import Markup
 
@@ -19,28 +17,3 @@ def render(markdown):
         extensions=m.EXT_AUTOLINK | m.EXT_STRIKETHROUGH | m.EXT_NO_INTRA_EMPHASIS,
         render_flags=m.HTML_SKIP_HTML | m.HTML_TOC | m.HTML_SMARTYPANTS | m.HTML_SAFELINK
     ))
-
-
-def render_like_npm(markdown, package=None):
-    """Process markdown the same way npm does.
-
-    Package should be a dict representing the package. If it includes `name`
-    and `description` then the first h1 and paragraph will have a
-    'package-{name,description}-redundant' class added to them if they're
-    similar enough. If it includes `repository.url` then links will be changed
-    somehow. For details consult the docs and code:
-
-    https://github.com/npm/marky-markdown
-
-    """
-    if type(markdown) is unicode:
-        markdown = markdown.encode('utf8')
-    cmd = ("bin/our-marky-markdown.js", "/dev/stdin")
-    if package:
-        cmd += (json.dumps(package),)
-    marky = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
-    out, err = marky.communicate(markdown)
-    if marky.wait() > 0:
-        raise OSError(err)
-    return out
-

tests/py/test_markdown.py

@@ -1,10 +1,8 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-from gratipay.testing import Harness, skipif_missing_marky_markdown
+from gratipay.testing import Harness
 from gratipay.utils import markdown
 
-from HTMLParser import HTMLParser
-
 
 class TestMarkdown(Harness):
 
@@ -55,23 +53,3 @@ def test_render_no_intra_emphasis(self):
         expected = '<p>Examples like this_one and this other_one.</p>\n'
         actual = markdown.render('Examples like this_one and this other_one.')
         assert expected == actual
-
-
-    # rln - render_like_npm
-
-    @skipif_missing_marky_markdown
-    def test_rln_works(self):
-        md = "**Hello World!**"
-        actual = HTMLParser().unescape(markdown.render_like_npm(md)).strip()
-        expected = '<p><strong>Hello World!</strong></p>'
-        assert actual == expected
-
-    @skipif_missing_marky_markdown
-    def test_rln_handles_npm_package(self):
-        md = "# Greetings, program!\nGreetings. Program."
-        pkg = {'name': 'greetings-program', 'description': 'Greetings, program.'}
-        actual = HTMLParser().unescape(markdown.render_like_npm(md, pkg)).strip()
-        expected = '''\
-<h1 class="package-name-redundant package-description-redundant"><a id="user-content-greetings-program" class="deep-link" href="#greetings-program"><svg aria-hidden="true" class="deep-link-icon" height="16" version="1.1" width="16"><path d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Greetings, program!</h1>
-<p class="package-description-redundant">Greetings. Program.</p>'''
-        assert actual == expected

tests/py/test_sync_npm.py

@@ -5,8 +5,7 @@
 
 import pytest
 
-from gratipay.testing import Harness, skipif_missing_marky_markdown
-from gratipay.sync_npm import fetch_readmes, process_readmes
+from gratipay.testing import Harness
 
 
 def load(raw):
@@ -109,89 +108,3 @@ def test_sentrifier_starts_at_zero(self):
 
     def test_sentrifier_fail_fails(self):
         pytest.raises(RuntimeError, Sentrifier().fail)
-
-
-    # fr - fetch_readmes
-
-    def make_package_without_readme_raw(self):
-        self.db.run("INSERT INTO packages (package_manager, name, description, emails) "
-                    "VALUES ('npm', 'foo-package', 'A package', ARRAY[]::text[])")
-
-    def test_fr_fetches_a_readme(self):
-        self.make_package_without_readme_raw()
-
-        def fetch(name):
-            return 200, {'name': 'foo-package', 'readme': '# Greetings, program!'}
-
-        fetch_readmes.main({}, [], self.db, lambda a: a, fetch)
-
-        package = self.db.one('SELECT * FROM packages')
-        assert package.name == 'foo-package'
-        assert package.description == 'A package'
-        assert package.readme == ''
-        assert package.readme_needs_to_be_processed
-        assert package.readme_raw == '# Greetings, program!'
-        assert package.readme_type == 'x-markdown/marky'
-        assert package.emails == []
-
-    def test_fr_adds_empty_readme_as_needed(self):
-        self.make_package_without_readme_raw()
-        def fetch(name):
-            return 200, {'name': 'foo-package', 'redmeat': '# Greetings, program!'}
-        fetch_readmes.main({}, [], self.db, lambda a: a, fetch)
-        package = self.db.one('SELECT * FROM packages')
-        assert package.readme_raw == ''
-
-    def test_fr_replaces_non_unicode_with_empty_readme(self):
-        self.make_package_without_readme_raw()
-        def fetch(name):
-            return 200, {'name': 'foo-package', 'readme': {'private': True}}
-        fetch_readmes.main({}, [], self.db, lambda a: a, fetch)
-        package = self.db.one('SELECT * FROM packages')
-        assert package.readme_raw == ''
-
-    def test_fr_deletes_a_readme(self):
-        self.make_package_without_readme_raw()
-        fetch_readmes.main({}, [], self.db, lambda a: a, lambda n: (404, {}))
-        assert self.db.one('SELECT * FROM packages') is None
-
-    def test_fr_tells_sentry_about_problems(self):
-        self.make_package_without_readme_raw()
-        sentrified = Sentrifier()
-        fetch_readmes.main({}, [], self.db, sentrified, sentrified.fail)
-        assert sentrified.ncalls == 1
-
-
-    # pr - process_readmes
-
-    def make_package_with_readme_raw(self):
-        self.db.run('''
-
-            INSERT
-              INTO packages (package_manager, name, description, readme_raw, readme_type, emails)
-            VALUES ('npm', 'foo-package', 'A package', '# Greetings, program!', 'x-markdown/marky',
-                                                                                   ARRAY[]::text[])
-
-        ''')
-
-    @skipif_missing_marky_markdown
-    def test_pr_processes_a_readme(self):
-        self.make_package_with_readme_raw()
-
-        process_readmes.main({}, [], self.db, lambda a: a)
-
-        package = self.db.one('SELECT * FROM packages')
-        assert package.name == 'foo-package'
-        assert package.description == 'A package'
-        assert package.readme == '<h1><a id="user-content-greetings-program" class="deep-link" href="#greetings-program"><svg aria-hidden="true" class="deep-link-icon" height="16" version="1.1" width="16"><path d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Greetings, program!</h1>\n'
-        assert not package.readme_needs_to_be_processed
-        assert package.readme_raw == '# Greetings, program!'
-        assert package.readme_type == 'x-markdown/marky'
-        assert package.emails == []
-
-
-    def test_pr_tells_sentry_about_problems(self):
-        self.make_package_with_readme_raw()
-        sentrified = Sentrifier()
-        process_readmes.main({}, [], self.db, sentrified, sentrified.fail)
-        assert sentrified.ncalls == 1