From 9c59e8cfaa75c0aad005872ad4813e7b59d02370 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 10 Jan 2018 18:58:53 -0500 Subject: [PATCH 1/8] Fix regex for getting project and user Check the .git to the end of the string --- readthedocs/builds/utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/readthedocs/builds/utils.py b/readthedocs/builds/utils.py index 1457a123c67..4805b279885 100644 --- a/readthedocs/builds/utils.py +++ b/readthedocs/builds/utils.py @@ -7,23 +7,23 @@ import re GH_REGEXS = [ - re.compile('github.com/(.+)/(.+)(?:\.git){1}'), + re.compile('github.com/(.+)/(.+)(?:\.git){1}$'), re.compile('github.com/(.+)/(.+)'), - re.compile('github.com:(.+)/(.+).git'), + re.compile('github.com:(.+)/(.+).git$'), ] BB_REGEXS = [ re.compile('bitbucket.org/(.+)/(.+)/'), re.compile('bitbucket.org/(.+)/(.+)'), - re.compile('bitbucket.org:(.+)/(.+)\.git'), + re.compile('bitbucket.org:(.+)/(.+)\.git$'), ] # TODO: I think this can be different than `gitlab.com` # self.adapter.provider_base_url GL_REGEXS = [ - re.compile('gitlab.com/(.+)/(.+)(?:\.git){1}'), + re.compile('gitlab.com/(.+)/(.+)(?:\.git){1}$'), re.compile('gitlab.com/(.+)/(.+)'), - re.compile('gitlab.com:(.+)/(.+)\.git'), + re.compile('gitlab.com:(.+)/(.+)\.git$'), ] From 17da91418892fd5147f0dd65a6bdb6b149b4c9d7 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 11 Jan 2018 17:56:25 -0500 Subject: [PATCH 2/8] Add additional tests For the http clone url --- .../rtd_tests/tests/test_repo_parsing.py | 37 ++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/readthedocs/rtd_tests/tests/test_repo_parsing.py b/readthedocs/rtd_tests/tests/test_repo_parsing.py index 33e4ac212d6..c6bb40353bb 100644 --- a/readthedocs/rtd_tests/tests/test_repo_parsing.py +++ b/readthedocs/rtd_tests/tests/test_repo_parsing.py @@ -22,9 +22,21 @@ def test_github(self): self.pip.repo = 'https://github.com/user/repo/' self.assertEqual(self.version.get_github_url(docroot='/docs/', filename='file'), 'https://github.com/user/repo/blob/master/docs/file.rst') + self.pip.repo = 'https://github.com/user/repo.github.io' + self.assertEqual(self.version.get_github_url(docroot='/docs/', filename='file'), 'https://github.com/user/repo.github.io/blob/master/docs/file.rst') + + self.pip.repo = 'https://github.com/user/repo.github.io/' + self.assertEqual(self.version.get_github_url(docroot='/docs/', filename='file'), 'https://github.com/user/repo.github.io/blob/master/docs/file.rst') + self.pip.repo = 'https://github.com/user/repo.git' self.assertEqual(self.version.get_github_url(docroot='/docs/', filename='file'), 'https://github.com/user/repo/blob/master/docs/file.rst') + self.pip.repo = 'https://github.com/user/repo.github.io.git' + self.assertEqual(self.version.get_github_url(docroot='/docs/', filename='file'), 'https://github.com/user/repo.github.io/blob/master/docs/file.rst') + + self.pip.repo = 'https://github.com/user/repo.git.git' + self.assertEqual(self.version.get_github_url(docroot='/docs/', filename='file'), 'https://github.com/user/repo.git/blob/master/docs/file.rst') + def test_gitlab(self): self.pip.repo = 'https://gitlab.com/user/repo' self.assertEqual(self.version.get_gitlab_url(docroot='/foo/bar/', filename='file'), 'https://gitlab.com/user/repo/blob/master/foo/bar/file.rst') @@ -32,9 +44,21 @@ def test_gitlab(self): self.pip.repo = 'https://gitlab.com/user/repo/' self.assertEqual(self.version.get_gitlab_url(docroot='/foo/bar/', filename='file'), 'https://gitlab.com/user/repo/blob/master/foo/bar/file.rst') + self.pip.repo = 'https://gitlab.com/user/repo.gitlab.io' + self.assertEqual(self.version.get_gitlab_url(docroot='/foo/bar/', filename='file'), 'https://gitlab.com/user/repo.gitlab.io/blob/master/foo/bar/file.rst') + + self.pip.repo = 'https://gitlab.com/user/repo.gitlab.io/' + self.assertEqual(self.version.get_gitlab_url(docroot='/foo/bar/', filename='file'), 'https://gitlab.com/user/repo.gitlab.io/blob/master/foo/bar/file.rst') + self.pip.repo = 'https://gitlab.com/user/repo.git' self.assertEqual(self.version.get_gitlab_url(docroot='/foo/bar/', filename='file'), 'https://gitlab.com/user/repo/blob/master/foo/bar/file.rst') + self.pip.repo = 'https://gitlab.com/user/repo.gitlab.io.git' + self.assertEqual(self.version.get_gitlab_url(docroot='/foo/bar/', filename='file'), 'https://gitlab.com/user/repo.gitlab.io/blob/master/foo/bar/file.rst') + + self.pip.repo = 'https://gitlab.com/user/repo.git.git' + self.assertEqual(self.version.get_gitlab_url(docroot='/foo/bar/', filename='file'), 'https://gitlab.com/user/repo.git/blob/master/foo/bar/file.rst') + def test_bitbucket(self): self.pip.repo = 'https://bitbucket.org/user/repo' self.assertEqual(self.version.get_bitbucket_url(docroot='/foo/bar/', filename='file'), 'https://bitbucket.org/user/repo/src/master/foo/bar/file.rst') @@ -42,6 +66,17 @@ def test_bitbucket(self): self.pip.repo = 'https://bitbucket.org/user/repo/' self.assertEqual(self.version.get_bitbucket_url(docroot='/foo/bar/', filename='file'), 'https://bitbucket.org/user/repo/src/master/foo/bar/file.rst') + self.pip.repo = 'https://bitbucket.org/user/repo.gitbucket.io' + self.assertEqual(self.version.get_bitbucket_url(docroot='/foo/bar/', filename='file'), 'https://bitbucket.org/user/repo.gitbucket.io/src/master/foo/bar/file.rst') + + self.pip.repo = 'https://bitbucket.org/user/repo.gitbucket.io/' + self.assertEqual(self.version.get_bitbucket_url(docroot='/foo/bar/', filename='file'), 'https://bitbucket.org/user/repo.gitbucket.io/src/master/foo/bar/file.rst') + self.pip.repo = 'https://bitbucket.org/user/repo.git' - self.assertEqual(self.version.get_bitbucket_url(docroot='/foo/bar/', filename='file'), 'https://bitbucket.org/user/repo/src/master/foo/bar/file.rst') + self.assertEqual(self.version.get_bitbucket_url(docroot='/foo/bar/', filename='file'), 'https://bitbucket.org/user/repo.git/src/master/foo/bar/file.rst') + + self.pip.repo = 'https://bitbucket.org/user/repo.gitbucket.io.git' + self.assertEqual(self.version.get_bitbucket_url(docroot='/foo/bar/', filename='file'), 'https://bitbucket.org/user/repo.gitbucket.io.git/src/master/foo/bar/file.rst') + self.pip.repo = 'https://bitbucket.org/user/repo.git.git' + self.assertEqual(self.version.get_bitbucket_url(docroot='/foo/bar/', filename='file'), 'https://bitbucket.org/user/repo.git.git/src/master/foo/bar/file.rst') From a736049ad8e4cc44d18342f07c2eac0f803cd33a Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 11 Jan 2018 18:00:33 -0500 Subject: [PATCH 3/8] Move valid regex to constants --- readthedocs/projects/constants.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/readthedocs/projects/constants.py b/readthedocs/projects/constants.py index 8971262399d..202c23682aa 100644 --- a/readthedocs/projects/constants.py +++ b/readthedocs/projects/constants.py @@ -292,19 +292,19 @@ PROJECT_SLUG_REGEX = '(?:[-\w]+)' GITHUB_REGEXS = [ - re.compile('github.com/(.+)/(.+)(?:\.git){1}'), + re.compile('github.com/(.+)/(.+)(?:\.git){1}$'), re.compile('github.com/(.+)/(.+)'), - re.compile('github.com:(.+)/(.+).git'), + re.compile('github.com:(.+)/(.+).git$'), ] BITBUCKET_REGEXS = [ - re.compile('bitbucket.org/(.+)/(.+).git'), re.compile('bitbucket.org/(.+)/(.+)/'), re.compile('bitbucket.org/(.+)/(.+)'), + re.compile('bitbucket.org:(.+)/(.+)\.git$'), ] GITLAB_REGEXS = [ - re.compile('gitlab.com/(.+)/(.+)(?:\.git){1}'), + re.compile('gitlab.com/(.+)/(.+)(?:\.git){1}$'), re.compile('gitlab.com/(.+)/(.+)'), - re.compile('gitlab.com:(.+)/(.+).git'), + re.compile('gitlab.com:(.+)/(.+)\.git$'), ] GITHUB_URL = ( 'https://github.com/{user}/{repo}/' From bc7e0c8549ce10b136051aa2d2ef4613df39c366 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 11 Jan 2018 18:01:07 -0500 Subject: [PATCH 4/8] Use regex from constants --- readthedocs/builds/utils.py | 29 +++++------------------------ 1 file changed, 5 insertions(+), 24 deletions(-) diff --git a/readthedocs/builds/utils.py b/readthedocs/builds/utils.py index 4805b279885..7fcc245dbb8 100644 --- a/readthedocs/builds/utils.py +++ b/readthedocs/builds/utils.py @@ -4,32 +4,13 @@ from __future__ import ( absolute_import, division, print_function, unicode_literals) -import re - -GH_REGEXS = [ - re.compile('github.com/(.+)/(.+)(?:\.git){1}$'), - re.compile('github.com/(.+)/(.+)'), - re.compile('github.com:(.+)/(.+).git$'), -] - -BB_REGEXS = [ - re.compile('bitbucket.org/(.+)/(.+)/'), - re.compile('bitbucket.org/(.+)/(.+)'), - re.compile('bitbucket.org:(.+)/(.+)\.git$'), -] - -# TODO: I think this can be different than `gitlab.com` -# self.adapter.provider_base_url -GL_REGEXS = [ - re.compile('gitlab.com/(.+)/(.+)(?:\.git){1}$'), - re.compile('gitlab.com/(.+)/(.+)'), - re.compile('gitlab.com:(.+)/(.+)\.git$'), -] +from readthedocs.projects.constants import ( + BITBUCKET_REGEXS, GITHUB_REGEXS, GITLAB_REGEXS) def get_github_username_repo(url): if 'github' in url: - for regex in GH_REGEXS: + for regex in GITHUB_REGEXS: match = regex.search(url) if match: return match.groups() @@ -38,7 +19,7 @@ def get_github_username_repo(url): def get_bitbucket_username_repo(url=None): if 'bitbucket' in url: - for regex in BB_REGEXS: + for regex in BITBUCKET_REGEXS: match = regex.search(url) if match: return match.groups() @@ -47,7 +28,7 @@ def get_bitbucket_username_repo(url=None): def get_gitlab_username_repo(url=None): if 'gitlab' in url: - for regex in GL_REGEXS: + for regex in GITLAB_REGEXS: match = regex.search(url) if match: return match.groups() From 162f12de76080e9a4fdffc16ed015011e6fc1985 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 11 Jan 2018 18:01:45 -0500 Subject: [PATCH 5/8] Do not repeat yourself --- readthedocs/builds/models.py | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index e97f6002bbd..58f85321a9f 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -21,8 +21,7 @@ from readthedocs.core.utils import broadcast from readthedocs.projects.constants import ( - BITBUCKET_REGEXS, BITBUCKET_URL, GITHUB_REGEXS, GITHUB_URL, GITLAB_REGEXS, - GITLAB_URL, PRIVACY_CHOICES, PRIVATE) + BITBUCKET_URL, GITHUB_URL, GITLAB_URL, PRIVACY_CHOICES, PRIVATE) from readthedocs.projects.models import APIProject, Project from .constants import ( @@ -30,6 +29,9 @@ NON_REPOSITORY_VERSIONS, STABLE, TAG, VERSION_TYPES) from .managers import VersionManager from .querysets import BuildQuerySet, RelatedBuildQuerySet, VersionQuerySet +from .utils import ( + get_bitbucket_username_repo, get_github_username_repo, + get_gitlab_username_repo) from .version_slug import VersionSlugField DEFAULT_VERSION_PRIVACY_LEVEL = getattr( @@ -277,12 +279,8 @@ def get_github_url( elif action == 'edit': action_string = 'edit' - for regex in GITHUB_REGEXS: - match = regex.search(repo_url) - if match: - user, repo = match.groups() - break - else: + user, repo = get_github_username_repo(repo_url) + if not user and not repo: return '' repo = repo.rstrip('/') @@ -315,12 +313,8 @@ def get_gitlab_url( elif action == 'edit': action_string = 'edit' - for regex in GITLAB_REGEXS: - match = regex.search(repo_url) - if match: - user, repo = match.groups() - break - else: + user, repo = get_gitlab_username_repo(repo_url) + if not user and not repo: return '' repo = repo.rstrip('/') @@ -341,12 +335,8 @@ def get_bitbucket_url(self, docroot, filename, source_suffix='.rst'): if not docroot: return '' - for regex in BITBUCKET_REGEXS: - match = regex.search(repo_url) - if match: - user, repo = match.groups() - break - else: + user, repo = get_bitbucket_username_repo(repo_url) + if not user and not repo: return '' repo = repo.rstrip('/') From d36a2d2975981aaf8411a112ed4880196125deaf Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 11 Jan 2018 18:53:02 -0500 Subject: [PATCH 6/8] Add regex for bitbucket https clone url --- readthedocs/projects/constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/readthedocs/projects/constants.py b/readthedocs/projects/constants.py index 202c23682aa..9c1ea90896b 100644 --- a/readthedocs/projects/constants.py +++ b/readthedocs/projects/constants.py @@ -297,6 +297,7 @@ re.compile('github.com:(.+)/(.+).git$'), ] BITBUCKET_REGEXS = [ + re.compile('@bitbucket.org/(.+)/(.+)\.git$'), re.compile('bitbucket.org/(.+)/(.+)/'), re.compile('bitbucket.org/(.+)/(.+)'), re.compile('bitbucket.org:(.+)/(.+)\.git$'), From c603bd941bfb0c7eb580c19793e5d88b11d11597 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 11 Jan 2018 18:53:44 -0500 Subject: [PATCH 7/8] Escape . on regex --- readthedocs/projects/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/projects/constants.py b/readthedocs/projects/constants.py index 9c1ea90896b..8837f13b086 100644 --- a/readthedocs/projects/constants.py +++ b/readthedocs/projects/constants.py @@ -294,7 +294,7 @@ GITHUB_REGEXS = [ re.compile('github.com/(.+)/(.+)(?:\.git){1}$'), re.compile('github.com/(.+)/(.+)'), - re.compile('github.com:(.+)/(.+).git$'), + re.compile('github.com:(.+)/(.+)\.git$'), ] BITBUCKET_REGEXS = [ re.compile('@bitbucket.org/(.+)/(.+)\.git$'), From a7b8a2e1afeff31801d02765e9b90d49a99d30c9 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 11 Jan 2018 18:54:24 -0500 Subject: [PATCH 8/8] Add tests for ssh clone url --- .../rtd_tests/tests/test_repo_parsing.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/readthedocs/rtd_tests/tests/test_repo_parsing.py b/readthedocs/rtd_tests/tests/test_repo_parsing.py index c6bb40353bb..43231d68036 100644 --- a/readthedocs/rtd_tests/tests/test_repo_parsing.py +++ b/readthedocs/rtd_tests/tests/test_repo_parsing.py @@ -37,6 +37,13 @@ def test_github(self): self.pip.repo = 'https://github.com/user/repo.git.git' self.assertEqual(self.version.get_github_url(docroot='/docs/', filename='file'), 'https://github.com/user/repo.git/blob/master/docs/file.rst') + def test_github_ssh(self): + self.pip.repo = 'git@github.com:user/repo.git' + self.assertEqual(self.version.get_github_url(docroot='/docs/', filename='file'), 'https://github.com/user/repo/blob/master/docs/file.rst') + + self.pip.repo = 'git@github.com:user/repo.github.io.git' + self.assertEqual(self.version.get_github_url(docroot='/docs/', filename='file'), 'https://github.com/user/repo.github.io/blob/master/docs/file.rst') + def test_gitlab(self): self.pip.repo = 'https://gitlab.com/user/repo' self.assertEqual(self.version.get_gitlab_url(docroot='/foo/bar/', filename='file'), 'https://gitlab.com/user/repo/blob/master/foo/bar/file.rst') @@ -59,6 +66,13 @@ def test_gitlab(self): self.pip.repo = 'https://gitlab.com/user/repo.git.git' self.assertEqual(self.version.get_gitlab_url(docroot='/foo/bar/', filename='file'), 'https://gitlab.com/user/repo.git/blob/master/foo/bar/file.rst') + def test_gitlab_ssh(self): + self.pip.repo = 'git@gitlab.com:user/repo.git' + self.assertEqual(self.version.get_gitlab_url(docroot='/foo/bar/', filename='file'), 'https://gitlab.com/user/repo/blob/master/foo/bar/file.rst') + + self.pip.repo = 'git@gitlab.com:user/repo.gitlab.io.git' + self.assertEqual(self.version.get_gitlab_url(docroot='/foo/bar/', filename='file'), 'https://gitlab.com/user/repo.gitlab.io/blob/master/foo/bar/file.rst') + def test_bitbucket(self): self.pip.repo = 'https://bitbucket.org/user/repo' self.assertEqual(self.version.get_bitbucket_url(docroot='/foo/bar/', filename='file'), 'https://bitbucket.org/user/repo/src/master/foo/bar/file.rst') @@ -80,3 +94,17 @@ def test_bitbucket(self): self.pip.repo = 'https://bitbucket.org/user/repo.git.git' self.assertEqual(self.version.get_bitbucket_url(docroot='/foo/bar/', filename='file'), 'https://bitbucket.org/user/repo.git.git/src/master/foo/bar/file.rst') + + def test_bitbucket_https(self): + self.pip.repo = 'https://user@bitbucket.org/user/repo.git' + self.assertEqual(self.version.get_bitbucket_url(docroot='/foo/bar/', filename='file'), 'https://bitbucket.org/user/repo/src/master/foo/bar/file.rst') + + self.pip.repo = 'https://user@bitbucket.org/user/repo.gitbucket.io.git' + self.assertEqual(self.version.get_bitbucket_url(docroot='/foo/bar/', filename='file'), 'https://bitbucket.org/user/repo.gitbucket.io/src/master/foo/bar/file.rst') + + def test_bitbucket_ssh(self): + self.pip.repo = 'git@bitbucket.org:user/repo.git' + self.assertEqual(self.version.get_bitbucket_url(docroot='/foo/bar/', filename='file'), 'https://bitbucket.org/user/repo/src/master/foo/bar/file.rst') + + self.pip.repo = 'git@bitbucket.org:user/repo.gitbucket.io.git' + self.assertEqual(self.version.get_bitbucket_url(docroot='/foo/bar/', filename='file'), 'https://bitbucket.org/user/repo.gitbucket.io/src/master/foo/bar/file.rst')