From 15b434a9dad651916b013164d4f7db999264b5f9 Mon Sep 17 00:00:00 2001 From: Parth Sharma Date: Sun, 30 Jun 2019 15:50:12 +0530 Subject: [PATCH 1/3] refactor timeseries metric implementations to return a 'date' field Signed-off-by: Parth Sharma --- augur/datasources/augur_db/augur_db.py | 28 +++++++++---------- augur/datasources/augur_db/routes.py | 38 +++++++++++++------------- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/augur/datasources/augur_db/augur_db.py b/augur/datasources/augur_db/augur_db.py index 652c90bcdf..8c48250d10 100644 --- a/augur/datasources/augur_db/augur_db.py +++ b/augur/datasources/augur_db/augur_db.py @@ -110,8 +110,8 @@ def code_changes(self, repo_group_id, repo_id=None, period='day', begin_date=Non FROM commits WHERE repo_id = :repo_id AND cmt_committer_date BETWEEN :begin_date AND :end_date - GROUP BY commit_date - ORDER BY commit_date + GROUP BY date + ORDER BY date """) results = pd.read_sql(code_changes_SQL, self.db, params={'repo_id': repo_id, 'period': period, @@ -529,15 +529,15 @@ def code_changes_lines(self, repo_group_id, repo_id=None, period='day', begin_da if not repo_id: code_changes_lines_SQL = s.sql.text(""" SELECT - date_trunc(:period, cmt_author_date::DATE) as commit_date, + date_trunc(:period, cmt_author_date::DATE) as date, repo_id, SUM(cmt_added) as added, SUM(cmt_removed) as removed FROM commits WHERE repo_id IN (SELECT repo_id FROM repo WHERE repo_group_id = :repo_group_id) AND cmt_author_date BETWEEN :begin_date AND :end_date - GROUP BY commit_date, repo_id - ORDER BY repo_id, commit_date + GROUP BY date, repo_id + ORDER BY repo_id, date """) results = pd.read_sql(code_changes_lines_SQL, self.db, params={'repo_group_id': repo_group_id, 'period': period, @@ -548,14 +548,14 @@ def code_changes_lines(self, repo_group_id, repo_id=None, period='day', begin_da else: code_changes_lines_SQL = s.sql.text(""" SELECT - date_trunc(:period, cmt_author_date::DATE) as commit_date, + date_trunc(:period, cmt_author_date::DATE) as date, SUM(cmt_added) AS added, SUM(cmt_removed) as removed FROM commits WHERE repo_id = :repo_id AND cmt_author_date BETWEEN :begin_date AND :end_date - GROUP BY commit_date - ORDER BY commit_date; + GROUP BY date + ORDER BY date; """) results = pd.read_sql(code_changes_lines_SQL, self.db, params={'repo_id': repo_id, 'period': period, @@ -583,14 +583,14 @@ def issues_new(self, repo_group_id, repo_id=None, period='day', begin_date=None, if not repo_id: issues_new_SQL = s.sql.text(""" SELECT - date_trunc(:period, created_at::DATE) as issue_date, + date_trunc(:period, created_at::DATE) as date, repo_id, COUNT(issue_id) as issues FROM issues WHERE repo_id IN (SELECT repo_id FROM repo WHERE repo_group_id = :repo_group_id) AND created_at BETWEEN to_timestamp(:begin_date, 'YYYY-MM-DD HH24:MI:SS') AND to_timestamp(:end_date, 'YYYY-MM-DD HH24:MI:SS') - GROUP BY issue_date, repo_id - ORDER BY repo_id, issue_date + GROUP BY date, repo_id + ORDER BY repo_id, date """) results = pd.read_sql(issues_new_SQL, self.db, params={'repo_group_id': repo_group_id, 'period': period, @@ -600,12 +600,12 @@ def issues_new(self, repo_group_id, repo_id=None, period='day', begin_date=None, else: issues_new_SQL = s.sql.text(""" - SELECT date_trunc(:period, created_at::DATE) as issue_date, COUNT(issue_id) as issues + SELECT date_trunc(:period, created_at::DATE) as date, COUNT(issue_id) as issues FROM issues WHERE repo_id = :repo_id AND created_at BETWEEN to_timestamp(:begin_date, 'YYYY-MM-DD HH24:MI:SS') AND to_timestamp(:end_date, 'YYYY-MM-DD HH24:MI:SS') - GROUP BY issue_date - ORDER BY issue_date; + GROUP BY date + ORDER BY date; """) results = pd.read_sql(issues_new_SQL, self.db, params={'repo_id': repo_id, 'period': period, diff --git a/augur/datasources/augur_db/routes.py b/augur/datasources/augur_db/routes.py index 414509f4f3..d64dd4addb 100644 --- a/augur/datasources/augur_db/routes.py +++ b/augur/datasources/augur_db/routes.py @@ -136,17 +136,17 @@ def get_repo(owner, repo): @apiSuccessExample {json} Success-Response: [ { - "commit_date": "2018-01-01T00:00:00.000Z", + "date": "2018-01-01T00:00:00.000Z", "repo_id": 1, "commit_count": 5140 }, { - "commit_date": "2019-01-01T00:00:00.000Z", + "date": "2019-01-01T00:00:00.000Z", "repo_id": 1, "commit_count": 711 }, { - "commit_date": "2015-01-01T00:00:00.000Z", + "date": "2015-01-01T00:00:00.000Z", "repo_id": 25001, "commit_count": 1071 } @@ -168,19 +168,19 @@ def get_repo(owner, repo): @apiSuccessExample {json} Success-Response: [ { - "commit_date": "2018-01-01T00:00:00.000Z", + "date": "2018-01-01T00:00:00.000Z", "commit_count": 2287 }, { - "commit_date": "2018-02-01T00:00:00.000Z", + "date": "2018-02-01T00:00:00.000Z", "commit_count": 1939 }, { - "commit_date": "2018-03-01T00:00:00.000Z", + "date": "2018-03-01T00:00:00.000Z", "commit_count": 1979 }, { - "commit_date": "2018-04-01T00:00:00.000Z", + "date": "2018-04-01T00:00:00.000Z", "commit_count": 2159 } ] @@ -200,25 +200,25 @@ def get_repo(owner, repo): @apiSuccessExample {json} Success-Response: [ { - "commit_date": "2018-01-01T00:00:00.000Z", + "date": "2018-01-01T00:00:00.000Z", "repo_id": 1, "added": 640098, "removed": 694608 }, { - "commit_date": "2019-01-01T00:00:00.000Z", + "date": "2019-01-01T00:00:00.000Z", "repo_id": 1, "added": 56549, "removed": 48962 }, { - "commit_date": "2014-01-01T00:00:00.000Z", + "date": "2014-01-01T00:00:00.000Z", "repo_id": 25001, "added": 19, "removed": 1 }, { - "commit_date": "2015-01-01T00:00:00.000Z", + "date": "2015-01-01T00:00:00.000Z", "repo_id": 25001, "added": 429535, "removed": 204015 @@ -241,22 +241,22 @@ def get_repo(owner, repo): @apiSuccessExample {json} Success-Response: [ { - "commit_date": "2014-01-01T00:00:00.000Z", + "date": "2014-01-01T00:00:00.000Z", "added": 19, "removed": 1 }, { - "commit_date": "2015-01-01T00:00:00.000Z", + "date": "2015-01-01T00:00:00.000Z", "added": 429535, "removed": 204015 }, { - "commit_date": "2016-01-01T00:00:00.000Z", + "date": "2016-01-01T00:00:00.000Z", "added": 2739765, "removed": 944568 }, { - "commit_date": "2017-01-01T00:00:00.000Z", + "date": "2017-01-01T00:00:00.000Z", "added": 3945001, "removed": 1011396 } @@ -277,12 +277,12 @@ def get_repo(owner, repo): @apiSuccessExample {json} Success-Response: [ { - "issue_date": "2019-05-01T00:00:00.000Z", + "date": "2019-05-01T00:00:00.000Z", "repo_id": 1, "issues": 3 }, { - "issue_date": "2019-05-01T00:00:00.000Z", + "date": "2019-05-01T00:00:00.000Z", "repo_id": 25001, "issues": 1 } @@ -304,11 +304,11 @@ def get_repo(owner, repo): @apiSuccessExample {json} Success-Response: [ { - "issue_date": "2019-05-01T00:00:00.000Z", + "date": "2019-05-01T00:00:00.000Z", "issues": 1 }, { - "issue_date": "2019-06-01T00:00:00.000Z", + "date": "2019-06-01T00:00:00.000Z", "issues": 31 } ] From eb7071b0b00d7900f005d5eeb46c1f1b65582ef2 Mon Sep 17 00:00:00 2001 From: Parth Sharma Date: Sun, 30 Jun 2019 19:35:57 +0530 Subject: [PATCH 2/3] add & update unit tests in test_augur_db.py Signed-off-by: Parth Sharma --- augur/datasources/augur_db/test_augur_db.py | 97 +++++++++++++++++++-- 1 file changed, 88 insertions(+), 9 deletions(-) diff --git a/augur/datasources/augur_db/test_augur_db.py b/augur/datasources/augur_db/test_augur_db.py index ccecf5b43f..ee01b76bef 100644 --- a/augur/datasources/augur_db/test_augur_db.py +++ b/augur/datasources/augur_db/test_augur_db.py @@ -51,24 +51,73 @@ def test_code_changes_lines(augur_db): def test_issues_new(augur_db): #repo_id - assert augur_db.issues_new(23, 21430, period='year').iloc[0]['issues'] == 2 + assert augur_db.issues_new(23, 21403, period='year').iloc[0]['issues'] == 1 #repo_group_id - assert augur_db.issues_new(23, period='year').iloc[0]['issues'] == 2 + assert augur_db.issues_new(23, period='year').iloc[1]['issues'] == 1 #begin_date & end_date assert augur_db.issues_new(24, 21979, period='week', begin_date='2017', end_date='2017-05').iloc[1]['issues'] == 4 assert augur_db.issues_new(24, period='month', begin_date='2017-05', - end_date='2018').iloc[2]['issues'] == 7 + end_date='2018').iloc[2]['issues'] == 2 + +def test_issues_active(augur_db): + # repo + assert augur_db.issues_active(22, 21326, period='year').iloc[0]['issues'] == 98 + + # repo_group + assert augur_db.issues_active(22, period='year').iloc[5]['issues'] == 20 + + # begin_date & end_date + assert augur_db.issues_active(22, 21326, period='month', begin_date='2015', + end_date='2015-09').iloc[0]['issues'] == 32 + + assert augur_db.issues_active(22, period='week', begin_date='2015-01', + end_date='2015-08-05') .iloc[0]['issues'] == 32 + +def test_issues_closed(augur_db): + # repo + assert augur_db.issues_closed(24, 21681, period='year').iloc[0]['issues'] == 189 + + #repo_group + assert augur_db.issues_closed(24, period='year').iloc[1]['issues'] == 97 + + # begin_date & end_date + assert augur_db.issues_closed(24, 21681, period='week', begin_date='2012', + end_date='2012-07').iloc[0]['issues'] == 10 + + assert augur_db.issues_closed(24, period='month', begin_date='2012-05', + end_date='2012-08-15').iloc[0]['issues'] == 50 + +def test_issue_duration(augur_db): + # repo + assert augur_db.issue_duration(24, 21681).iloc[0]['duration'] == '0 days 01:06:31.000000000' + + # repo_group + assert augur_db.issue_duration(24).iloc[3]['duration'] == '5 days 05:18:21.000000000' + +def test_issue_participants(augur_db): + # repo + assert augur_db.issue_participants(23, 21403).iloc[0]['participants'] == 1 + + # repo_group + assert augur_db.issue_participants(22).iloc[4]['participants'] == 4 + +def test_issue_throughput(augur_db): + # repo + assert augur_db.issue_throughput(20, 21009).iloc[0]['throughput'] == 0.263158 + + # repo_group + assert augur_db.issue_throughput(24).iloc[0]['throughput'] == 0.861896 def test_issue_backlog(augur_db): #repo_id assert augur_db.issue_backlog(21, 21166).iloc[0]['issue_backlog'] == 4 #repo_group_id - assert augur_db.issue_backlog(21).iloc[2]['issue_backlog'] == 3 - + assert augur_db.issue_backlog(21).iloc[2]['issue_backlog'] == 20 + def test_issues_first_time_closed(augur_db): @@ -148,17 +197,47 @@ def test_contributors_new(augur_db): assert augur_db.contributors_new(24, repo_id=21524, period='year', begin_date='2019-1-1 00:00:00', end_date='2019-12-31 23:59:59').isin([pd.Timestamp('2019-01-01 00:00:00', tz='UTC')]).any().any() +def test_open_issues_count(augur_db): + # repo + assert augur_db.open_issues_count(22, 21326).iloc[0]['open_count'] == 1 + + # repo_group + assert augur_db.open_issues_count(23).iloc[1]['open_count'] == 4 + +def test_closed_issues_count(augur_db): + # repo + assert augur_db.closed_issues_count(24, 21684).iloc[0]['closed_count'] == 3 + + # repo_group + assert augur_db.closed_issues_count(20).iloc[0]['closed_count'] == 1 + def test_issues_open_age(augur_db): #repo group assert augur_db.issues_open_age(24).iloc[0]['datedifference'] > 100 - # repo + # repo assert augur_db.issues_open_age(20,21000).iloc[0]['datedifference'] > 100 def test_issues_closed_resolution_duration(augur_db): - # repo group + # repo group assert augur_db.issues_closed_resolution_duration(24).iloc[0]['diffdate'] >= 0 - # repo + # repo assert augur_db.issues_closed_resolution_duration(24,21682).iloc[0]['diffdate'] >= 0 def test_get_repo(augur_db): - assert augur_db.get_repo_by_name('Comcast','zucchini').iloc[0].repo_id == 21116 \ No newline at end of file + assert augur_db.get_repo_by_name('Comcast','zucchini').iloc[0].repo_id == 21116 + +def test_cii_best_practices_badge(augur_db): + # repo + assert augur_db.cii_best_practices_badge(21, 21252).iloc[0]['badge_level'] == 'in_progress' + + # repo_group + assert augur_db.cii_best_practices_badge(21).iloc[0]['badge_level'] == 'passing' + +def test_languages(augur_db): + # TODO + pass + +def test_license_declared(augur_db): + assert augur_db.license_declared(21, 21252).iloc[0]['license'] == 'Apache-2.0' + + assert augur_db.license_declared(21).iloc[0]['license'] == 'Apache-2.0' From 0219b03d0129ea53bbfc7b0c7ae7f4f6a83a5fa1 Mon Sep 17 00:00:00 2001 From: Parth Sharma Date: Sun, 30 Jun 2019 20:20:25 +0530 Subject: [PATCH 3/3] add & update API tests in test_augur_db_routes.py Signed-off-by: Parth Sharma --- .../augur_db/test_augur_db_routes.py | 130 +++++++++++++++++- 1 file changed, 125 insertions(+), 5 deletions(-) diff --git a/augur/datasources/augur_db/test_augur_db_routes.py b/augur/datasources/augur_db/test_augur_db_routes.py index 5a2c919b45..18b48ef9aa 100644 --- a/augur/datasources/augur_db/test_augur_db_routes.py +++ b/augur/datasources/augur_db/test_augur_db_routes.py @@ -67,26 +67,82 @@ def test_issues_new_by_group(augur_db_routes): assert data[0]['issues'] > 0 def test_issues_new_by_repo(augur_db_routes): - response = requests.get('http://localhost:5000/api/unstable/repo-groups/23/repos/21350/issues-new') + response = requests.get('http://localhost:5000/api/unstable/repo-groups/23/repos/21339/issues-new') + data = response.json() + assert response.status_code == 200 + assert len(data) >= 1 + assert data[0]['issues'] >= 1 + +def test_issues_active_by_group(augur_db_routes): + response = requests.get('http://localhost:5000/api/unstable/repo-groups/23/issues-active') + data = response.json() + assert response.status_code == 200 + assert len(data) >= 1 + assert data[0]['issues'] >= 1 + +def test_issues_active_by_repo(augur_db_routes): + response = requests.get('http://localhost:5000/api/unstable/repo-groups/23/repos/21339/issues-active') data = response.json() assert response.status_code == 200 assert len(data) >= 1 assert data[0]['issues'] >= 1 def test_issues_closed_by_group(augur_db_routes): - response = requests.get('http://localhost:5000/api/unstable/repo-groups/23/issues-closed') + response = requests.get('http://localhost:5000/api/unstable/repo-groups/24/issues-closed') data = response.json() assert response.status_code == 200 assert len(data) >= 1 assert data[0]['issues'] > 0 def test_issues_closed_by_repo(augur_db_routes): - response = requests.get('http://localhost:5000/api/unstable/repo-groups/23/repos/21350/issues-closed') + response = requests.get('http://localhost:5000/api/unstable/repo-groups/24/repos/21681/issues-closed') data = response.json() assert response.status_code == 200 assert len(data) >= 1 assert data[0]['issues'] > 0 +def test_issue_duration_by_group(augur_db_routes): + response = requests.get('http://localhost:5000/api/unstable/repo-groups/20/issue-duration') + data = response.json() + assert response.status_code == 200 + assert len(data) >= 1 + assert data[0]['duration'] == '60 days 19:41:27.000000000' + +def test_issue_duration_by_repo(augur_db_routes): + response = requests.get('http://localhost:5000/api/unstable/repo-groups/20/repos/21009/issue-duration') + data = response.json() + assert response.status_code == 200 + assert len(data) >= 1 + assert data[0]['duration'] == '60 days 19:41:27.000000000' + +def test_issue_participants_by_group(augur_db_routes): + response = requests.get('http://localhost:5000/api/unstable/repo-groups/23/issue-participants') + data = response.json() + assert response.status_code == 200 + assert len(data) >= 1 + assert data[0]['participants'] == 1 + +def test_issue_participants_by_repo(augur_db_routes): + response = requests.get('http://localhost:5000/api/unstable/repo-groups/23/repos/21403/issue-participants') + data = response.json() + assert response.status_code == 200 + assert len(data) >= 1 + assert data[0]['participants'] == 1 + +def test_issue_throughput_by_group(augur_db_routes): + response = requests.get('http://localhost:5000/api/unstable/repo-groups/20/issue-throughput') + data = response.json() + assert response.status_code == 200 + assert len(data) >= 1 + assert data[0]['throughput'] == 0.263158 + +def test_issue_throughput_by_repo(augur_db_routes): + response = requests.get('http://localhost:5000/api/unstable/repo-groups/24/repos/21682/issue-throughput') + data = response.json() + assert response.status_code == 200 + assert len(data) >= 1 + assert data[0]['throughput'] == 0.861896 + def test_issue_backlog_by_group(augur_db_routes): response = requests.get('http://localhost:5000/api/unstable/repo-groups/23/issue-backlog') data = response.json() @@ -95,7 +151,7 @@ def test_issue_backlog_by_group(augur_db_routes): assert data[0]['issue_backlog'] > 0 def test_issue_backlog_by_repo(augur_db_routes): - response = requests.get('http://localhost:5000/api/unstable/repo-groups/23/repos/21350/issue-backlog') + response = requests.get('http://localhost:5000/api/unstable/repo-groups/23/repos/21403/issue-backlog') data = response.json() assert response.status_code == 200 assert len(data) >= 1 @@ -187,6 +243,34 @@ def test_contributors_new_by_repo(augur_db_routes): assert len(data) >= 1 assert data[0]["count"] > 0 +def test_open_issues_count_by_group(augur_db_routes): + response = requests.get('http://localhost:5000/api/unstable/repo-groups/22/open-issues-count') + data = response.json() + assert response.status_code == 200 + assert len(data) >= 1 + assert data[0]['open_count'] == 1 + +def test_open_issues_count_by_repo(augur_db_routes): + response = requests.get('http://localhost:5000/api/unstable/repo-groups/22/repos/21326/open-issues-count') + data = response.json() + assert response.status_code == 200 + assert len(data) >= 1 + assert data[0]['open_count'] == 1 + +def test_closed_issues_count_by_group(augur_db_routes): + response = requests.get('http://localhost:5000/api/unstable/repo-groups/20/closed-issues-count') + data = response.json() + assert response.status_code == 200 + assert len(data) >= 1 + assert data[0]['closed_count'] == 1 + +def test_closed_issues_count_by_repo(augur_db_routes): + response = requests.get('http://localhost:5000/api/unstable/repo-groups/22/repos/21684/closed-issues-count') + data = response.json() + assert response.status_code == 200 + assert len(data) >= 1 + assert data[0]['closed_count'] == 3 + def test_issues_open_age_by_group(augur_db_routes): response = requests.get('http://localhost:5000/api/unstable/repo-groups/20/issues-open-age/') data = response.json() @@ -214,4 +298,40 @@ def test_issues_closed_resolution_duration_by_repo(augur_db_routes): data = response.json() assert response.status_code == 200 assert len(data) >= 1 - assert data[0]["diffdate"] >= 0 \ No newline at end of file + assert data[0]["diffdate"] >= 0 + +def test_cii_best_practices_badge_by_group(augur_db_routes): + response = requests.get('http://localhost:5000/api/unstable/repo-groups/21/cii-best-practices-badge') + data = response.json() + assert response.status_code == 200 + assert len(data) >= 1 + assert data[0]["badge_level"] == 'passing' + +def test_cii_best_practices_badge_by_repo(augur_db_routes): + response = requests.get('http://localhost:5000/api/unstable/repo-groups/21/repos/21252/cii-best-practices-badge') + data = response.json() + assert response.status_code == 200 + assert len(data) >= 1 + assert data[0]["badge_level"] == 'in_progress' + +def test_languages_by_group(augur_db_routes): + # TODO need data + pass + +def test_languages_by_repo(augur_db_routes): + # TODO need data + pass + +def test_license_declared_by_group(augur_db_routes): + response = requests.get('http://localhost:5000/api/unstable/repo-groups/21/license-declared') + data = response.json() + assert response.status_code == 200 + assert len(data) >= 1 + assert data[0]["license"] == 'Apache-2.0' + +def test_license_declared_by_repo(augur_db_routes): + response = requests.get('http://localhost:5000/api/unstable/repo-groups/21/repos/21252/license-declared') + data = response.json() + assert response.status_code == 200 + assert len(data) >= 1 + assert data[0]["license"] == 'Apache-2.0'