Update github-merge.py

contrib/devtools/github-merge.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# Copyright (c) 2016 The Bitcoin Core developers
+# Copyright (c) 2016-2017 Bitcoin Core Developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
@@ -14,17 +14,16 @@
 
 # In case of a clean merge that is accepted by the user, the local branch with
 # name $BRANCH is overwritten with the merged result, and optionally pushed.
-from __future__ import division,print_function,unicode_literals
 import os
 from sys import stdin,stdout,stderr
 import argparse
 import hashlib
 import subprocess
-import json,codecs
-try:
-    from urllib.request import Request,urlopen
-except:
-    from urllib2 import Request,urlopen
+import sys
+import json
+import codecs
+from urllib.request import Request, urlopen
+from urllib.error import HTTPError
 
 # External tools (can be overridden using environment)
 GIT = os.getenv('GIT','git')
@@ -45,24 +44,61 @@ def git_config_get(option, default=None):
     '''
     try:
         return subprocess.check_output([GIT,'config','--get',option]).rstrip().decode('utf-8')
-    except subprocess.CalledProcessError as e:
+    except subprocess.CalledProcessError:
         return default
 
-def retrieve_pr_info(repo,pull):
+def get_response(req_url, ghtoken):
+    req = Request(req_url)
+    if ghtoken is not None:
+        req.add_header('Authorization', 'token ' + ghtoken)
+    return urlopen(req)
+
+def retrieve_json(req_url, ghtoken, use_pagination=False):
     '''
-    Retrieve pull request information from github.
-    Return None if no title can be found, or an error happens.
+    Retrieve json from github.
+    Return None if an error happens.
     '''
     try:
-        req = Request("https://api.github.com/repos/"+repo+"/pulls/"+pull)
-        result = urlopen(req)
         reader = codecs.getreader('utf-8')
-        obj = json.load(reader(result))
+        if not use_pagination:
+            return json.load(reader(get_response(req_url, ghtoken)))
+
+        obj = []
+        page_num = 1
+        while True:
+            req_url_page = '{}?page={}'.format(req_url, page_num)
+            result = get_response(req_url_page, ghtoken)
+            obj.extend(json.load(reader(result)))
+
+            link = result.headers.get('link', None)
+            if link is not None:
+                link_next = [l for l in link.split(',') if 'rel="next"' in l]
+                if len(link_next) > 0:
+                    page_num = int(link_next[0][link_next[0].find("page=")+5:link_next[0].find(">")])
+                    continue
+            break
         return obj
+    except HTTPError as e:
+        error_message = e.read()
+        print('Warning: unable to retrieve pull information from github: %s' % e)
+        print('Detailed error: %s' % error_message)
+        return None
     except Exception as e:
         print('Warning: unable to retrieve pull information from github: %s' % e)
         return None
 
+def retrieve_pr_info(repo,pull,ghtoken):
+    req_url = "https://api.github.com/repos/"+repo+"/pulls/"+pull
+    return retrieve_json(req_url,ghtoken)
+
+def retrieve_pr_comments(repo,pull,ghtoken):
+    req_url = "https://api.github.com/repos/"+repo+"/issues/"+pull+"/comments"
+    return retrieve_json(req_url,ghtoken,use_pagination=True)
+
+def retrieve_pr_reviews(repo,pull,ghtoken):
+    req_url = "https://api.github.com/repos/"+repo+"/pulls/"+pull+"/reviews"
+    return retrieve_json(req_url,ghtoken,use_pagination=True)
+
 def ask_prompt(text):
     print(text,end=" ",file=stderr)
     stderr.flush()
@@ -127,12 +163,26 @@ def tree_sha512sum(commit='HEAD'):
         raise IOError('Non-zero return value executing git cat-file')
     return overall.hexdigest()
 
+def get_acks_from_comments(head_commit, comments):
+    assert len(head_commit) == 6
+    ack_str ='\n\nACKs for commit {}:\n'.format(head_commit)
+    for c in comments:
+        review = [l for l in c['body'].split('\r\n') if 'ACK' in l and head_commit in l]
+        if review:
+            ack_str += '  {}:\n'.format(c['user']['login'])
+            ack_str += '    {}\n'.format(review[0])
+    return ack_str
+
+def print_merge_details(pull, title, branch, base_branch, head_branch):
+    print('%s#%s%s %s %sinto %s%s' % (ATTR_RESET+ATTR_PR,pull,ATTR_RESET,title,ATTR_RESET+ATTR_PR,branch,ATTR_RESET))
+    subprocess.check_call([GIT,'log','--graph','--topo-order','--pretty=format:'+COMMIT_FORMAT,base_branch+'..'+head_branch])
 
 def parse_arguments():
     epilog = '''
         In addition, you can set the following git configuration variables:
         githubmerge.repository (mandatory),
         user.signingkey (mandatory),
+        user.ghtoken (default: none).
         githubmerge.host (default: [email protected]),
         githubmerge.branch (no default),
         githubmerge.testcmd (default: none).
@@ -151,27 +201,35 @@ def main():
     host = git_config_get('githubmerge.host','[email protected]')
     opt_branch = git_config_get('githubmerge.branch',None)
     testcmd = git_config_get('githubmerge.testcmd')
+    ghtoken = git_config_get('user.ghtoken')
     signingkey = git_config_get('user.signingkey')
     if repo is None:
         print("ERROR: No repository configured. Use this command to set:", file=stderr)
         print("git config githubmerge.repository <owner>/<repo>", file=stderr)
-        exit(1)
+        sys.exit(1)
     if signingkey is None:
         print("ERROR: No GPG signing key set. Set one using:",file=stderr)
         print("git config --global user.signingkey <key>",file=stderr)
-        exit(1)
+        sys.exit(1)
 
-    host_repo = host+":"+repo # shortcut for push/pull target
+    if host.startswith(('https:','http:')):
+        host_repo = host+"/"+repo+".git"
+    else:
+        host_repo = host+":"+repo
 
     # Extract settings from command line
     args = parse_arguments()
     pull = str(args.pull[0])
 
     # Receive pull information from github
-    info = retrieve_pr_info(repo,pull)
+    info = retrieve_pr_info(repo,pull,ghtoken)
     if info is None:
-        exit(1)
-    title = info['title']
+        sys.exit(1)
+    comments = retrieve_pr_comments(repo,pull,ghtoken) + retrieve_pr_reviews(repo,pull,ghtoken)
+    if comments is None:
+        sys.exit(1)
+    title = info['title'].strip()
+    body = info['body'].strip()
     # precedence order for destination branch argument:
     #   - command line argument
     #   - githubmerge.branch setting
@@ -185,32 +243,28 @@ def main():
     merge_branch = 'pull/'+pull+'/merge'
     local_merge_branch = 'pull/'+pull+'/local-merge'
 
-    devnull = open(os.devnull,'w')
+    devnull = open(os.devnull, 'w', encoding="utf8")
     try:
         subprocess.check_call([GIT,'checkout','-q',branch])
-    except subprocess.CalledProcessError as e:
+    except subprocess.CalledProcessError:
         print("ERROR: Cannot check out branch %s." % (branch), file=stderr)
-        exit(3)
+        sys.exit(3)
     try:
-        subprocess.check_call([GIT,'fetch','-q',host_repo,'+refs/pull/'+pull+'/*:refs/heads/pull/'+pull+'/*'])
-    except subprocess.CalledProcessError as e:
-        print("ERROR: Cannot find pull request #%s on %s." % (pull,host_repo), file=stderr)
-        exit(3)
+        subprocess.check_call([GIT,'fetch','-q',host_repo,'+refs/pull/'+pull+'/*:refs/heads/pull/'+pull+'/*',
+                                                          '+refs/heads/'+branch+':refs/heads/'+base_branch])
+    except subprocess.CalledProcessError:
+        print("ERROR: Cannot find pull request #%s or branch %s on %s." % (pull,branch,host_repo), file=stderr)
+        sys.exit(3)
     try:
         subprocess.check_call([GIT,'log','-q','-1','refs/heads/'+head_branch], stdout=devnull, stderr=stdout)
-    except subprocess.CalledProcessError as e:
+    except subprocess.CalledProcessError:
         print("ERROR: Cannot find head of pull request #%s on %s." % (pull,host_repo), file=stderr)
-        exit(3)
+        sys.exit(3)
     try:
         subprocess.check_call([GIT,'log','-q','-1','refs/heads/'+merge_branch], stdout=devnull, stderr=stdout)
-    except subprocess.CalledProcessError as e:
+    except subprocess.CalledProcessError:
         print("ERROR: Cannot find merge of pull request #%s on %s." % (pull,host_repo), file=stderr)
-        exit(3)
-    try:
-        subprocess.check_call([GIT,'fetch','-q',host_repo,'+refs/heads/'+branch+':refs/heads/'+base_branch])
-    except subprocess.CalledProcessError as e:
-        print("ERROR: Cannot find branch %s on %s." % (branch,host_repo), file=stderr)
-        exit(3)
+        sys.exit(3)
     subprocess.check_call([GIT,'checkout','-q',base_branch])
     subprocess.call([GIT,'branch','-q','-D',local_merge_branch], stderr=devnull)
     subprocess.check_call([GIT,'checkout','-q','-b',local_merge_branch])
@@ -226,45 +280,46 @@ def main():
             firstline = 'Merge #%s' % (pull,)
         message = firstline + '\n\n'
         message += subprocess.check_output([GIT,'log','--no-merges','--topo-order','--pretty=format:%h %s (%an)',base_branch+'..'+head_branch]).decode('utf-8')
+        message += '\n\nPull request description:\n\n  ' + body.replace('\n', '\n  ') + '\n'
+        message += get_acks_from_comments(head_commit=subprocess.check_output([GIT,'log','-1','--pretty=format:%H',head_branch]).decode('utf-8')[:6], comments=comments)
         try:
-            subprocess.check_call([GIT,'merge','-q','--commit','--no-edit','--no-ff','-m',message.encode('utf-8'),head_branch])
-        except subprocess.CalledProcessError as e:
+            subprocess.check_call([GIT,'merge','-q','--commit','--no-edit','--no-ff','--no-gpg-sign','-m',message.encode('utf-8'),head_branch])
+        except subprocess.CalledProcessError:
             print("ERROR: Cannot be merged cleanly.",file=stderr)
             subprocess.check_call([GIT,'merge','--abort'])
-            exit(4)
+            sys.exit(4)
         logmsg = subprocess.check_output([GIT,'log','--pretty=format:%s','-n','1']).decode('utf-8')
         if logmsg.rstrip() != firstline.rstrip():
             print("ERROR: Creating merge failed (already merged?).",file=stderr)
-            exit(4)
+            sys.exit(4)
 
         symlink_files = get_symlink_files()
         for f in symlink_files:
             print("ERROR: File %s was a symlink" % f)
         if len(symlink_files) > 0:
-            exit(4)
+            sys.exit(4)
 
         # Put tree SHA512 into the message
         try:
             first_sha512 = tree_sha512sum()
             message += '\n\nTree-SHA512: ' + first_sha512
-        except subprocess.CalledProcessError as e:
-            printf("ERROR: Unable to compute tree hash")
-            exit(4)
+        except subprocess.CalledProcessError:
+            print("ERROR: Unable to compute tree hash")
+            sys.exit(4)
         try:
-            subprocess.check_call([GIT,'commit','--amend','-m',message.encode('utf-8')])
-        except subprocess.CalledProcessError as e:
-            printf("ERROR: Cannot update message.",file=stderr)
-            exit(4)
+            subprocess.check_call([GIT,'commit','--amend','--no-gpg-sign','-m',message.encode('utf-8')])
+        except subprocess.CalledProcessError:
+            print("ERROR: Cannot update message.", file=stderr)
+            sys.exit(4)
 
-        print('%s#%s%s %s %sinto %s%s' % (ATTR_RESET+ATTR_PR,pull,ATTR_RESET,title,ATTR_RESET+ATTR_PR,branch,ATTR_RESET))
-        subprocess.check_call([GIT,'log','--graph','--topo-order','--pretty=format:'+COMMIT_FORMAT,base_branch+'..'+head_branch])
+        print_merge_details(pull, title, branch, base_branch, head_branch)
         print()
 
         # Run test command if configured.
         if testcmd:
             if subprocess.call(testcmd,shell=True):
                 print("ERROR: Running %s failed." % testcmd,file=stderr)
-                exit(5)
+                sys.exit(5)
 
             # Show the created merge.
             diff = subprocess.check_output([GIT,'diff',merge_branch+'..'+local_merge_branch])
@@ -275,13 +330,7 @@ def main():
                 if reply.lower() == 'ignore':
                     print("Difference with github ignored.",file=stderr)
                 else:
-                    exit(6)
-            reply = ask_prompt("Press 'd' to accept the diff.")
-            if reply.lower() == 'd':
-                print("Diff accepted.",file=stderr)
-            else:
-                print("ERROR: Diff rejected.",file=stderr)
-                exit(6)
+                    sys.exit(6)
         else:
             # Verify the result manually.
             print("Dropping you on a shell so you can try building/testing the merged source.",file=stderr)
@@ -290,29 +339,25 @@ def main():
             if os.path.isfile('/etc/debian_version'): # Show pull number on Debian default prompt
                 os.putenv('debian_chroot',pull)
             subprocess.call([BASH,'-i'])
-            reply = ask_prompt("Type 'm' to accept the merge.")
-            if reply.lower() == 'm':
-                print("Merge accepted.",file=stderr)
-            else:
-                print("ERROR: Merge rejected.",file=stderr)
-                exit(7)
 
         second_sha512 = tree_sha512sum()
         if first_sha512 != second_sha512:
             print("ERROR: Tree hash changed unexpectedly",file=stderr)
-            exit(8)
+            sys.exit(8)
 
         # Sign the merge commit.
-        reply = ask_prompt("Type 's' to sign off on the merge.")
-        if reply == 's':
-            try:
-                subprocess.check_call([GIT,'commit','-q','--gpg-sign','--amend','--no-edit'])
-            except subprocess.CalledProcessError as e:
-                print("Error signing, exiting.",file=stderr)
-                exit(1)
-        else:
-            print("Not signing off on merge, exiting.",file=stderr)
-            exit(1)
+        print_merge_details(pull, title, branch, base_branch, head_branch)
+        while True:
+            reply = ask_prompt("Type 's' to sign off on the above merge, or 'x' to reject and exit.").lower()
+            if reply == 's':
+                try:
+                    subprocess.check_call([GIT,'commit','-q','--gpg-sign','--amend','--no-edit'])
+                    break
+                except subprocess.CalledProcessError:
+                    print("Error while signing, asking again.",file=stderr)
+            elif reply == 'x':
+                print("Not signing off on merge, exiting.",file=stderr)
+                sys.exit(1)
 
         # Put the result in branch.
         subprocess.check_call([GIT,'checkout','-q',branch])
@@ -326,9 +371,14 @@ def main():
         subprocess.call([GIT,'branch','-q','-D',local_merge_branch],stderr=devnull)
 
     # Push the result.
-    reply = ask_prompt("Type 'push' to push the result to %s, branch %s." % (host_repo,branch))
-    if reply.lower() == 'push':
-        subprocess.check_call([GIT,'push',host_repo,'refs/heads/'+branch])
+    while True:
+        reply = ask_prompt("Type 'push' to push the result to %s, branch %s, or 'x' to exit without pushing." % (host_repo,branch)).lower()
+        if reply == 'push':
+            subprocess.check_call([GIT,'push',host_repo,'refs/heads/'+branch])
+            break
+        elif reply == 'x':
+            sys.exit(1)
 
 if __name__ == '__main__':
     main()
+