From 12b134b0094d81701f8d41157044090f92cecbcb Mon Sep 17 00:00:00 2001 From: BlBana <635373043@qq.com> Date: Mon, 26 Mar 2018 14:01:55 +0800 Subject: [PATCH 1/3] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=BA=86=E5=AF=B9?= =?UTF-8?q?=E6=8A=A5=E5=91=8A=E5=AF=BC=E5=87=BA=E5=88=B0=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E6=97=B6=EF=BC=8C=E6=89=93=E5=BC=80=E6=96=87=E4=BB=B6=E7=9A=84?= =?UTF-8?q?=E5=BC=82=E5=B8=B8=E5=88=A4=E6=96=AD=E4=BB=A5=E5=8F=8A=E4=BF=AE?= =?UTF-8?q?=E6=94=B9=E4=BA=86=E6=89=B9=E9=87=8Fpush=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E7=9A=84=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cobra/export.py | 38 +++++++++++++++++++++----------------- git_projects.py | 32 ++++++++++++++++++++------------ 2 files changed, 41 insertions(+), 29 deletions(-) diff --git a/cobra/export.py b/cobra/export.py index 9330d707..7da1ec94 100644 --- a/cobra/export.py +++ b/cobra/export.py @@ -144,24 +144,28 @@ def write_to_file(target, sid, output_format='', filename=None): logger.info('Vulnerabilities\n' + str(dict_to_pretty_table(scan_data.get('vulnerabilities')))) elif output_format == 'json' or output_format == 'JSON': - if not os.path.exists(filename): - with open(filename, 'w', encoding='utf-8') as f: - json_data = { - sid: scan_data, - } - f.write(dict_to_json(json_data)) - else: - with open(filename, 'r+', encoding='utf-8') as f: - try: - json_data = json.load(f) - json_data.update({sid: scan_data}) - # 使用 r+ 模式不会覆盖，调整文件指针到开头 - f.seek(0) - f.truncate() + try: + if not os.path.exists(filename): + with open(filename, 'w', encoding='utf-8') as f: + json_data = { + sid: scan_data, + } f.write(dict_to_json(json_data)) - except ValueError: - logger.warning('[EXPORT] The json file have invaild token or None: {}'.format(os.path.join(export_path, filename))) - return False + else: + with open(filename, 'r+', encoding='utf-8') as f: + try: + json_data = json.load(f) + json_data.update({sid: scan_data}) + # 使用 r+ 模式不会覆盖，调整文件指针到开头 + f.seek(0) + f.truncate() + f.write(dict_to_json(json_data)) + except ValueError: + logger.warning('[EXPORT] The json file have invaild token or None: {}'.format(os.path.join(export_path, filename))) + return False + except IOError: + logger.warning('[EXPORT] Please input a file path after the -o parameter') + return False elif output_format == 'xml' or output_format == 'XML': xml_data = { diff --git a/git_projects.py b/git_projects.py index 88ff6954..4bcf9da4 100644 --- a/git_projects.py +++ b/git_projects.py @@ -26,6 +26,9 @@ import Queue as queue +git_urls = [] + + def start(): url = Config('git', 'gitlab_url').value private_token = Config('git', 'private_token').value @@ -40,34 +43,37 @@ def start(): q_pages.put(i + 1) for i in range(10): - thread = threading.Thread(target=get_git_urls, args=(url, private_token, cobra_ip, key, q_pages, fi)) + thread = threading.Thread(target=get_git_urls, args=(url, private_token, q_pages, fi)) thread.start() threads.append(thread) for thread in threads: thread.join() + res = push_to_api(git_urls, cobra_ip, key, fi) + + if res: + logger.info("Git push success: {}".format(len(git_urls))) + else: + logger.info("Git push fail") + fi.close() logger.info("All projects have been pushed") -def get_git_urls(url, private_token, cobra_ip, key, q_pages, fi): +def get_git_urls(url, private_token, q_pages, fi): """ :param url: The gitlab's projects api ,example:http://xxx.gitlab.com/api/v3/projects :param private_token: The user's private_token - :param cobra_ip: The Cobra server's ip - :param key: The Cobra api key :param q_pages: The Queue of pages :param fi: The result in this file :return: """ while not q_pages.empty(): - git_urls = [] page = q_pages.get() params = {'private_token': private_token, 'page': page} url = url r = request_target(url, params, method="get") - if r.status_code == 200: data = r.json() # 一个页面中的Json数据，默认20条 for j in range(len(data)): @@ -80,12 +86,8 @@ def get_git_urls(url, private_token, cobra_ip, key, q_pages, fi): else: request_url = git_url + fi.write(request_url + '\n') git_urls.append(request_url) - res = push_to_api(git_urls, cobra_ip, key, fi) - if res: - logger.info("page %d git push success" % page) - else: - logger.info("page %d git push fail" % page) elif r.status_code == 404: logger.warning("page %d 404" % page) @@ -107,12 +109,14 @@ def request_target(target_url, params=None, header=None, method="get"): def push_to_api(urls, cobra_ip, key, fi): headers = {"Content-Type": "application/json"} url = cobra_ip + "/api/add" - payload = {"key": key, "target": urls} + payload = {"key": key, "target": urls, "dels": True, "rule": "cvi-190009"} r = request_target(url, payload, headers, method="post") if r.status_code == 200: fi.write(str(r.json()) + '\n') logger.info(r.json()) return True + elif r.status_code == 404: + logger.info("The page is 404") else: logger.info(r.json()) return False @@ -126,3 +130,7 @@ def get_pages(url, private_token): res = re.search(r"all\?page=(\d*)&per_page=0", res) pages = res.group(1) return pages + + +if __name__ == '__main__': + start() From 41add931a287716655a87319608878f3aed1b5b7 Mon Sep 17 00:00:00 2001 From: BlBana <635373043@qq.com> Date: Mon, 26 Mar 2018 14:48:40 +0800 Subject: [PATCH 2/3] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=BA=86=E5=AF=B9?= =?UTF-8?q?=E9=82=AE=E4=BB=B6=E9=99=84=E4=BB=B6=E6=98=AF=E5=90=A6=E5=AD=98?= =?UTF-8?q?=E5=9C=A8=E7=9A=84=E5=88=A4=E6=96=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cobra/send_mail.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/cobra/send_mail.py b/cobra/send_mail.py index bd02f744..0a630c41 100644 --- a/cobra/send_mail.py +++ b/cobra/send_mail.py @@ -31,10 +31,14 @@ def send_mail(target, filename, receiver): msg.attach(MIMEText('扫描项目：{t}\n报告见附件'.format(t=target), 'plain', 'utf-8')) - with open(filename, 'rb') as f: - attachment = MIMEApplication(f.read()) - attachment.add_header('Content-Disposition', 'attachment', filename=os.path.split(filename)[1]) - msg.attach(attachment) + try: + with open(filename, 'rb') as f: + attachment = MIMEApplication(f.read()) + attachment.add_header('Content-Disposition', 'attachment', filename=os.path.split(filename)[1]) + msg.attach(attachment) + except IOError: + logger.warning('[EMAIL] No such file {}, please check input parameter'.format(filename)) + return False try: server.login(user=username, password=password) From 306fbd490d156bc95563313b39802c4bdc983a0e Mon Sep 17 00:00:00 2001 From: BlBana <635373043@qq.com> Date: Tue, 27 Mar 2018 01:04:20 +0800 Subject: [PATCH 3/3] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E4=BA=86Python3=E4=B8=8B?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=E6=8B=89=E5=8F=96bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cobra/pickup.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/cobra/pickup.py b/cobra/pickup.py index d5422ba3..a602827c 100644 --- a/cobra/pickup.py +++ b/cobra/pickup.py @@ -387,6 +387,10 @@ def clone(self): p = subprocess.Popen(cmd, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) (clone_out, clone_err) = p.communicate() + + clone_out = clone_out.decode('utf-8') + clone_err = clone_err.decode('utf-8') + clone_err = clone_err.replace('{0}:{1}'.format(self.repo_username, self.repo_password), '') logger.debug('[PICKUP] [CLONE] ' + clone_out.strip()) @@ -421,6 +425,10 @@ def diff(self, new_version, old_version, raw_output=False): cmd = 'git diff ' + old_version + ' ' + new_version p = subprocess.Popen(cmd, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) (diff_out, diff_err) = p.communicate() + + diff_out = diff_out.decode('utf-8') + diff_err = diff_err.decode('utf-8') + logger.info(diff_out) # change the work directory back. @@ -448,6 +456,10 @@ def checkout(self, branch): cmd = "git fetch origin && git checkout " + branch p = subprocess.Popen(cmd, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) (checkout_out, checkout_err) = p.communicate() + + checkout_out = checkout_out.decode('utf-8') + checkout_err = checkout_err.decode('utf-8') + logger.info('[PICKUP] [CHECKOUT] ' + checkout_err.strip()) # Already on @@ -572,6 +584,10 @@ def __init__(self, filename, current_version=None, online_version=None): ) p = subprocess.Popen(cmd, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) (diff_out, diff_err) = p.communicate() + + diff_out = diff_out.decode('utf-8') + diff_err = diff_err.decode('utf-8') + if len(diff_err) == 0: logger.debug("[PICKUP] svn diff success") elif 'authorization failed' in diff_err: @@ -589,6 +605,8 @@ def log(self): ) p = subprocess.Popen(cmd, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) log_out = p.communicate()[0] + log_out = log_out.decode('utf-8') + return log_out def diff(self): @@ -601,6 +619,7 @@ def diff(self): ) p = subprocess.Popen(cmd, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) diff_out = p.communicate()[0] + diff_out = diff_out.decode('utf-8') added, removed, changed = [], [], [] diff = {}