From 760e19f37d0aca5e64f8cfbf6b2d6185316d9749 Mon Sep 17 00:00:00 2001 From: 40huo Date: Tue, 5 Sep 2017 09:59:34 +0800 Subject: [PATCH 01/24] fix #521 --- Dockerfile | 14 ++++++++++++++ cobra/api.py | 6 +++++- 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..3c9b6e7f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,14 @@ +FROM ubuntu:xenial + +COPY . /code/ +WORKDIR /code + +RUN apt-get update && apt-get install -y python-pip curl \ + && apt-get autoremove \ + && apt-get clean \ + && apt-get autoclean \ + && pip install -r requirements.txt \ + && cp config.template config + +EXPOSE 5000 +CMD ["python", "cobra.py", "-H", "0.0.0.0", "-P", "5000"] diff --git a/cobra/api.py b/cobra/api.py index ddb24d3a..b5075a16 100644 --- a/cobra/api.py +++ b/cobra/api.py @@ -42,6 +42,7 @@ q = queue.Queue() app = Flask(__name__, static_folder='templates/asset') +running_port = 5000 def producer(task): @@ -309,7 +310,8 @@ def summary(): return render_template(template_name_or_list='index.html', key=key) - status_url = request.url_root + 'api/status' + status_url = 'http://127.0.0.1:{port}/api/status'.format(port=running_port) + logger.critical(status_url) post_data = { 'key': key, 'sid': a_sid, @@ -481,6 +483,8 @@ def start(host, port, debug): i.start() try: + global running_port + running_port = port app.run(debug=debug, host=host, port=int(port), threaded=True, processes=1) except socket.error as v: if v.errno == errno.EACCES: From 1fa751dcd9a17400031246707633bbe3eae7d385 Mon Sep 17 00:00:00 2001 From: 40huo Date: Tue, 5 Sep 2017 15:06:28 +0800 Subject: [PATCH 02/24] add running host judge --- cobra/api.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cobra/api.py b/cobra/api.py index b5075a16..6e7ed5cb 100644 --- a/cobra/api.py +++ b/cobra/api.py @@ -42,6 +42,7 @@ q = queue.Queue() app = Flask(__name__, static_folder='templates/asset') +running_host = '0.0.0.0' running_port = 5000 @@ -310,7 +311,7 @@ def summary(): return render_template(template_name_or_list='index.html', key=key) - status_url = 'http://127.0.0.1:{port}/api/status'.format(port=running_port) + status_url = 'http://{host}:{port}/api/status'.format(host=running_host, port=running_port) logger.critical(status_url) post_data = { 'key': key, @@ -483,7 +484,8 @@ def start(host, port, debug): i.start() try: - global running_port + global running_port, running_host + running_host = host if host != '0.0.0.0' else '127.0.0.1' running_port = port app.run(debug=debug, host=host, port=int(port), threaded=True, processes=1) except socket.error as v: From 06702183c0cda07cf31b0452de72f0c3d94c562f Mon Sep 17 00:00:00 2001 From: PSoul Date: Tue, 5 Sep 2017 15:45:35 +0800 Subject: [PATCH 03/24] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E4=B8=8A=E4=BC=A0?= =?UTF-8?q?=E9=9D=9E=E6=94=AF=E6=8C=81=E7=9A=84=E5=90=8E=E7=BC=80BUG?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修复上传非支持的后缀BUG --- cobra/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cobra/api.py b/cobra/api.py index ddb24d3a..2dc8a212 100644 --- a/cobra/api.py +++ b/cobra/api.py @@ -204,7 +204,7 @@ def post(): code, result = 1001, {'sid': a_sid} return {'code': code, 'result': result} else: - return {'code': 1002, 'msg': "This extension can't support!"} + return {'code': 1002, 'result': "This extension can't support!"} class ResultData(Resource): From d9ebaf546e07759141d21697c0b9bc67cdf541a6 Mon Sep 17 00:00:00 2001 From: PSoul Date: Tue, 5 Sep 2017 15:46:37 +0800 Subject: [PATCH 04/24] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E4=B8=8A=E4=BC=A0?= =?UTF-8?q?=E9=9D=9E=E6=94=AF=E6=8C=81=E7=9A=84=E5=90=8E=E7=BC=80BUG?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修复上传非支持的后缀BUG --- cobra/templates/index.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cobra/templates/index.html b/cobra/templates/index.html index 3eb311fd..721d7993 100644 --- a/cobra/templates/index.html +++ b/cobra/templates/index.html @@ -261,6 +261,7 @@

Drag and drop upload

get_status(); } else { alert(result.result.result); + location.reload(); } }, start: function (e) { @@ -283,4 +284,4 @@

Drag and drop upload

- \ No newline at end of file + From 402f6fb9410c5878f6177d8ed12166f273eab6d0 Mon Sep 17 00:00:00 2001 From: lightless Date: Tue, 5 Sep 2017 16:19:04 +0800 Subject: [PATCH 05/24] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=9C=A8virtualenv?= =?UTF-8?q?=E7=8E=AF=E5=A2=83=E4=B8=8B=E6=97=A0=E6=B3=95=E9=80=9A=E8=BF=87?= =?UTF-8?q?./cobra.py=E6=89=A7=E8=A1=8C=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 修改首行为`#!/usr/bin/env python` - 格式化代码样式，使其符合PEP8 --- cobra.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cobra.py b/cobra.py index cccb2168..aa7f1ca6 100755 --- a/cobra.py +++ b/cobra.py @@ -1,11 +1,13 @@ -#!/usr/bin/python - +#!/usr/bin/env python # -*- coding: utf-8 -*- + import re import sys from cobra import main + if __name__ == '__main__': sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) sys.exit(main()) + From 345e37d086a57f3a55a0c96299e8c8a58c588a72 Mon Sep 17 00:00:00 2001 From: Feei Date: Tue, 5 Sep 2017 16:49:23 +0800 Subject: [PATCH 06/24] add @Bre4k --- docs/contributors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/contributors.md b/docs/contributors.md index 6d10ecb7..82dd1970 100644 --- a/docs/contributors.md +++ b/docs/contributors.md @@ -9,6 +9,7 @@ | [LiGhT1EsS](https://github.com/LiGhT1EsS) | [https://lightless.me](https://lightless.me) | root@lightless.me | | [40huo](https://github.com/40huo) | [https://www.40huo.cn](https://www.40huo.cn) | i@40huo.cn | | [BlBana](https://github.com/BlBana) | [http://drops.blbana.cc](http://drops.blbana.cc) | 635373043@qq.com | +| [Bre4k](https://github.com/bk7477890) | [http://bre4k.cn](http://bre4k.cn) | bk7477890@outlook.com | ## 主要贡献者 braveghz、M2shad0w、alioth310、l4yn3、boltomli、JoyChou93 From 8ecb923f3225a31e4704d11963521bf0c3c628d1 Mon Sep 17 00:00:00 2001 From: BlBana <635373043@qq.com> Date: Tue, 5 Sep 2017 16:51:11 +0800 Subject: [PATCH 07/24] Added cve vulnerability in API scan display --- cobra/cve_parse.py | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/cobra/cve_parse.py b/cobra/cve_parse.py index e43c5419..ebc3690d 100644 --- a/cobra/cve_parse.py +++ b/cobra/cve_parse.py @@ -241,9 +241,9 @@ def log_result(self): for cve_child in self._scan_result[module_]: cve_id = cve_child level = self._scan_result[module_][cve_id] - logger.warning('Find the module ' + module_ + ' have ' + cve_id + ',level: ' + level) + logger.debug('Find the module ' + module_ + ' have ' + cve_id + ',level: ' + level) count = len(self._scan_result[module_]) - logger.warning('The ' + module_ + ' module have ' + str(count) + ' CVE Vul(s)') + logger.debug('The ' + module_ + ' module have ' + str(count) + ' CVE Vul(s)') def get_scan_result(self): return self._scan_result @@ -337,7 +337,7 @@ def store(results): for module_ in results[0]: for cve_id, cve_level in results[0][module_].items(): cve_path = results[1] - cve_vul = parse_math(cve_path, cve_id, cve_level, module_) + cve_vul = parse_math(cve_path, cve_id, cve_level, module_, target_directory) cve_vuls.append(cve_vul) else: logger.debug('[SCAN] [STORE] Not found vulnerabilities on this rule!') @@ -371,7 +371,9 @@ def scan_single(target_directory, cve_path): return cve.get_scan_result(), cve_path -def parse_math(cve_path, cve_id, cve_level, module_): +def parse_math(cve_path, cve_id, cve_level, module_, target_directory): + flag = 0 + file_path = 'unkown' mr = VulnerabilityResult() module_name, module_version = module_.split(':') cvi = cve_path.lower().split('cvi-')[1][:6] @@ -385,13 +387,36 @@ def parse_math(cve_path, cve_id, cve_level, module_): elif cve_level == 'HIGH': cve_level = 8 + for root, dirs, filenames in os.walk(target_directory): + for filename in filenames: + if filename == 'pom.xml' and flag != 2: + file_path = os.path.join(root, filename) + file_path = file_path.replace(target_directory, '') + flag = 1 + + elif filename == 'requirements.txt' and flag != 1: + file_path = os.path.join(root, filename) + file_path = file_path.replace(target_directory, '') + flag = 2 + + if flag != 0: + mr.file_path = file_path + + else: + mr.file_path = 'unkown' mr.language = cve_id mr.id = cvi mr.rule_name = rule_name mr.level = cve_level - mr.file_path = module_name mr.line_number = 1 mr.code_content = module_name + ':' + module_version + mr.solution = """ + 三方依赖**""" + module_name + """:""" + module_version + """**存在CVE漏洞，CVE漏洞编号为: **""" + cve_id + """** + ## 安全风险 + + ## 安全修复 + 请根据对应厂商公告，及时更新三方依赖至安全版本 + """ logger.debug('[CVE {i}] {r}:Find {n}:{v} have vul {c} and level is {l}'.format(i=mr.id, r=mr.rule_name, n=mr.file_path, v=mr.line_number, From a211ab7b47eeccfce6f14dbe86d7922050bdb50f Mon Sep 17 00:00:00 2001 From: sarleon Date: Tue, 5 Sep 2017 16:46:28 +0800 Subject: [PATCH 08/24] fix the problem : path of the grep and find command (in some distro these utils are in /bin/ while others in /usr/binor /usr/local/bin --- cobra/utils.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/cobra/utils.py b/cobra/utils.py index d932e9e2..61ae2159 100644 --- a/cobra/utils.py +++ b/cobra/utils.py @@ -395,10 +395,28 @@ def get_safe_ex_string(ex, encoding=None): class Tool: def __init__(self): + # `grep` (`ggrep` on Mac) - self.grep = '/bin/grep' + if os.path.isfile('/bin/grep'): + self.grep = '/bin/grep' + elif os.path.isfile('/usr/bin/grep'): + self.grep = '/usr/bin/grep' + elif os.path.isfile('/usr/local/bin/grep'): + self.grep='/usr/local/bin/grep' + else: + self.grep = 'grep' + + # `find` (`gfind` on Mac) - self.find = '/bin/find' + if os.path.isfile('/bin/find'): + self.find = '/bin/find' + elif os.path.isfile('/usr/bin/find'): + self.find = '/usr/bin/find' + elif os.path.isfile('/usr/local/bin/find'): + self.find='/usr/local/bin/find' + else: + self.find = 'find' + if 'darwin' == sys.platform: ggrep = '' From b0c8b5abe20790c1c47b4ab85c98ad2a99ec3ec5 Mon Sep 17 00:00:00 2001 From: 40huo Date: Tue, 5 Sep 2017 17:41:34 +0800 Subject: [PATCH 09/24] fix #527 --- tests/test_apiserver.py | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/tests/test_apiserver.py b/tests/test_apiserver.py index 340ecaf0..33b71236 100644 --- a/tests/test_apiserver.py +++ b/tests/test_apiserver.py @@ -35,7 +35,7 @@ def test_add_job(): url = "http://127.0.0.1:5000/api/add" post_data = { "key": "your_secret_key", - "target": ["tests/vulnerabilities"], + "target": [os.path.join(project_directory, 'tests/vulnerabilities')] } headers = { "Content-Type": "application/json", @@ -106,22 +106,11 @@ def test_close_api(): p.join() # wait for scan process - while True: - cobra_process = os.popen('ps aux | grep python').read() - cobra_process_num = len(cobra_process.strip().split('\n')) - if cobra_process_num <= 3: - # grep python - # sh -c ps aux | grep python - # python pytest - break - time.sleep(1) - - # whether port 5000 is closed s = socket.socket() s.settimeout(0.5) - try: - assert s.connect_ex(('localhost', 5000)) != 0 - finally: - s.close() + while s.connect_ex(('localhost', 5000)) == 0: + time.sleep(1) + # whether port 5000 is closed + assert s.connect_ex(('localhost', 5000)) != 0 assert not os.path.exists(config_path) From 33e4a403058c0730bbef94a9e7b5dac99349961d Mon Sep 17 00:00:00 2001 From: Feei Date: Tue, 5 Sep 2017 17:51:35 +0800 Subject: [PATCH 10/24] add CONTRIBUTING link --- CONTRIBUTING.md | 5 +++-- docs/index.md | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 46dc534c..b013ce90 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,8 +5,9 @@ 仔细描述问题的复现步骤，并提供对应的运行环境信息（Python版本、系统版本） ## 提交代码 -- Fork项目，切换到`develop`分支开发，或新建分支`feature-xxx` +- Fork项目，切换到`develop`分支开发 - 按照PEP8格式 - 所有代码都需要有对应的单元测试用例 - 运行所有测试用例 -- 提交Pull Request \ No newline at end of file +- 提交Pull Request到`develop`分支 +- 等待测试稳定后合并到`master`分支 \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index dd793f80..2b23ce2b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -91,4 +91,5 @@ - [程序目录结构](https://wufeifei.github.io/cobra/tree) - 贡献代码 - [单元测试](https://wufeifei.github.io/cobra/test) - - [贡献者](https://wufeifei.github.io/cobra/contributors) \ No newline at end of file + - [贡献者](https://wufeifei.github.io/cobra/contributors) + - [如何贡献代码？](https://github.com/wufeifei/cobra/blob/master/CONTRIBUTING.md) \ No newline at end of file From 316bef330c0770739e95f9c1108e07697655d27e Mon Sep 17 00:00:00 2001 From: Feei Date: Tue, 5 Sep 2017 18:18:44 +0800 Subject: [PATCH 11/24] fix when multi python version bug --- cobra/__version__.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cobra/__version__.py b/cobra/__version__.py index 1b27a50b..1df6f919 100644 --- a/cobra/__version__.py +++ b/cobra/__version__.py @@ -22,10 +22,10 @@ Cobra is a static code analysis system that automates the detecting vulnerabilities and security issue.""".format(version=__version__) __epilog__ = """Usage: - {m} -t {td} - {m} -t {td} -r cvi-190001,cvi-190002 - {m} -t {td} -f json -o /tmp/report.json - {m} -t {tg} -f json -o feei@feei.cn - {m} -t {tg} -f json -o http://push.to.com/api - sudo {m} -H 127.0.0.1 -P 80 -""".format(m='./cobra.py', td='tests/vulnerabilities', tg='https://github.com/ethicalhack3r/DVWA') + python {m} -t {td} + python {m} -t {td} -r cvi-190001,cvi-190002 + python {m} -t {td} -f json -o /tmp/report.json + python {m} -t {tg} -f json -o feei@feei.cn + python {m} -t {tg} -f json -o http://push.to.com/api + sudo python {m} -H 127.0.0.1 -P 80 +""".format(m='cobra.py', td='tests/vulnerabilities', tg='https://github.com/ethicalhack3r/DVWA') From f294aed0f76140d36c53bf9a66404216d2dd17ef Mon Sep 17 00:00:00 2001 From: Feei Date: Tue, 5 Sep 2017 18:24:09 +0800 Subject: [PATCH 12/24] fix when multi python version bug --- docs/api.md | 2 +- docs/cli.md | 32 ++++++++++++++++---------------- docs/installation.md | 2 +- docs/rule_flow.md | 2 +- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/docs/api.md b/docs/api.md index 273f9f50..5a059a67 100644 --- a/docs/api.md +++ b/docs/api.md @@ -55,7 +55,7 @@ # 完整的例子 ## 启动HTTP服务 ```bash -sudo ./cobra.py -H 127.0.0.1 -P 80 +sudo python cobra.py -H 127.0.0.1 -P 80 ``` ## 添加扫描任务 diff --git a/docs/cli.md b/docs/cli.md index 6f5aec2a..60871a92 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -3,36 +3,36 @@ ## Examples（使用例子） ```bash # 扫描一个文件夹的代码 -$ ./cobra.py -t tests/vulnerabilities +$ python cobra.py -t tests/vulnerabilities # 扫描一个Git项目代码 -$ ./cobra.py -t https://github.com/wufeifei/grw.git +$ python cobra.py -t https://github.com/wufeifei/grw.git # 扫描一个文件夹，并将扫描结果导出为JSON文件 -$ ./cobra.py -t tests/vulnerabilities -f json -o /tmp/report.json +$ python cobra.py -t tests/vulnerabilities -f json -o /tmp/report.json # 扫描一个Git项目，并将扫描结果JSON文件推送到API上 -$ ./cobra.py -f json -o http://push.to.com/api -t https://github.com/wufeifei/vc.git +$ python cobra.py -f json -o http://push.to.com/api -t https://github.com/wufeifei/vc.git # 扫描一个Git项目，并将扫描结果JSON文件发送到邮箱中 -$ ./cobra.py -f json -o feei@feei.cn -t https://github.com/wufeifei/vc.git +$ python cobra.py -f json -o feei@feei.cn -t https://github.com/wufeifei/vc.git # 扫描一个文件夹代码的某两种漏洞 -$ ./cobra.py -t tests/vulnerabilities -r cvi-190001,cvi-190002 +$ python cobra.py -t tests/vulnerabilities -r cvi-190001,cvi-190002 # 开启一个Cobra HTTP Server，然后可以使用API接口来添加扫描任务 -$ ./cobra.py -H 127.0.0.1 -P 80 +$ python cobra.py -H 127.0.0.1 -P 80 # 查看版本 -$ ./cobra.py --version +$ python cobra.py --version # 查看帮助 -$ ./cobra.py --help +$ python cobra.py --help ``` ## Help（帮助） ```bash -➜ cobra git:(master) ✗ ./cobra.py --help +➜ cobra git:(master) ✗ python cobra.py --help usage: cobra [-h] [-t ] [-f ] [-o ] [-r ] [-d] [-sid SID] [-H ] [-P ] @@ -68,12 +68,12 @@ RESTful: REST-JSON API Service Port Usage: - ./cobra.py -t tests/vulnerabilities - ./cobra.py -t tests/vulnerabilities -r cvi-190001,cvi-190002 - ./cobra.py -t tests/vulnerabilities -f json -o /tmp/report.json - ./cobra.py -t https://github.com/wufeifei/vc.git -f json -o feei@feei.cn - ./cobra.py -t https://github.com/wufeifei/vc.git -f json -o http://push.to.com/api - sudo ./cobra.py -H 127.0.0.1 -P 80 + python cobra.py -t tests/vulnerabilities + python cobra.py -t tests/vulnerabilities -r cvi-190001,cvi-190002 + python cobra.py -t tests/vulnerabilities -f json -o /tmp/report.json + python cobra.py -t https://github.com/ethicalhack3r/DVWA -f json -o feei@feei.cn + python cobra.py -t https://github.com/ethicalhack3r/DVWA -f json -o http://push.to.com/api + sudo python cobra.py -H 127.0.0.1 -P 80 ``` --- 下一章：[API模式使用方法](https://wufeifei.github.io/cobra/api) \ No newline at end of file diff --git a/docs/installation.md b/docs/installation.md index 7a164a96..54bee770 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -29,7 +29,7 @@ brew install grep findutils ```bash git clone https://github.com/wufeifei/cobra.git && cd cobra pip install -r requirements.txt -./cobra.py --help +python cobra.py --help ``` --- diff --git a/docs/rule_flow.md b/docs/rule_flow.md index 45ad3a7c..c32066c8 100644 --- a/docs/rule_flow.md +++ b/docs/rule_flow.md @@ -7,7 +7,7 @@ #### 2. 编写漏洞代码`tests/vulnerabilities/v.language` 编写实际可能出现的业务场景代码（只需编写一处即可）。 -#### 3. 测试规则扫描`./cobra.py -t tests/vulnerabilities/` +#### 3. 测试规则扫描`python cobra.py -t tests/vulnerabilities/` 测试扫描结果 --- From e9f6b0b2a843c52d911aca81ccc287e7299080b1 Mon Sep 17 00:00:00 2001 From: cclauss Date: Tue, 5 Sep 2017 12:25:46 +0200 Subject: [PATCH 13/24] Use feature detection instead of version detection The best practice for code that runs on both Python 2 and Python 3 is to use feature detection instead of version detection. https://docs.python.org/3/howto/pyporting.html#use-feature-detection-instead-of-version-detection --- cobra/utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cobra/utils.py b/cobra/utils.py index d932e9e2..d4e6d51d 100644 --- a/cobra/utils.py +++ b/cobra/utils.py @@ -427,10 +427,10 @@ def secure_filename(filename): _filename_utf8_strip_re = re.compile(u"[^\u4e00-\u9fa5A-Za-z0-9_.\-\+]") _windows_device_files = ('CON', 'AUX', 'COM1', 'COM2', 'COM3', 'COM4', 'LPT1', 'LPT2', 'LPT3', 'PRN', 'NUL') - if PY2: - text_type = unicode - else: - text_type = str + try: + text_type = unicode # Python 2 + except NameError: + text_type = str # Python 3 if isinstance(filename, text_type): from unicodedata import normalize From 7118a39d716ba62630177129761cb24c25dc7b3a Mon Sep 17 00:00:00 2001 From: Feei Date: Tue, 5 Sep 2017 18:57:31 +0800 Subject: [PATCH 14/24] status to off --- rules/CVI-160002.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules/CVI-160002.xml b/rules/CVI-160002.xml index 6faab099..2afb0fb7 100644 --- a/rules/CVI-160002.xml +++ b/rules/CVI-160002.xml @@ -51,6 +51,6 @@ 尽量不要使用拼接的SQL语句。若不得不使用，尽量不要用户可控SQL语句的参数。 若必须用户可控，请对用户输入的参数进行严格的限制和过滤。 - + From af1edbc79ed74ef7b56a47b1636b8f8eeaff27bf Mon Sep 17 00:00:00 2001 From: 40huo Date: Tue, 5 Sep 2017 19:00:05 +0800 Subject: [PATCH 15/24] =?UTF-8?q?=E6=A3=80=E6=B5=8B=E6=89=AB=E6=8F=8F?= =?UTF-8?q?=E7=BB=93=E6=9E=9C=E6=96=87=E4=BB=B6=E5=88=A4=E6=96=AD=E8=BF=9B?= =?UTF-8?q?=E7=A8=8B=E6=98=AF=E5=90=A6=E7=BB=93=E6=9D=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/test_apiserver.py | 58 ++++++++++++++++++++++++++++++----------- 1 file changed, 43 insertions(+), 15 deletions(-) diff --git a/tests/test_apiserver.py b/tests/test_apiserver.py index 33b71236..5c571c50 100644 --- a/tests/test_apiserver.py +++ b/tests/test_apiserver.py @@ -19,7 +19,7 @@ import os import shutil import socket -from cobra.config import project_directory +from cobra.config import project_directory, running_path from cobra.api import start p = multiprocessing.Process(target=start, args=('127.0.0.1', 5000, False)) @@ -30,6 +30,9 @@ template_path = os.path.join(project_directory, 'config.template') shutil.copyfile(template_path, config_path) +a_sid = '' +s_sid = '' + def test_add_job(): url = "http://127.0.0.1:5000/api/add" @@ -41,6 +44,17 @@ def test_add_job(): "Content-Type": "application/json", } re = requests.post(url=url, data=json.dumps(post_data), headers=headers) + result = json.loads(re.text) + + global a_sid + a_sid = result.get('result').get('sid') + + a_sid_file = os.path.join(running_path, '{sid}_list'.format(sid=a_sid)) + with open(a_sid_file, 'r') as f: + scan_list = json.load(f) + global s_sid + s_sid = scan_list.get('sids').keys()[0] + assert "1001" in re.text assert "Add scan job successfully" in re.text assert "sid" in re.text @@ -50,15 +64,15 @@ def test_job_status(): url = "http://127.0.0.1:5000/api/status" post_data = { "key": "your_secret_key", - "sid": 24, + "sid": a_sid, } headers = { "Content-Type": "application/json", } re = requests.post(url=url, data=json.dumps(post_data), headers=headers) - assert "1004" in re.text + assert "1001" in re.text assert "msg" in re.text - assert "sid" in re.text + assert a_sid in re.text assert "status" in re.text assert "report" in re.text @@ -66,28 +80,41 @@ def test_job_status(): def test_result_data(): url = 'http://127.0.0.1:5000/api/list' post_data = { - 'sid': 24, + 'sid': s_sid, } headers = { "Content-Type": "application/json", } re = requests.post(url=url, data=json.dumps(post_data), headers=headers) - assert '1002' in re.text - assert 'No such target' in re.text + + s_sid_file = os.path.join(running_path, '{sid}_data'.format(sid=s_sid)) + if os.path.exists(s_sid_file): + assert '1001' in re.text + assert 'result' in re.text + assert 'rule_filter' in re.text + else: + assert '1002' in re.text + assert 'No such target' in re.text def test_result_detail(): url = 'http://127.0.0.1:5000/api/detail' post_data = { - 'sid': 'abcdeft', - 'file_path': 'setup.py', + 'sid': s_sid, + 'file_path': 'v.php', } headers = { "Content-Type": "application/json", } re = requests.post(url=url, data=json.dumps(post_data), headers=headers) - assert '1002' in re.text - assert 'No such target' in re.text + + s_sid_file = os.path.join(running_path, '{sid}_data'.format(sid=s_sid)) + if os.path.exists(s_sid_file): + assert '1001' in re.text + assert 'file_content' in re.text + else: + assert '1002' in re.text + assert 'No such target' in re.text def test_index(): @@ -105,12 +132,13 @@ def test_close_api(): p.terminate() p.join() - # wait for scan process - s = socket.socket() - s.settimeout(0.5) - while s.connect_ex(('localhost', 5000)) == 0: + # wait for scan data + s_sid_file = os.path.join(running_path, '{sid}_data'.format(sid=s_sid)) + while not os.path.exists(s_sid_file): time.sleep(1) # whether port 5000 is closed + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(0.5) assert s.connect_ex(('localhost', 5000)) != 0 assert not os.path.exists(config_path) From b5ff04d8507430689544f49740ce6e999d8430f0 Mon Sep 17 00:00:00 2001 From: 40huo Date: Tue, 5 Sep 2017 19:40:05 +0800 Subject: [PATCH 16/24] =?UTF-8?q?Python=203=20=E5=86=99=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E8=BE=83=E6=85=A2=EF=BC=8C=E7=AD=89=E5=BE=85=E5=86=99=E5=85=A5?= =?UTF-8?q?asid=5Flist=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/test_apiserver.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/tests/test_apiserver.py b/tests/test_apiserver.py index 5c571c50..366f9de5 100644 --- a/tests/test_apiserver.py +++ b/tests/test_apiserver.py @@ -12,15 +12,17 @@ :copyright: Copyright (c) 2017 Feei. All rights reserved """ -import requests import json import multiprocessing -import time import os import shutil import socket -from cobra.config import project_directory, running_path +import time + +import requests + from cobra.api import start +from cobra.config import project_directory, running_path p = multiprocessing.Process(target=start, args=('127.0.0.1', 5000, False)) p.start() @@ -50,10 +52,18 @@ def test_add_job(): a_sid = result.get('result').get('sid') a_sid_file = os.path.join(running_path, '{sid}_list'.format(sid=a_sid)) - with open(a_sid_file, 'r') as f: - scan_list = json.load(f) + + # wait writing scan_list + while True: + with open(a_sid_file, 'r') as f: + scan_list = json.load(f) + print(scan_list) + if len(scan_list.get('sids')) > 0: + break + time.sleep(0.1) + global s_sid - s_sid = scan_list.get('sids').keys()[0] + s_sid = list(scan_list.get('sids').keys())[0] assert "1001" in re.text assert "Add scan job successfully" in re.text From 8f540454644655cf6551c680fccd24600ae91146 Mon Sep 17 00:00:00 2001 From: 40huo Date: Tue, 5 Sep 2017 19:43:48 +0800 Subject: [PATCH 17/24] remove port check in test API --- tests/test_apiserver.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/test_apiserver.py b/tests/test_apiserver.py index 366f9de5..0deaa9cd 100644 --- a/tests/test_apiserver.py +++ b/tests/test_apiserver.py @@ -147,8 +147,4 @@ def test_close_api(): while not os.path.exists(s_sid_file): time.sleep(1) - # whether port 5000 is closed - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.settimeout(0.5) - assert s.connect_ex(('localhost', 5000)) != 0 assert not os.path.exists(config_path) From 2344c67d275ba52773498d290fb9f3d9ca978be0 Mon Sep 17 00:00:00 2001 From: 40huo Date: Tue, 5 Sep 2017 19:52:16 +0800 Subject: [PATCH 18/24] =?UTF-8?q?=E7=AD=89=E5=BE=85=E7=AB=AF=E5=8F=A3?= =?UTF-8?q?=E5=85=B3=E9=97=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/test_apiserver.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_apiserver.py b/tests/test_apiserver.py index 0deaa9cd..9d5cd0c7 100644 --- a/tests/test_apiserver.py +++ b/tests/test_apiserver.py @@ -147,4 +147,10 @@ def test_close_api(): while not os.path.exists(s_sid_file): time.sleep(1) + # wait for port closed + s = socket.socket() + s.settimeout(0.5) + while s.connect_ex(('localhost', 5000)) == 0: + time.sleep(0.5) + assert not os.path.exists(config_path) From 518f000da1eb908f3d660ef0423d47a2c9571cc5 Mon Sep 17 00:00:00 2001 From: Feei Date: Wed, 6 Sep 2017 10:19:31 +0800 Subject: [PATCH 19/24] upgrade blog url --- docs/contributors.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributors.md b/docs/contributors.md index 82dd1970..b70d84dc 100644 --- a/docs/contributors.md +++ b/docs/contributors.md @@ -9,7 +9,7 @@ | [LiGhT1EsS](https://github.com/LiGhT1EsS) | [https://lightless.me](https://lightless.me) | root@lightless.me | | [40huo](https://github.com/40huo) | [https://www.40huo.cn](https://www.40huo.cn) | i@40huo.cn | | [BlBana](https://github.com/BlBana) | [http://drops.blbana.cc](http://drops.blbana.cc) | 635373043@qq.com | -| [Bre4k](https://github.com/bk7477890) | [http://bre4k.cn](http://bre4k.cn) | bk7477890@outlook.com | +| [Bre4k](https://github.com/bk7477890) | http://bre4k.cn | bk7477890@outlook.com | ## 主要贡献者 braveghz、M2shad0w、alioth310、l4yn3、boltomli、JoyChou93 From 23d036fe4b3a281d5160e990819dcbb1f8fee5d5 Mon Sep 17 00:00:00 2001 From: Feei Date: Wed, 6 Sep 2017 12:33:08 +0800 Subject: [PATCH 20/24] improves codes --- cobra/cve.py | 426 ++++++++++++++++++++++++++++++++++++++ cobra/dependencies.py | 6 +- cobra/engine.py | 2 +- cobra/export.py | 2 +- cobra/git_projects.py | 7 +- cobra/parser.py | 6 +- cobra/scheduler/report.js | 119 ----------- cobra/scheduler/report.py | 143 ------------- cobra/scheduler/scan.py | 45 ---- cobra/templite.py | 2 +- tests/test_cve_parse.py | 4 +- 11 files changed, 441 insertions(+), 321 deletions(-) create mode 100644 cobra/cve.py delete mode 100644 cobra/scheduler/report.js delete mode 100644 cobra/scheduler/report.py delete mode 100644 cobra/scheduler/scan.py diff --git a/cobra/cve.py b/cobra/cve.py new file mode 100644 index 00000000..af37edfd --- /dev/null +++ b/cobra/cve.py @@ -0,0 +1,426 @@ +# -*- coding: utf-8 -*- + +""" + CVE + ~~~ + + Implements CVE Rules Parser + + :author: BlBana <635373043@qq.com> + :homepage: https://github.com/wufeifei/cobra + :license: MIT, see LICENSE for more details. + :copyright: Copyright (c) 2017 Feei. All rights reserved +""" +import datetime +import os +import requests +import threading +import gzip +import xml.etree.cElementTree as eT +import multiprocessing +from .config import project_directory, Config, config_path +from .log import logger +from .dependencies import Dependencies +from .result import VulnerabilityResult + +try: + from urllib import urlretrieve # Python2 +except ImportError: + from urllib.request import urlretrieve # Python3 + +try: + from configparser import ConfigParser +except ImportError: + from ConfigParser import ConfigParser + + +class CveParse(object): + def __init__(self, target_file, project_path, year=None): + """ + :param target_file: The cve_file's path + """ + self.cve_file = target_file + self.pro_file = project_path + self.year = year + self._result = {} # {'cve_id':{'access-complexity':xxx, 'cpe':[]}} access-complexity and cpe may be None + self._rule = {} + self._scan_result = {} + self.res = True + self.CVSS = "{http://scap.nist.gov/schema/cvss-v2/0.2}" + self.VULN = "{http://scap.nist.gov/schema/vulnerability/0.4}" + self.NS = "{http://scap.nist.gov/schema/feed/vulnerability/2.0}" + + def cve_parse(self): + """ + Resolve the latest rules,parse new rule from cve.xml + :return: None + """ + cve_file = self.get_cve_file() + if not isinstance(cve_file, list): + tree = self.parse_xml(cve_file) + root = tree.getroot() + childs = root.findall('.//%sentry' % self.NS) + for child in childs: # child is entry Element + cve_id = child.attrib['id'] + cve_info = self.cve_info(child) + if len(cve_info) != 0: + self._result[cve_id] = cve_info + else: + for filename in cve_file: + tree = self.parse_xml(filename) + root = tree.getroot() + childs = root.findall('.//%sentry' % self.NS) + for child in childs: # child is entry Element + cve_id = child.attrib['id'] + cve_info = self.cve_info(child) + if len(cve_info) != 0: + self._result[cve_id] = cve_info + + def get_cve_file(self): + cve_file = [] + if os.path.isfile(self.cve_file): + return self.cve_file + else: + for root, dirs, filenames in os.walk(self.cve_file): + for filename in filenames: + cve_file.append(os.path.join(root, filename)) + return cve_file + + def cve_info(self, entry): + """ + :param entry: every entry Element + :return:Information inside each entry node + """ + cpe_list = [] + cve_info = {} + black_lists = ['ffmpeg', 'linux', 'ie', 'apple_tv', 'iphone_os', 'watchos', 'mac_os_x', 'windows', 'ios', + 'android', 'flash_player', 'office', 'wireshark', 'safari', 'mysql', 'word', '.net_framework', + 'samba', 'ntp', 'tomcat', 'unixware', 'vpn', 'netware', 'proxy_server', 'http_server', 'irix', + 'solaris', 'weblogic_server', 'kde', 'dhcpd', 'database_server', 'mandrake_linux', 'openssl' + 'suse_linux', + 'vim', 'debian_linux', 'putty', 'ubuntu_linux', 'mozilla', 'ftp_server', 'cvs' + 'oracle', + 'ws_ftp_server', 'surgemail', 'opera_web_browser', 'sql_server', 'ethereal', 'gaim', + 'wu-ftpd', 'cluster_server', 'catos', 'mantis', 'quicktime', 'security_linux', 'firefox', + 'jetty_http_server', 'php:', 'enterprise_linux', 'oracle10g', 'oracle9g', 'oracle8g', 'firehol', + 'fetchmail', 'postgresql', 'freebsd', 'chrome'] + products = entry.findall('.//%sproduct' % self.VULN) + access_complexity = entry.find('.//%saccess-complexity' % self.CVSS) + for product in products: + module_version = product.text.split(':') + if len(module_version) > 4: + module_ = module_version[3] + ':' + module_version[4] + elif len(module_version) == 4: + module_ = module_version[3] + else: + module_ = module_version[2] + for black_list in black_lists: + if str(module_).startswith(black_list): + cve_info = {} + return cve_info + cpe_list.append(module_) + if len(cpe_list) == 0: + return cve_info + else: + cve_info['cpe'] = cpe_list + + if access_complexity is None: + cve_info['level'] = 'unknown' + else: + cve_info['level'] = access_complexity.text + return cve_info + + @staticmethod + def parse_xml(file_path): + return eT.parse(file_path) + + def get_result(self): + """ + :return:The result from cve.xml,this is new rule + """ + return self._result + + def rule_xml(self): + """ + If you want to update rule, Please use this function, it will auto parse rule, and write in file + :return: + """ + starttime = datetime.datetime.now() + logger.info('The rule CVE-999' + str(self.year)[1:] + '.xml are being updated. Please wait for a moment....') + self.cve_parse() + cobra = eT.Element('cobra') # root Ele + cobra.set('document', 'https://github.com/wufeifei/cobra') + for cve_id in self._result.keys(): + cve_child = eT.Element('cve') # cve Ele + cve_child.set('id', cve_id) + cve_child.set('level', self._result[cve_id]['level']) + if 'cpe' in self._result[cve_id]: + for product_ in self._result[cve_id]['cpe']: + product = eT.Element('product') + product.text = product_ + cve_child.append(product) # product in products + cobra.append(cve_child) # cve in cobra + self.pretty(cobra) + tree = eT.ElementTree(cobra) + rule_path = project_directory + '/rules/CVI-999' + tree.write(rule_path + str(self.year)[1:] + '.xml') + endtime = datetime.datetime.now() + logger.info( + 'CVE-999' + str(self.year)[1:] + '.xml Rule update succeeds, times:%ds' % (endtime - starttime).seconds) + + def pretty(self, e, level=0): + """ + :param e:The root Element + :param level: + :return: None,pretty the xml file + """ + if len(e) > 0: + e.text = '\n' + '\t' * (level + 1) + for child in e: + self.pretty(child, level + 1) + child.tail = child.tail[:-1] + e.tail = '\n' + '\t' * level + + def rule_parse(self, file_): + """ + :return: rules from CVI-999999.xml and CVI-999999.xml + """ + tree = self.parse_xml(file_) + root = tree.getroot() + cves = root.findall('.//cve') + for cve_child in cves: + cve_id = cve_child.attrib['id'] + cve_level = cve_child.attrib['level'] + rule_info = self.rule_info(cve_child) + rule_info['level'] = cve_level + self._rule[cve_id] = rule_info + + @staticmethod + def rule_info(cve_child): + rule_info = {} + cpe_list = [] + products = cve_child.findall('.//product') + for product in products: + cpe_list.append(product.text.lower()) + rule_info['cpe'] = cpe_list + return rule_info + + def get_rule(self): + """ + :return: The rule from CVI-999999.xml and CVI-999999.xml + """ + return self._rule + + def scan_cve(self, file_): + """ + :return:Analytical dependency，Match the rules and get the result + """ + self.rule_parse(file_) + cve = self.get_rule() + dependency = Dependencies(self.pro_file) + project_info = dependency.get_result + for pro_info in project_info: + module_version = pro_info.lower() + ':' + project_info[pro_info] + self.set_scan_result(cve, module_version) + self.log_result() + + def set_scan_result(self, cves, module_version): + """ + :param cves: + :param module_version: + :return:set the scan result + """ + scan_cves = {} + for cve_child in cves: + if module_version in cves[cve_child]['cpe']: + scan_cves[cve_child] = cves[cve_child]['level'] + if len(scan_cves): + self._scan_result[module_version] = scan_cves + + def log_result(self): + for module_ in self._scan_result: + for cve_child in self._scan_result[module_]: + cve_id = cve_child + level = self._scan_result[module_][cve_id] + logger.debug('Find the module ' + module_ + ' have ' + cve_id + ',level: ' + level) + count = len(self._scan_result[module_]) + logger.debug('The ' + module_ + ' module have ' + str(count) + ' CVE Vul(s)') + + def get_scan_result(self): + return self._scan_result + + +def rule_parse(): + if is_update(): + gz_files = download_rule_gz() + un_gz(gz_files) + pool = multiprocessing.Pool() + for year in range(2002, datetime.datetime.now().year + 1): + cve_xml = project_directory + "/rules/%d.xml" % year + pool.apply_async(rule_single, args=(cve_xml, year)) + pool.close() + pool.join() + for year in range(2002, datetime.datetime.now().year + 1): + os.remove(project_directory + "/rules/%d.xml" % year) + logger.info("The rule update success, start scan cve vuls") + return True + else: + logger.info("The CVE Rule not update, start scan cve vuls") + + +def download_rule_gz(): + threads = [] + files = [] + start_time = datetime.datetime.now() + for year in range(2002, datetime.datetime.now().year + 1): + url = "https://static.nvd.nist.gov/feeds/xml/cve/2.0/nvdcve-2.0-" + str(year) + ".xml.gz" + logger.info("start download " + str(year) + ".xml.gz") + thread = threading.Thread(target=urlretrieve, + args=(url, project_directory + "/rules/" + str(year) + ".xml.gz")) + thread.start() + threads.append(thread) + files.append(project_directory + "/rules/" + str(year) + ".xml.gz") + for t in threads: + t.join() + end_time = datetime.datetime.now() + logger.info("All CVE xml file already download success, use time:%ds" % (end_time - start_time).seconds) + return files + + +def un_gz(gz_files): + """ungz zip file""" + start_time = datetime.datetime.now() + logger.info("Start decompress rule files, Please wait a moment....") + for gz_file in gz_files: + f_name = gz_file.replace(".gz", "") + g_file = gzip.GzipFile(gz_file) + open(f_name, "wb+").write(g_file.read()) + g_file.close() + os.remove(gz_file) + end_time = datetime.datetime.now() + logger.info("Decompress success, use time:%ds" % (end_time - start_time).seconds) + return True + + +def rule_single(target_directory, year): + CveParse(target_directory, '.', year).rule_xml() + + +def is_update(): + url = "https://static.nvd.nist.gov/feeds/xml/cve/2.0/nvdcve-2.0-modified.meta" + requests.packages.urllib3.disable_warnings() + r = requests.get(url, verify=False) + index = r.text.find('sha256:') + sha256_now = r.text[index + 7:].strip() + sha256_local = Config(level1='cve', level2='modified').value + if sha256_local != sha256_now: + logger.info("The CVE Rule already update, start update local rule") + config = ConfigParser() + config.read(config_path) + config.set('cve', 'modified', sha256_now) + try: + fi = open(config_path, 'w') + config.write(fi) + fi.close() + except IOError as e: + logger.warning(e) + logger.info("The sha256 been update") + return True + return False + + +def scan_cve(target_directory): + cve_vuls = [] + cve_files = [] + + def store(results): + if len(results[0]) != 0: + for module_ in results[0]: + for cve_id, cve_level in results[0][module_].items(): + cve_path = results[1] + cve_vul = parse_math(cve_path, cve_id, cve_level, module_, target_directory) + cve_vuls.append(cve_vul) + else: + logger.debug('[SCAN] [STORE] Not found vulnerabilities on this rule!') + + rule_path = os.path.join(project_directory, 'rules') + files = os.listdir(rule_path) + for cvi_file in files: + if cvi_file.startswith('CVI-999'): + cve_files.append(cvi_file) + if len(cve_files) == 0: + logger.info("Can't find the rules, please update rules") + return + pool = multiprocessing.Pool() + logger.info('[PUSH] {rc} CVE Rules'.format(rc=len(cve_files))) + for cve_file in cve_files: + cve_path = os.path.join(rule_path, cve_file) + pool.apply_async(scan_single, args=(target_directory, cve_path), callback=store) + pool.close() + pool.join() + return cve_vuls + + +def scan_single(target_directory, cve_path): + """ + :param target_directory: scan path + :param cve_path: CVI-999***.xml + :return: + """ + cve = CveParse('.', target_directory) + cve.scan_cve(cve_path) + return cve.get_scan_result(), cve_path + + +def parse_math(cve_path, cve_id, cve_level, module_, target_directory): + flag = 0 + file_path = 'unkown' + mr = VulnerabilityResult() + module_name, module_version = module_.split(':') + cvi = cve_path.lower().split('cvi-')[1][:6] + rule_name = '引用了存在漏洞的三方组件' + if cve_level == 'LOW': + cve_level = 2 + + elif cve_level == 'MEDIUM': + cve_level = 5 + + elif cve_level == 'HIGH': + cve_level = 8 + + for root, dirs, filenames in os.walk(target_directory): + for filename in filenames: + if filename == 'pom.xml' and flag != 2: + file_path = os.path.join(root, filename) + file_path = file_path.replace(target_directory, '') + flag = 1 + + elif filename == 'requirements.txt' and flag != 1: + file_path = os.path.join(root, filename) + file_path = file_path.replace(target_directory, '') + flag = 2 + + if flag != 0: + mr.file_path = file_path + + else: + mr.file_path = 'unkown' + mr.language = cve_id + mr.id = cvi + mr.rule_name = rule_name + mr.level = cve_level + mr.line_number = 1 + mr.code_content = module_name + ':' + module_version + mr.solution = """ + 三方依赖**""" + module_name + """:""" + module_version + """**存在CVE漏洞，CVE漏洞编号为: **""" + cve_id + """** + ## 安全风险 + + ## 安全修复 + 请根据对应厂商公告，及时更新三方依赖至安全版本 + """ + + logger.debug('[CVE {i}] {r}:Find {n}:{v} have vul {c} and level is {l}'.format(i=mr.id, r=mr.rule_name, + n=mr.file_path, v=mr.line_number, + c=mr.language, l=mr.level)) + + return mr diff --git a/cobra/dependencies.py b/cobra/dependencies.py index 53d65284..ed7ddb98 100644 --- a/cobra/dependencies.py +++ b/cobra/dependencies.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- """ - cobra - ~~~~~ + dependencies + ~~~~~~~~~~~~ - Implements cobra main + Implements Dependencies Check :author: BlBana <635373043@qq.com> :homepage: https://github.com/wufeifei/cobra diff --git a/cobra/engine.py b/cobra/engine.py index d09604e8..a02b67aa 100644 --- a/cobra/engine.py +++ b/cobra/engine.py @@ -26,7 +26,7 @@ from .result import VulnerabilityResult from .cast import CAST from .parser import scan_parser -from .cve_parse import scan_cve +from .cve import scan_cve from prettytable import PrettyTable diff --git a/cobra/export.py b/cobra/export.py index 2e7adc91..4648dd4e 100644 --- a/cobra/export.py +++ b/cobra/export.py @@ -2,7 +2,7 @@ """ export - ~~~~~~~~~~~~~~~~~~~ + ~~~~~~ Export scan result to files or console diff --git a/cobra/git_projects.py b/cobra/git_projects.py index b3c8b20c..8d0e0ff8 100644 --- a/cobra/git_projects.py +++ b/cobra/git_projects.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- + """ cobra ~~~~~ @@ -17,6 +17,7 @@ import threading from .log import logger from .config import code_path, Config + try: # Python 3 import queue @@ -36,7 +37,7 @@ def start(): result_path = code_path + '/result_sid' fi = open(result_path, 'w+') for i in range(int(pages)): - q_pages.put(i+1) + q_pages.put(i + 1) for i in range(10): thread = threading.Thread(target=get_git_urls, args=(url, private_token, cobra_ip, key, q_pages, fi)) @@ -74,7 +75,7 @@ def get_git_urls(url, private_token, cobra_ip, key, q_pages, fi): git_branch = data[j]['default_branch'] if git_branch is not None: - request_url = git_url+':'+git_branch + request_url = git_url + ':' + git_branch else: request_url = git_url diff --git a/cobra/parser.py b/cobra/parser.py index ccbeef9e..f3dc3db3 100644 --- a/cobra/parser.py +++ b/cobra/parser.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- """ - cobra - ~~~~~ + parser + ~~~~~~ - Implements cobra main + Implements Code Parser :author: BlBana <635373043@qq.com> :homepage: https://github.com/wufeifei/cobra diff --git a/cobra/scheduler/report.js b/cobra/scheduler/report.js deleted file mode 100644 index 1e6ee614..00000000 --- a/cobra/scheduler/report.js +++ /dev/null @@ -1,119 +0,0 @@ -/** - * Use PhantomJS to capture the report page - * - * @author Feei - * @homepage https://github.com/wufeifei/cobra - * @license MIT, see LICENSE for more details. - * @copyright Copyright (c) 2017 Feei. All rights reserved - */ -"use strict"; -/** - * http://phantomjs.org/api/webpage/ - * - * Web page api - */ -var page = require('webpage').create(); -/** - * http://phantomjs.org/api/fs/ - * - * file system api - */ -var fs = require('fs'); -/** - * http://phantomjs.org/api/system/ - * - * system - */ -var system = require('system'); - -/** - * filename generator helper - * @param viewport - * @param tt time type - * @returns {string} - */ -function getFileName(viewport, tt) { - var d = new Date(); - var date = [ - d.getUTCFullYear(), - d.getUTCMonth() + 1, - d.getUTCDate() - ]; - var time = [ - d.getHours() <= 9 ? '0' + d.getHours() : d.getHours(), - d.getMinutes() <= 9 ? '0' + d.getMinutes() : d.getMinutes(), - d.getSeconds() <= 9 ? '0' + d.getSeconds() : d.getSeconds(), - d.getMilliseconds() - ]; - var resolution = viewport.width + "x" + viewport.height; - return tt + '_' + date.join('-') + '_' + time.join('-') + "_" + resolution + '.png'; -} - - -/** - * Read the Cobra config - * @type {string} - */ -if (system.args.length < 3 || system.args[0] in ['w', 'm', 'q']) { - console.log('Usage: report.js = 4) { - var month = system.args[3]; - domain += '&month=' + month; -} -var file = 'reports/' + getFileName(page.viewportSize, tt); -/** - * Capture - */ -page.open(domain, function (status) { - if (status !== 'success') { - console.log('Critical: Unable to load the address!'); - phantom.exit(1); - } else { - /** - * Draw chart need time - */ - window.setTimeout(function () { - page.render(file); - console.log('Success: ' + file); - phantom.exit(); - }, 1000); - } -}); \ No newline at end of file diff --git a/cobra/scheduler/report.py b/cobra/scheduler/report.py deleted file mode 100644 index e77cc3c0..00000000 --- a/cobra/scheduler/report.py +++ /dev/null @@ -1,143 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - scheduler.report - ~~~~~~~~~~~~~~~~ - - Implements automation report Cobra data - - :author: Feei - :homepage: https://github.com/wufeifei/cobra - :license: MIT, see LICENSE for more details. - :copyright: Copyright (c) 2017 Feei. All rights reserved -""" -import os -import subprocess -import base64 -import datetime -from cobra.utils.log import logging -from cobra.utils.config import Config - -import smtplib -from smtplib import SMTPException -from email.mime.text import MIMEText -from email.mime.multipart import MIMEMultipart - -logging = logging.getLogger(__name__) - -phantomjs = '/usr/local/bin/phantomjs' -time_types = ['w', 'm', 'q'] -time_type_des = { - 'w': '周', - 'm': '月', - 'q': '季' -} - - -class Report(object): - def __init__(self, time_type, month=None): - if time_type not in time_types: - logging.critical('Time type exception') - return - - self.time_type_de = time_type_des[time_type] - - # mail - mark = '' - if month is None: - c_month = int(datetime.datetime.today().strftime("%m")) - else: - c_month = int(month) - - if time_type == 'w': - c_week = int(datetime.datetime.today().strftime("%U")) - mark = 'W{week}'.format(week=c_week) - elif time_type == 'm': - mark = 'M{month}'.format(month=c_month) - elif time_type == 'q': - c_quarter = 0 - if c_month in [1, 2, 3]: - c_quarter = 1 - elif c_month in [4, 5, 6]: - c_quarter = 2 - elif c_month in [7, 8, 9]: - c_quarter = 3 - elif c_month in [10, 11, 12]: - c_quarter = 4 - mark = 'Q{quarter}'.format(quarter=c_quarter) - self.subject = '[Cobra] 代码安全{0}报({mark})'.format(self.time_type_de, mark=mark) - self.user = Config('email', 'user').value - self.name = Config('email', 'name').value - self.to = Config('report', 'to').value - self.host = Config('email', 'host').value - self.port = Config('email', 'port').value - self.password = Config('email', 'password').value - - self.param = [phantomjs, os.path.join(Config().project_directory, 'scheduler', 'report.js'), Config().project_directory, time_type] - if month is not None: - self.param.append(month) - - def run(self): - capture = self.capture() - if capture is False: - logging.critical('Capture failed') - return False - - # send notification - if self.notification(capture): - return True - else: - logging.critical('Notification failed') - return False - - def capture(self): - """ - Use PhantomJS to capture report page - :return: boolean - """ - capture = None - p = subprocess.Popen(self.param, stdout=subprocess.PIPE) - result, err = p.communicate() - if 'Critical' in result: - logging.critical('Capture exception') - return False - lines = result.split('\n') - for l in lines: - if 'reports' in l: - capture = l.split(':')[1].strip() - - if capture is None: - logging.critical('get capture image file failed') - return False - else: - return os.path.join(Config().project_directory, capture) - - def notification(self, capture_path): - """ - Email notification - :param capture_path: - :return: boolean - """ - msg = MIMEMultipart() - msg['Subject'] = self.subject - msg['From'] = '{0}<{1}>'.format(self.name, self.user) - msg['To'] = self.to - - with open(capture_path, "rb") as image_file: - encoded_string = base64.b64encode(image_file.read()) - - text = MIMEText(''.format(encoded_string), 'html') - msg.attach(text) - - try: - s = smtplib.SMTP(self.host, self.port) - s.ehlo() - s.starttls() - s.ehlo() - s.login(self.user, self.password) - s.sendmail(self.user, self.to, msg.as_string()) - s.quit() - return True - except SMTPException: - logging.critical('Send mail failed') - return False diff --git a/cobra/scheduler/scan.py b/cobra/scheduler/scan.py deleted file mode 100644 index 9870aba0..00000000 --- a/cobra/scheduler/scan.py +++ /dev/null @@ -1,45 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - scheduler.scan - ~~~~~~~~~~~~~~ - - Implements periodic scan job - - :author: Feei - :homepage: https://github.com/wufeifei/cobra - :license: MIT, see LICENSE for more details. - :copyright: Copyright (c) 2017 Feei. All rights reserved -""" -import json -import requests -from cobra.utils.log import logging -from cobra.utils import common, config -from cobra.app.models import CobraProjects - -logging = logging.getLogger(__name__) - - -class Scan(object): - def __init__(self): - domain = '{0}:{1}'.format(config.Config('cobra', 'host').value, config.Config('cobra', 'port').value) - self.api = 'http://' + domain + '/api/{0}' - self.headers = {"Content-Type": "application/json"} - self.key = common.md5('CobraAuthKey') - self.branch = 'master' - - def all(self): - projects = CobraProjects.query.with_entities(CobraProjects.repository).filter(CobraProjects.status == CobraProjects.get_status('on')).all() - for project in projects: - payload = json.dumps({ - "key": self.key, - "target": project.repository, - "branch": self.branch - }) - - try: - response = requests.post(self.api.format('add'), data=payload, headers=self.headers) - response_json = response.json() - logging.info(project.repository, response_json) - except (requests.ConnectionError, requests.HTTPError) as e: - logging.critical("API Add failed: {0}".format(e)) diff --git a/cobra/templite.py b/cobra/templite.py index de14580b..ddfc92c6 100644 --- a/cobra/templite.py +++ b/cobra/templite.py @@ -2,7 +2,7 @@ """ templite - ~~~~~~~~~~~~~~~~~~~ + ~~~~~~~~ A simple template engine diff --git a/tests/test_cve_parse.py b/tests/test_cve_parse.py index 57e79e7d..2be6d05e 100644 --- a/tests/test_cve_parse.py +++ b/tests/test_cve_parse.py @@ -14,8 +14,8 @@ """ import pytest import xml.etree.ElementTree as eT -from cobra.cve_parse import * -from cobra.cve_parse import CveParse, project_directory +from cobra.cve import * +from cobra.cve import CveParse, project_directory try: from configparser import ConfigParser, NoSectionError except ImportError: From bd31aae0ec858c46fc03b02f0667744147c7e869 Mon Sep 17 00:00:00 2001 From: Feei Date: Wed, 6 Sep 2017 12:33:41 +0800 Subject: [PATCH 21/24] rename --- cobra/cve_parse.py | 425 --------------------------------------------- 1 file changed, 425 deletions(-) delete mode 100644 cobra/cve_parse.py diff --git a/cobra/cve_parse.py b/cobra/cve_parse.py deleted file mode 100644 index ebc3690d..00000000 --- a/cobra/cve_parse.py +++ /dev/null @@ -1,425 +0,0 @@ -# -*- coding: utf-8 -*- -""" - cobra - ~~~~~ - - Implements cobra main - - :author: BlBana <635373043@qq.com> - :homepage: https://github.com/wufeifei/cobra - :license: MIT, see LICENSE for more details. - :copyright: Copyright (c) 2017 Feei. All rights reserved -""" -import datetime -import os -import requests -import threading -import gzip -import xml.etree.cElementTree as eT -import multiprocessing -from .config import project_directory, Config, config_path -from .log import logger -from .dependencies import Dependencies -from .result import VulnerabilityResult - -try: - from urllib import urlretrieve # Python2 -except ImportError: - from urllib.request import urlretrieve # Python3 - -try: - from configparser import ConfigParser -except ImportError: - from ConfigParser import ConfigParser - - -class CveParse(object): - def __init__(self, target_file, project_path, year=None): - """ - :param target_file: The cve_file's path - """ - self.cve_file = target_file - self.pro_file = project_path - self.year = year - self._result = {} # {'cve_id':{'access-complexity':xxx, 'cpe':[]}} access-complexity and cpe may be None - self._rule = {} - self._scan_result = {} - self.res = True - self.CVSS = "{http://scap.nist.gov/schema/cvss-v2/0.2}" - self.VULN = "{http://scap.nist.gov/schema/vulnerability/0.4}" - self.NS = "{http://scap.nist.gov/schema/feed/vulnerability/2.0}" - - def cve_parse(self): - """ - Resolve the latest rules,parse new rule from cve.xml - :return: None - """ - cve_file = self.get_cve_file() - if not isinstance(cve_file, list): - tree = self.parse_xml(cve_file) - root = tree.getroot() - childs = root.findall('.//%sentry' % self.NS) - for child in childs: # child is entry Element - cve_id = child.attrib['id'] - cve_info = self.cve_info(child) - if len(cve_info) != 0: - self._result[cve_id] = cve_info - else: - for filename in cve_file: - tree = self.parse_xml(filename) - root = tree.getroot() - childs = root.findall('.//%sentry' % self.NS) - for child in childs: # child is entry Element - cve_id = child.attrib['id'] - cve_info = self.cve_info(child) - if len(cve_info) != 0: - self._result[cve_id] = cve_info - - def get_cve_file(self): - cve_file = [] - if os.path.isfile(self.cve_file): - return self.cve_file - else: - for root, dirs, filenames in os.walk(self.cve_file): - for filename in filenames: - cve_file.append(os.path.join(root, filename)) - return cve_file - - def cve_info(self, entry): - """ - :param entry: every entry Element - :return:Information inside each entry node - """ - cpe_list = [] - cve_info = {} - black_lists = ['ffmpeg', 'linux', 'ie', 'apple_tv', 'iphone_os', 'watchos', 'mac_os_x', 'windows', 'ios', - 'android', 'flash_player', 'office', 'wireshark', 'safari', 'mysql', 'word', '.net_framework', - 'samba', 'ntp', 'tomcat', 'unixware', 'vpn', 'netware', 'proxy_server', 'http_server', 'irix', - 'solaris', 'weblogic_server', 'kde', 'dhcpd', 'database_server', 'mandrake_linux', 'openssl' - 'suse_linux', - 'vim', 'debian_linux', 'putty', 'ubuntu_linux', 'mozilla', 'ftp_server', 'cvs' - 'oracle', - 'ws_ftp_server', 'surgemail', 'opera_web_browser', 'sql_server', 'ethereal', 'gaim', - 'wu-ftpd', 'cluster_server', 'catos', 'mantis', 'quicktime', 'security_linux', 'firefox', - 'jetty_http_server', 'php:', 'enterprise_linux', 'oracle10g', 'oracle9g', 'oracle8g', 'firehol', - 'fetchmail', 'postgresql', 'freebsd', 'chrome'] - products = entry.findall('.//%sproduct' % self.VULN) - access_complexity = entry.find('.//%saccess-complexity' % self.CVSS) - for product in products: - module_version = product.text.split(':') - if len(module_version) > 4: - module_ = module_version[3] + ':' + module_version[4] - elif len(module_version) == 4: - module_ = module_version[3] - else: - module_ = module_version[2] - for black_list in black_lists: - if str(module_).startswith(black_list): - cve_info = {} - return cve_info - cpe_list.append(module_) - if len(cpe_list) == 0: - return cve_info - else: - cve_info['cpe'] = cpe_list - - if access_complexity is None: - cve_info['level'] = 'unknown' - else: - cve_info['level'] = access_complexity.text - return cve_info - - @staticmethod - def parse_xml(file_path): - return eT.parse(file_path) - - def get_result(self): - """ - :return:The result from cve.xml,this is new rule - """ - return self._result - - def rule_xml(self): - """ - If you want to update rule, Please use this function, it will auto parse rule, and write in file - :return: - """ - starttime = datetime.datetime.now() - logger.info('The rule CVE-999' + str(self.year)[1:] + '.xml are being updated. Please wait for a moment....') - self.cve_parse() - cobra = eT.Element('cobra') # root Ele - cobra.set('document', 'https://github.com/wufeifei/cobra') - for cve_id in self._result.keys(): - cve_child = eT.Element('cve') # cve Ele - cve_child.set('id', cve_id) - cve_child.set('level', self._result[cve_id]['level']) - if 'cpe' in self._result[cve_id]: - for product_ in self._result[cve_id]['cpe']: - product = eT.Element('product') - product.text = product_ - cve_child.append(product) # product in products - cobra.append(cve_child) # cve in cobra - self.pretty(cobra) - tree = eT.ElementTree(cobra) - rule_path = project_directory + '/rules/CVI-999' - tree.write(rule_path + str(self.year)[1:] + '.xml') - endtime = datetime.datetime.now() - logger.info( - 'CVE-999' + str(self.year)[1:] + '.xml Rule update succeeds, times:%ds' % (endtime - starttime).seconds) - - def pretty(self, e, level=0): - """ - :param e:The root Element - :param level: - :return: None,pretty the xml file - """ - if len(e) > 0: - e.text = '\n' + '\t' * (level + 1) - for child in e: - self.pretty(child, level + 1) - child.tail = child.tail[:-1] - e.tail = '\n' + '\t' * level - - def rule_parse(self, file_): - """ - :return: rules from CVI-999999.xml and CVI-999999.xml - """ - tree = self.parse_xml(file_) - root = tree.getroot() - cves = root.findall('.//cve') - for cve_child in cves: - cve_id = cve_child.attrib['id'] - cve_level = cve_child.attrib['level'] - rule_info = self.rule_info(cve_child) - rule_info['level'] = cve_level - self._rule[cve_id] = rule_info - - @staticmethod - def rule_info(cve_child): - rule_info = {} - cpe_list = [] - products = cve_child.findall('.//product') - for product in products: - cpe_list.append(product.text.lower()) - rule_info['cpe'] = cpe_list - return rule_info - - def get_rule(self): - """ - :return: The rule from CVI-999999.xml and CVI-999999.xml - """ - return self._rule - - def scan_cve(self, file_): - """ - :return:Analytical dependency，Match the rules and get the result - """ - self.rule_parse(file_) - cve = self.get_rule() - dependency = Dependencies(self.pro_file) - project_info = dependency.get_result - for pro_info in project_info: - module_version = pro_info.lower() + ':' + project_info[pro_info] - self.set_scan_result(cve, module_version) - self.log_result() - - def set_scan_result(self, cves, module_version): - """ - :param cves: - :param module_version: - :return:set the scan result - """ - scan_cves = {} - for cve_child in cves: - if module_version in cves[cve_child]['cpe']: - scan_cves[cve_child] = cves[cve_child]['level'] - if len(scan_cves): - self._scan_result[module_version] = scan_cves - - def log_result(self): - for module_ in self._scan_result: - for cve_child in self._scan_result[module_]: - cve_id = cve_child - level = self._scan_result[module_][cve_id] - logger.debug('Find the module ' + module_ + ' have ' + cve_id + ',level: ' + level) - count = len(self._scan_result[module_]) - logger.debug('The ' + module_ + ' module have ' + str(count) + ' CVE Vul(s)') - - def get_scan_result(self): - return self._scan_result - - -def rule_parse(): - if is_update(): - gz_files = download_rule_gz() - un_gz(gz_files) - pool = multiprocessing.Pool() - for year in range(2002, datetime.datetime.now().year + 1): - cve_xml = project_directory + "/rules/%d.xml" % year - pool.apply_async(rule_single, args=(cve_xml, year)) - pool.close() - pool.join() - for year in range(2002, datetime.datetime.now().year + 1): - os.remove(project_directory + "/rules/%d.xml" % year) - logger.info("The rule update success, start scan cve vuls") - return True - else: - logger.info("The CVE Rule not update, start scan cve vuls") - - -def download_rule_gz(): - threads = [] - files = [] - start_time = datetime.datetime.now() - for year in range(2002, datetime.datetime.now().year + 1): - url = "https://static.nvd.nist.gov/feeds/xml/cve/2.0/nvdcve-2.0-" + str(year) + ".xml.gz" - logger.info("start download " + str(year) + ".xml.gz") - thread = threading.Thread(target=urlretrieve, - args=(url, project_directory + "/rules/" + str(year) + ".xml.gz")) - thread.start() - threads.append(thread) - files.append(project_directory + "/rules/" + str(year) + ".xml.gz") - for t in threads: - t.join() - end_time = datetime.datetime.now() - logger.info("All CVE xml file already download success, use time:%ds" % (end_time - start_time).seconds) - return files - - -def un_gz(gz_files): - """ungz zip file""" - start_time = datetime.datetime.now() - logger.info("Start decompress rule files, Please wait a moment....") - for gz_file in gz_files: - f_name = gz_file.replace(".gz", "") - g_file = gzip.GzipFile(gz_file) - open(f_name, "wb+").write(g_file.read()) - g_file.close() - os.remove(gz_file) - end_time = datetime.datetime.now() - logger.info("Decompress success, use time:%ds" % (end_time - start_time).seconds) - return True - - -def rule_single(target_directory, year): - CveParse(target_directory, '.', year).rule_xml() - - -def is_update(): - url = "https://static.nvd.nist.gov/feeds/xml/cve/2.0/nvdcve-2.0-modified.meta" - requests.packages.urllib3.disable_warnings() - r = requests.get(url, verify=False) - index = r.text.find('sha256:') - sha256_now = r.text[index + 7:].strip() - sha256_local = Config(level1='cve', level2='modified').value - if sha256_local != sha256_now: - logger.info("The CVE Rule already update, start update local rule") - config = ConfigParser() - config.read(config_path) - config.set('cve', 'modified', sha256_now) - try: - fi = open(config_path, 'w') - config.write(fi) - fi.close() - except IOError as e: - logger.warning(e) - logger.info("The sha256 been update") - return True - return False - - -def scan_cve(target_directory): - cve_vuls = [] - cve_files = [] - - def store(results): - if len(results[0]) != 0: - for module_ in results[0]: - for cve_id, cve_level in results[0][module_].items(): - cve_path = results[1] - cve_vul = parse_math(cve_path, cve_id, cve_level, module_, target_directory) - cve_vuls.append(cve_vul) - else: - logger.debug('[SCAN] [STORE] Not found vulnerabilities on this rule!') - - rule_path = os.path.join(project_directory, 'rules') - files = os.listdir(rule_path) - for cvi_file in files: - if cvi_file.startswith('CVI-999'): - cve_files.append(cvi_file) - if len(cve_files) == 0: - logger.info("Can't find the rules, please update rules") - return - pool = multiprocessing.Pool() - logger.info('[PUSH] {rc} CVE Rules'.format(rc=len(cve_files))) - for cve_file in cve_files: - cve_path = os.path.join(rule_path, cve_file) - pool.apply_async(scan_single, args=(target_directory, cve_path), callback=store) - pool.close() - pool.join() - return cve_vuls - - -def scan_single(target_directory, cve_path): - """ - :param target_directory: scan path - :param cve_path: CVI-999***.xml - :return: - """ - cve = CveParse('.', target_directory) - cve.scan_cve(cve_path) - return cve.get_scan_result(), cve_path - - -def parse_math(cve_path, cve_id, cve_level, module_, target_directory): - flag = 0 - file_path = 'unkown' - mr = VulnerabilityResult() - module_name, module_version = module_.split(':') - cvi = cve_path.lower().split('cvi-')[1][:6] - rule_name = '引用了存在漏洞的三方组件' - if cve_level == 'LOW': - cve_level = 2 - - elif cve_level == 'MEDIUM': - cve_level = 5 - - elif cve_level == 'HIGH': - cve_level = 8 - - for root, dirs, filenames in os.walk(target_directory): - for filename in filenames: - if filename == 'pom.xml' and flag != 2: - file_path = os.path.join(root, filename) - file_path = file_path.replace(target_directory, '') - flag = 1 - - elif filename == 'requirements.txt' and flag != 1: - file_path = os.path.join(root, filename) - file_path = file_path.replace(target_directory, '') - flag = 2 - - if flag != 0: - mr.file_path = file_path - - else: - mr.file_path = 'unkown' - mr.language = cve_id - mr.id = cvi - mr.rule_name = rule_name - mr.level = cve_level - mr.line_number = 1 - mr.code_content = module_name + ':' + module_version - mr.solution = """ - 三方依赖**""" + module_name + """:""" + module_version + """**存在CVE漏洞，CVE漏洞编号为: **""" + cve_id + """** - ## 安全风险 - - ## 安全修复 - 请根据对应厂商公告，及时更新三方依赖至安全版本 - """ - - logger.debug('[CVE {i}] {r}:Find {n}:{v} have vul {c} and level is {l}'.format(i=mr.id, r=mr.rule_name, - n=mr.file_path, v=mr.line_number, - c=mr.language, l=mr.level)) - - return mr From 41c1b58e980ef82190956b06d8ab9ba4e6f06491 Mon Sep 17 00:00:00 2001 From: PSoul Date: Wed, 6 Sep 2017 15:11:33 +0800 Subject: [PATCH 22/24] =?UTF-8?q?fix=20#547,=E7=8E=B0=E5=9C=A8=E6=98=BE?= =?UTF-8?q?=E7=A4=BA=E7=A8=8B=E5=BA=8F=E8=BF=90=E8=A1=8C=E6=97=B6=E7=9A=84?= =?UTF-8?q?=E6=89=80=E6=9C=89=E6=97=B6=E9=97=B4=EF=BC=8C=E4=B9=8B=E5=89=8D?= =?UTF-8?q?=E6=98=AF=E7=A8=8B=E5=BA=8F=E4=BD=BF=E7=94=A8CPU=E7=9A=84?= =?UTF-8?q?=E6=97=B6=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cobra/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cobra/__init__.py b/cobra/__init__.py index 8c34c662..f624f152 100644 --- a/cobra/__init__.py +++ b/cobra/__init__.py @@ -37,7 +37,7 @@ def main(): try: # arg parse - t1 = time.clock() + t1 = time.time() parser = argparse.ArgumentParser(prog=__title__, description=__introduction__, epilog=__epilog__, formatter_class=argparse.RawDescriptionHelpFormatter) parser_group_scan = parser.add_argument_group('Scan') @@ -80,7 +80,7 @@ def main(): # API call CLI mode a_sid = args.sid cli.start(args.target, args.format, args.output, args.special_rules, a_sid) - t2 = time.clock() + t2 = time.time() logger.info('[INIT] Done! Consume Time:{ct}s'.format(ct=t2 - t1)) except Exception as e: err_msg = unhandled_exception_message() From 6089e8469dff2f66148029eb2f5020f6c1abf28c Mon Sep 17 00:00:00 2001 From: Feei Date: Wed, 6 Sep 2017 16:13:35 +0800 Subject: [PATCH 23/24] improves report and add analysis field --- cobra/cve.py | 6 ++-- cobra/engine.py | 85 +++++++++++++++++++++++++------------------------ cobra/result.py | 1 + 3 files changed, 47 insertions(+), 45 deletions(-) diff --git a/cobra/cve.py b/cobra/cve.py index af37edfd..6aa2bfbd 100644 --- a/cobra/cve.py +++ b/cobra/cve.py @@ -378,7 +378,6 @@ def parse_math(cve_path, cve_id, cve_level, module_, target_directory): mr = VulnerabilityResult() module_name, module_version = module_.split(':') cvi = cve_path.lower().split('cvi-')[1][:6] - rule_name = '引用了存在漏洞的三方组件' if cve_level == 'LOW': cve_level = 2 @@ -405,11 +404,12 @@ def parse_math(cve_path, cve_id, cve_level, module_, target_directory): else: mr.file_path = 'unkown' - mr.language = cve_id + mr.language = '*' mr.id = cvi - mr.rule_name = rule_name + mr.rule_name = cve_id mr.level = cve_level mr.line_number = 1 + mr.analysis = 'Dependencies Matched(依赖匹配)' mr.code_content = module_name + ':' + module_version mr.solution = """ 三方依赖**""" + module_name + """:""" + module_version + """**存在CVE漏洞，CVE漏洞编号为: **""" + cve_id + """** diff --git a/cobra/engine.py b/cobra/engine.py index a02b67aa..4aae64e7 100644 --- a/cobra/engine.py +++ b/cobra/engine.py @@ -70,7 +70,7 @@ def list(self, data=None): result = f.readline() return json.loads(result) else: - with open(file_path, 'r+') as f: # w+ causes a file reading bug + with open(file_path, 'r+') as f: # w+ causes a file reading bug fcntl.flock(f, fcntl.LOCK_EX) result = f.read() if result == '': @@ -132,7 +132,13 @@ def score2level(score): if level is None: return 'Unknown' else: - return '{l}-{s}: {ast}'.format(l=level[:1], s=score, ast='☆' * score) + if score < 10: + score_full = '0{s}'.format(s=score) + else: + score_full = score + + a = '{s}{e}'.format(s=score * '■', e=(10 - score) * '□') + return '{l}-{s}: {ast}'.format(l=level[:1], s=score_full, ast=a) def scan_single(target_directory, single_rule): @@ -194,12 +200,12 @@ def store(result): # print data = [] - table = PrettyTable(['#', 'CVI', 'VUL', 'Rule(ID/Name)', 'Lang/CVE-id', 'Level-Score', 'Target-File:Line-Number/Module:Version', 'Commit(Author/Time)', 'Source Code Content']) + table = PrettyTable(['#', 'CVI', 'VUL', 'Rule', 'Lang', 'Level-Score', 'Target', 'Commit(Time, Author)', 'Source Code Content', 'Analysis']) table.align = 'l' trigger_rules = [] for idx, x in enumerate(find_vulnerabilities): trigger = '{fp}:{ln}'.format(fp=x.file_path, ln=x.line_number) - commit = u'@{author},{time}'.format(author=x.commit_author, time=x.commit_time) + commit = u'{time}, @{author}'.format(author=x.commit_author, time=x.commit_time) level = score2level(x.level) cvi = x.id[0:3] if cvi in vulnerabilities: @@ -207,10 +213,10 @@ def store(result): else: cvn = 'Unknown' try: - code_content = x.code_content[:100].strip() + code_content = x.code_content[:50].strip() except AttributeError as e: code_content = x.code_content.decode('utf-8')[:100].strip() - row = [idx + 1, x.id, cvn, x.rule_name, x.language, level, trigger, commit, code_content] + row = [idx + 1, x.id, cvn, x.rule_name, x.language, level, trigger, commit, code_content, x.analysis] data.append(row) table.add_row(row) if x.id not in trigger_rules: @@ -334,12 +340,13 @@ def process(self): continue is_test = False try: - is_vulnerability, status_code = Core(self.target_directory, vulnerability, self.sr, 'project name', ['whitelist1', 'whitelist2'], test=is_test, index=index).scan() + is_vulnerability, reason = Core(self.target_directory, vulnerability, self.sr, 'project name', ['whitelist1', 'whitelist2'], test=is_test, index=index).scan() if is_vulnerability: - logger.debug('[CVI-{cvi}] [RET] Found {code}'.format(cvi=self.sr['id'], code=status_code)) + logger.debug('[CVI-{cvi}] [RET] Found {code}'.format(cvi=self.sr['id'], code=reason)) + vulnerability.analysis = reason self.rule_vulnerabilities.append(vulnerability) else: - logger.debug('Not vulnerability: {code}'.format(code=status_code)) + logger.debug('Not vulnerability: {code}'.format(code=reason)) except Exception: raise logger.debug('[CVI-{cvi}] {vn} Vulnerabilities: {count}'.format(cvi=self.sr['id'], vn=self.sr['name'], count=len(self.rule_vulnerabilities))) @@ -544,43 +551,47 @@ def scan(self): :return: is_vulnerability, code """ self.method = 0 + self.code_content = self.code_content + if len(self.code_content) > 512: + self.code_content = self.code_content[:500] + self.status = self.status_init + self.repair_code = self.repair_code_init if self.is_white_list(): logger.debug("[RET] Whitelist") - return False, 5001 + return False, 'Whitelists(白名单)' if self.is_special_file(): logger.debug("[RET] Special File") - return False, 5002 + return False, 'Special File(特殊文件)' if self.is_test_file(): logger.debug("[CORE] Test File") if self.is_annotation(): logger.debug("[RET] Annotation") - return False, 5004 + return False, 'Annotation(注释)' if self.rule_match_mode == const.mm_find_extension: # # Find-Extension # Match(extension) -> Done # - found_vul = True + return True, 'FIND-EXTENSION(后缀查找)' elif self.rule_match_mode == const.mm_regex_only_match: # # Regex-Only-Match # Match(regex) -> Repair -> Done # logger.debug("[CVI-{cvi}] [ONLY-MATCH]".format(cvi=self.cvi)) - found_vul = True if self.rule_match2 is not None: ast = CAST(self.rule_match, self.target_directory, self.file_path, self.line_number, self.code_content) is_match, data = ast.match(self.rule_match2, self.rule_match2_block) if is_match: logger.debug('[CVI-{cvi}] [MATCH2] True'.format(cvi=self.cvi)) - return True, 1001 + return True, 'REGEX-ONLY-MATCH+MATCH2(正则仅匹配+二次匹配)' else: logger.debug('[CVI-{cvi}] [MATCH2] False'.format(cvi=self.cvi)) - return False, 1002 + return False, 'REGEX-ONLY-MATCH+Not matched2(未匹配到二次规则)' if self.rule_repair is not None: logger.debug('[VERIFY-REPAIR]') @@ -589,10 +600,12 @@ def scan(self): if is_repair: # fixed logger.debug('[CVI-{cvi}] [RET] Vulnerability Fixed'.format(cvi=self.cvi)) - return False, 1002 + return False, 'REGEX-ONLY-MATCH+Vulnerability-Fixed(漏洞已修复)' else: logger.debug('[CVI-{cvi}] [REPAIR] [RET] Not fixed'.format(cvi=self.cvi)) - found_vul = True + return True, 'REGEX-ONLY-MATCH+NOT FIX(未修复)' + else: + return True, 'REGEX-ONLY-MATCH(正则仅匹配+无修复规则)' else: # # Function-Param-Controllable @@ -604,12 +617,9 @@ def scan(self): # Match(regex) -> Match2(regex) -> Param-Controllable -> Repair -> Done # logger.debug('[CVI-{cvi}] match-mode {mm}'.format(cvi=self.cvi, mm=self.rule_match_mode)) - found_vul = False - result = [] if self.file_path[-3:].lower() == 'php': try: ast = CAST(self.rule_match, self.target_directory, self.file_path, self.line_number, self.code_content) - # Match2 if self.rule_match_mode == const.mm_function_param_controllable: rule_match = self.rule_match.strip('()').split('|') logger.debug('[RULE_MATCH] {r}'.format(r=rule_match)) @@ -620,16 +630,16 @@ def scan(self): logger.debug('[AST] [RET] {c}'.format(c=result)) if len(result) > 0: if result[0]['code'] == 1: # 函数参数可控 - return True, 1001 + return True, 'FUNCTION-PARAM-CONTROLLABLE(函数入参可控)' if result[0]['code'] == 2: # 函数为敏感函数 - return True, 1001 + return False, 'FUNCTION-PARAM-CONTROLLABLE(函数入参来自所在函数)' if result[0]['code'] == 0: # 漏洞修复 - return False, 1002 + return False, 'FUNCTION-PARAM-CONTROLLABLE+Vulnerability-Fixed(漏洞已修复)' if result[0]['code'] == -1: # 函数参数不可控 - return False, 1002 + return False, 'FUNCTION-PARAM-CONTROLLABLE(入参不可控)' logger.debug('[AST] [CODE] {code}'.format(code=result[0]['code'])) else: @@ -638,14 +648,16 @@ def scan(self): logger.warning(traceback.format_exc()) raise + # Match2 if self.rule_match2 is not None: is_match, data = ast.match(self.rule_match2, self.rule_match2_block) if is_match: logger.debug('[CVI-{cvi}] [MATCH2] True'.format(cvi=self.cvi)) - return True, 1001 + return True, 'FPC+MATCH2(函数入参可控+二次匹配)' else: logger.debug('[CVI-{cvi}] [MATCH2] False'.format(cvi=self.cvi)) - return False, 1002 + return False, 'FPC+NOT-MATCH2(函数入参可控+二次未匹配)' + # Param-Controllable param_is_controllable, data = ast.is_controllable_param() if param_is_controllable: @@ -655,24 +667,13 @@ def scan(self): if is_repair: # fixed logger.debug('[CVI-{cvi}] [REPAIR] Vulnerability Fixed'.format(cvi=self.cvi)) - return False, 1002 + return False, 'Vulnerability-Fixed(漏洞已修复)' else: logger.debug('[CVI-{cvi}] [REPAIR] [RET] Not fixed'.format(cvi=self.cvi)) - found_vul = True + return True, 'MATCH+REPAIR(匹配+未修复)' else: logger.debug('[CVI-{cvi}] [PARAM-CONTROLLABLE] Param Not Controllable'.format(cvi=self.cvi)) - return False, 4002 + return False, 'Param-Not-Controllable(参数不可控)' except Exception as e: logger.debug(traceback.format_exc()) - return False, 4004 - - if found_vul: - self.code_content = self.code_content - if len(self.code_content) > 512: - self.code_content = self.code_content[:500] - self.status = self.status_init - self.repair_code = self.repair_code_init - return True, 1001 - else: - logger.debug("[CVI-{cvi}] [DONE] Not found vulnerability".format(cvi=self.cvi)) - return False, 4002 + return False, 'Exception' diff --git a/cobra/result.py b/cobra/result.py index a442830e..a10b1e9d 100644 --- a/cobra/result.py +++ b/cobra/result.py @@ -17,6 +17,7 @@ class VulnerabilityResult: def __init__(self): self.id = '' self.file_path = None + self.analysis = '' self.rule_name = '' self.language = '' From 2d979c5e7f8b54df88b0e4c79bbd414e097ef43e Mon Sep 17 00:00:00 2001 From: Feei Date: Wed, 6 Sep 2017 16:25:58 +0800 Subject: [PATCH 24/24] Released v2.0.0-alpha.2 --- CHANGES.md | 28 ++++++++++++++++++++++++++++ cobra/__version__.py | 2 +- cobra/scheduler/__init__.py | 0 3 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 CHANGES.md delete mode 100644 cobra/scheduler/__init__.py diff --git a/CHANGES.md b/CHANGES.md new file mode 100644 index 00000000..ba8708f5 --- /dev/null +++ b/CHANGES.md @@ -0,0 +1,28 @@ +Cobra Changelog +=============== + +Here you can see the full list of changes between each Cobra release. + +Version 2.0.0-alpha.2 +--------------------- + +Released on Sep 06 2017 + +- 修复上传非支持的后缀提示 +- 修复VirtualEnv环境下无法执行 +- 修复grep/find路径位置变动 +- 优化日志等级 +- 优化Docker下路径错误 +- 优化耗时计算 +- 其它细节优化和Bug修复 + +Version 2.0.0-alpha.1 +--------------------- + +Released on Sep 05 2017 + +内测正式版本 + +- 简化安装和使用成本 +- 增加CLI模式 +- 开源扫描规则 \ No newline at end of file diff --git a/cobra/__version__.py b/cobra/__version__.py index 1df6f919..a5a44fb7 100644 --- a/cobra/__version__.py +++ b/cobra/__version__.py @@ -7,7 +7,7 @@ __issue_page__ = 'https://github.com/wufeifei/cobra/issues/new' __python_version__ = sys.version.split()[0] __platform__ = platform.platform() -__version__ = '2.0.0-alpha' +__version__ = '2.0.0-alpha.2' __author__ = 'Feei' __author_email__ = 'feei@feei.cn' __license__ = 'MIT License' diff --git a/cobra/scheduler/__init__.py b/cobra/scheduler/__init__.py deleted file mode 100644 index e69de29b..00000000