Merge pull request #16 from Lightslayer/master

Add CLI and fix edge case with gzipped text files

.github/workflows/pythonpackage.yml

@@ -0,0 +1,45 @@
+name: Python package
+
+on: [push]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 4
+      matrix:
+        python-version: [2.7, 3.5, 3.6, 3.7]
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        # pip install -r requirements.txt
+    - name: Lint with flake8
+      run: |
+        pip install flake8
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --exclude=tests/testdata/ --max-line-length=120
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude=tests/testdata/ --max-line-length=120
+    - name: Test with nose
+      run: |
+        pip install nose coverage coveralls
+        pip install --editable .
+        nosetests dodgy -s --with-coverage --cover-inclusive --cover-package=dodgy
+    - name: Create package
+      run: |
+        pip install setuptools wheel
+        python setup.py bdist build bdist_wheel
+    - name: Upload package
+      run: |
+        pip install twine
+        # Needs TWINE_USERNAME and TWINE_PASSWORD set as github secrets:
+        twine upload dist/*.whl || echo 'Unable to upload to PyPI' 1>&2
+      if: endsWith(github.ref, 'master')

.travis.yml

@@ -4,6 +4,7 @@ python:
   - "3.4"
   - "3.5"
   - "3.6"
+  - "3.7"
 install:
   - "pip install nose coverage coveralls"
   - "pip install --editable ."

dodgy/__main__.py

@@ -0,0 +1,3 @@
+import dodgy.run
+
+dodgy.run.main()

dodgy/__pkginfo__.py

@@ -1,5 +1,5 @@
 
-VERSION = (0, 1, 9)
+VERSION = (0, 1, 10)
 
 
 def get_version():

dodgy/checks.py

@@ -1,5 +1,8 @@
 import re
 import codecs
+import gzip
+
+from functools import partial
 
 
 STRING_VALS = (
@@ -68,7 +71,13 @@ def check_line(line, check_list):
 
 
 def check_file(filepath):
-    with codecs.open(filepath, 'r', 'utf-8') as to_check:
+    if filepath.endswith('.gz'):
+        # this file looks like it is using gzip compression
+        fopen = partial(gzip.open, mode='rt')
+    else:
+        # otherwise treat as standard text file
+        fopen = partial(codecs.open, mode='r')
+    with fopen(filepath, encoding='utf-8') as to_check:
         return check_file_contents(to_check.read())
 
 
@@ -77,6 +86,7 @@ def check_file_contents(file_contents):
 
     for line_number0, line in enumerate(file_contents.split('\n')):
         for check_list in (STRING_VALS, LINE_VALS, VAR_NAMES):
-            messages += [(line_number0 + 1, key, msg) for key, msg in check_line(line, check_list)]
+            messages += [(line_number0 + 1, key, msg)
+                         for key, msg in check_line(line, check_list)]
 
     return messages

dodgy/run.py

@@ -3,10 +3,14 @@
 import os
 import mimetypes
 import json
+
+from argparse import ArgumentParser
+
 from dodgy.checks import check_file
 
 
-IGNORE_PATHS = [re.compile(patt % {'sep': re.escape(os.path.sep)}) for patt in (
+IGNORE_PATHS = [re.compile(patt % {'sep': re.escape(os.path.sep)})
+                for patt in (
     r'(^|%(sep)s)\.[^\.]',   # ignores any files or directories starting with '.'
     r'^tests?%(sep)s?',
     r'%(sep)stests?(%(sep)s|$)',
@@ -41,19 +45,23 @@ def run_checks(directory, ignore_paths=None):
         if mimetype[0] is None or not mimetype[0].startswith('text/'):
             continue
 
-        for msg_parts in check_file(filepath):
-            warnings.append({
-                'path': relpath,
-                'line': msg_parts[0],
-                'code': msg_parts[1],
-                'message': msg_parts[2]
-            })
+        try:
+            for msg_parts in check_file(filepath):
+                warnings.append({
+                    'path': relpath,
+                    'line': msg_parts[0],
+                    'code': msg_parts[1],
+                    'message': msg_parts[2]
+                })
+        except UnicodeDecodeError as err:
+            # This is a file which cannot be opened using codecs with UTF-8
+            print('Unable to read {!r}: {}'.format(filepath, err))
 
     return warnings
 
 
-def run():
-    warnings = run_checks(os.getcwd())
+def run(ignore_paths=None):
+    warnings = run_checks(os.getcwd(), ignore_paths=ignore_paths)
 
     if (warnings):
         output = json.dumps({'warnings': warnings}, indent=2)
@@ -62,6 +70,18 @@ def run():
 
     return 0
 
+def main(argv=sys.argv):
+    desc = ('A very basic tool to run against your codebase to search for "dodgy" looking values. '
+            'It is a series of simple regular expressions designed to detect things such as '
+            'accidental SCM diff checkins, or passwords/secret keys hardcoded into files.')
+    parser = ArgumentParser('dodgy', description=desc)
+    parser.add_argument('--ignore-paths', '-i', nargs='+',
+                        type=str, dest='ignore', default=None,
+                        metavar='IGNORE_PATH', help='Paths to ignore')
+    args, _ = parser.parse_known_args(argv)
+
+    run(ignore_paths=args.ignore)
+
 
 if __name__ == '__main__':
-    run()
+    main()