New plugin architecture

.bumpversion.cfg

@@ -1,6 +1,6 @@
 [bumpversion]
 current_version = 2.2.1
-files = setup.py src/scancode/__init__.py
+files = setup.py src/scancode_config.py
 commit = False
 tag = False
 

.gitignore

@@ -67,3 +67,4 @@ docs/_build
 
 # pyenv
 /.python-version
+/man/

configure.bat

@@ -1,6 +1,6 @@
 @echo OFF
 
-@rem Copyright (c) 2015 nexB Inc. http://www.nexb.com/ - All rights reserved.
+@rem Copyright (c) 2018 nexB Inc. http://www.nexb.com/ - All rights reserved.
 
 @rem ################################
 @rem # change these variables to customize this script locally
@@ -44,7 +44,7 @@ if not exist "c:\python27\python.exe" (
     echo Do NOT install Python v3 or any 64 bits edition.
     echo Instead download Python from this url and see the README.rst file for more details:
     echo(
-    echo    https://www.python.org/ftp/python/2.7.10/python-2.7.10.msi
+    echo    https://www.python.org/ftp/python/2.7.14/python-2.7.14.msi
     echo(
     exit /b 1
 )

etc/conf/base.py

@@ -5,17 +5,18 @@
 
 import sys
 
-
 """
 Check that we run a supported OS and architecture.
 """
 
+
 def unsupported(platform):
     print('Unsupported OS/platform %r.' % platform)
     print('See https://github.com/nexB/scancode-toolkit/ for supported OS/platforms.')
     print('Enter a ticket https://github.com/nexB/scancode-toolkit/issues asking for support of your OS/platform combo.')
     sys.exit(1)
 
+
 if sys.maxsize > 2 ** 32:
     arch = '64'
 else:
@@ -31,29 +32,12 @@ def unsupported(platform):
 else:
     unsupported(sys_platform)
 
-
 supported_combos = {
     'linux': ['32', '64'],
-    'win': ['32',],
-    'mac': ['64',],
+    'win': ['32', ],
+    'mac': ['64', ],
 }
 
 arches = supported_combos[os]
 if arch not in arches:
     unsupported(os + arch)
-
-
-"""
-Re/build the license cache on every configure run.
-"""
-
-def build_license_cache():
-    """
-    Force a rebuild of the license cache on configure.
-    """
-    from licensedcode import cache
-    print('* Building license index...')
-    cache.reindex()
-
-
-build_license_cache()

etc/conf/dev/base.py

@@ -12,23 +12,23 @@ def setup_dev_mode():
     not rely on license data to remain untouched and will always check the
     license index cache for consistency, rebuilding it if necessary.
     """
-    from scancode import root_dir
-    with open(os.path.join(root_dir, 'SCANCODE_DEV_MODE'), 'wb') as sdm:
+    from scancode_config import scancode_root_dir
+    with open(os.path.join(scancode_root_dir, 'SCANCODE_DEV_MODE'), 'wb') as sdm:
         sdm.write('This is a tag file to notify that ScanCode is used in development mode.')
 
 
 def setup_vscode():
     """
     Add base settings for .vscode
     """
-    from scancode import root_dir
+    from scancode_config import scancode_root_dir
     from commoncode.fileutils import create_dir
     from commoncode.fileutils import copyfile
 
-    settings = os.path.join(root_dir, 'etc', 'vscode', 'settings.json')
+    settings = os.path.join(scancode_root_dir, 'etc', 'vscode', 'settings.json')
 
     if os.path.exists(settings):
-        vscode = os.path.join(root_dir, '.vscode')
+        vscode = os.path.join(scancode_root_dir, '.vscode')
         create_dir(vscode)
         copyfile(settings, vscode)
 

etc/configure.py

@@ -64,7 +64,6 @@
 import shutil
 import subprocess
 
-
 # platform-specific file base names
 sys_platform = str(sys.platform).lower()
 on_win = False
@@ -79,7 +78,6 @@
     raise Exception('Unsupported OS/platform %r' % sys_platform)
     platform_names = tuple()
 
-
 # common file basenames for requirements and scripts
 base = ('base',)
 
@@ -213,7 +211,7 @@ def create_virtualenv(std_python, root_dir, tpp_dirs, quiet=False):
 
 def activate(root_dir):
     """ Activate a virtualenv in the current process."""
-    print("* Activating ...")
+    # print("* Activating...")
     bin_dir = os.path.join(root_dir, 'bin')
     activate_this = os.path.join(bin_dir, 'activate_this.py')
     with open(activate_this) as f:

etc/release/release.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2017 nexB Inc. http://www.nexb.com/ - All rights reserved.
+# Copyright (c) 2018 nexB Inc. http://www.nexb.com/ - All rights reserved.
 #
 
 # ScanCode release script
@@ -52,17 +52,36 @@ function test_scan {
             # this is needed for the zip
             chmod o+x scancode extractcode
 
-            # minimal test: update when new scans are available
-            ./scancode --quiet -lcip apache-2.0.LICENSE test_scan.json
-            echo "TEST JSON passed: ./scancode --quiet -lcip apache-2.0.LICENSE test_scan.json"
-            ./scancode --quiet -lcip --format json-pp apache-2.0.LICENSE test_scan.json
-            echo "TEST JSON-PP passed: ./scancode --quiet -lcip --format json-pp apache-2.0.LICENSE test_scan.json"
-            ./scancode --quiet -lcip --format html apache-2.0.LICENSE test_scan.html
-            echo "TEST HTML passed: ./scancode --quiet -lcip --format html apache-2.0.LICENSE test_scan.html"
-            ./scancode --quiet -lcip --format html-app apache-2.0.LICENSE test_scan_app.html
-            echo "TEST HTML-APP passed: ./scancode --quiet -lcip --format html-app apache-2.0.LICENSE test_scan_app.html"
-            ./extractcode --quiet samples/arch
-            echo "TEST EXTRACTCODE passed: ./extractcode --quiet samples/arch"
+            # minimal tests: update when new scans are available
+            cmd="./scancode --quiet -lcip apache-2.0.LICENSE --json test_scan.json"
+            echo "RUNNING TEST: $cmd"
+            $cmd
+            echo "TEST PASSED"
+
+            cmd="./scancode --quiet -lcip  apache-2.0.LICENSE --json-pp test_scan.json"
+            echo "RUNNING TEST: $cmd"
+            $cmd
+            echo "TEST PASSED"
+
+            cmd="./scancode --quiet -lcip apache-2.0.LICENSE --output-html test_scan.html"
+            echo "RUNNING TEST: $cmd"
+            $cmd
+            echo "TEST PASSED"
+
+            cmd="./scancode --quiet -lcip  apache-2.0.LICENSE --output-html-app test_scan_app.html"
+            echo "RUNNING TEST: $cmd"
+            $cmd
+            echo "TEST PASSED"
+
+            cmd="./scancode --quiet -lcip apache-2.0.LICENSE --output-spdx-tv test_scan.spdx"
+            echo "RUNNING TEST: $cmd"
+            $cmd
+            echo "TEST PASSED"
+
+            cmd="./extractcode --quiet samples/arch"
+            echo "RUNNING TEST: $cmd"
+            $cmd
+            echo "TEST PASSED"
 
             # cleanup
             cd ..

etc/scripts/sch2js/sch2js.py

@@ -46,10 +46,8 @@
 from schematics.types.compound import ListType
 from schematics.types.compound import ModelType
 
-
 __version__ = '1.0.1.patch'
 
-
 SCHEMATIC_TYPE_TO_JSON_TYPE = {
     'NumberType': 'number',
     'IntType': 'integer',

etc/scripts/synclic.py

@@ -31,22 +31,25 @@
 from collections import OrderedDict
 import json
 import os
+from os import mkdir
+from os.path import exists
+from os.path import join
 import zipfile
 
 import click
+from os.path import realpath
 click.disable_unicode_literals_warning = True
 import requests
 
-from commoncode import fileutils
 from commoncode import fetch
+from commoncode import fileutils
 
 import licensedcode
-from licensedcode.cache import get_licenses_db
 from licensedcode.cache import get_index
+from licensedcode.cache import get_licenses_db
 from licensedcode.models import load_licenses
 from licensedcode.models import License
 
-
 """
 Sync and update the ScanCode licenses against:
  - the SPDX license list
@@ -59,6 +62,7 @@
 TRACE_DEEP = False
 TRACE_FETCH = False
 
+
 class ExternalLicensesSource(object):
     """
     Base class to provide (including possibly fetch) licenses from an
@@ -80,44 +84,46 @@ def __init__(self, src_dir, match_text=False, match_approx=False):
         """
         `src_dir` is where the License objects are dumped.
         """
-        src_dir = os.path.realpath(src_dir)
+        src_dir = realpath(src_dir)
         self.src_dir = src_dir
 
         self.match_text = match_text
         self.match_approx = match_approx
 
         self.fetched = False
-        if os.path.exists(src_dir):
+        if exists(src_dir):
             # fetch ONLY if the directory is empty
             self.fetched = True
         else:
-            os.mkdir(src_dir)
+            mkdir(src_dir)
 
         self.update_dir = self.src_dir.rstrip('\\/') + '-update'
-        if not os.path.exists(self.update_dir):
-            os.mkdir(self.update_dir)
+        if not exists(self.update_dir):
+            mkdir(self.update_dir)
 
         self.new_dir = self.src_dir.rstrip('\\/') + '-new'
-        if not os.path.exists(self.new_dir):
-            os.mkdir(self.new_dir)
+        if not exists(self.new_dir):
+            mkdir(self.new_dir)
 
         self.del_dir = self.src_dir.rstrip('\\/') + '-del'
-        if not os.path.exists(self.del_dir):
-            os.mkdir(self.del_dir)
+        if not exists(self.del_dir):
+            mkdir(self.del_dir)
 
         self.scancodes_by_key = get_licenses_db()
 
         self.scancodes_by_spdx_key = {l.spdx_license_key.lower(): l
             for l in self.scancodes_by_key.values()
             if l.spdx_license_key}
 
-        composites_dir = os.path.join(licensedcode.data_dir, 'composites', 'licenses')
+        composites_dir = join(
+            licensedcode.models.data_dir, 'composites', 'licenses')
         self.composites_by_key = load_licenses(composites_dir, with_deprecated=True)
         self.composites_by_spdx_key = {l.spdx_license_key.lower(): l
             for l in self.composites_by_key.values()
             if l.spdx_license_key}
 
-        foreign_dir = os.path.join(licensedcode.data_dir, 'non-english', 'licenses')
+        foreign_dir = join(
+            licensedcode.models.data_dir, 'non-english', 'licenses')
         self.non_english_by_key = load_licenses(foreign_dir, with_deprecated=True)
         self.non_english_by_spdx_key = {l.spdx_license_key.lower(): l
             for l in self.non_english_by_key.values()
@@ -449,8 +455,8 @@ def __init__(self, src_dir, match_text=False, match_approx=False,
                  api_base_url=None, api_key=None):
         super(DejaSource, self).__init__(src_dir, match_text, match_approx)
 
-        self.api_base_url = api_base_url or os.environ.get('DEJACODE_API_URL', None)
-        self.api_key = api_key or os.environ.get('DEJACODE_API_KEY', None)
+        self.api_base_url = api_base_url or os.getenv('DEJACODE_API_URL')
+        self.api_key = api_key or os.getenv('DEJACODE_API_KEY')
 
         assert (self.api_key and self.api_base_url), (
             'You must set the DEJACODE_API_URL and DEJACODE_API_KEY ' +
@@ -608,11 +614,13 @@ def merge_licenses(scancode_license, other_license, updatable_attributes):
         (attribute name, value before, value after)
     """
     scancode_updated = []
+
     def update_sc(_attrib, _sc_val, _o_val):
         setattr(scancode_license, _attrib, _o_val)
         scancode_updated.append((_attrib, _sc_val, _o_val))
 
     other_updated = []
+
     def update_ot(_attrib, _sc_val, _o_val):
         setattr(other_license, _attrib, _sc_val)
         other_updated.append((_attrib, _o_val, _sc_val))
@@ -781,7 +789,7 @@ def synchronize_licenses(external_source):
         if not TRACE:print('.', end='')
 
         # Create a new ScanCode license
-        sc_license = ot_license.relocate(licensedcode.licenses_data_dir, o_key)
+        sc_license = ot_license.relocate(licensedcode.models.data_dir, o_key)
         scancodes_added.add(sc_license.key)
         scancodes_by_key[sc_license.key] = sc_license
         if TRACE: print('Other license key not in ScanCode:', ot_license.key, 'created in ScanCode.')
@@ -793,7 +801,6 @@ def synchronize_licenses(external_source):
     for k in others_changed | others_added:
         others_by_key[k].dump()
 
-
 # TODO: at last: print report of incorrect OTHER licenses to submit
 # updates eg. make API calls to DejaCode to create or update
 # licenses and submit review request e.g. submit requests to SPDX

etc/scripts/test_json2csv.py

@@ -208,16 +208,17 @@ class TestJson2CSVWithLiveScans(FileBasedTesting):
     test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata')
 
     def test_can_process_scan_from_json_scan(self):
-        import scancode
+        from scancode_config import scancode_root_dir
         from commoncode.command import execute
         test_dir = self.get_test_loc('livescan/scan')
         json_file = self.get_temp_file('json')
-        scan_cmd = os.path.join(scancode.root_dir, 'scancode')
+        scan_cmd = os.path.join(scancode_root_dir, 'scancode')
         rc, _stdout, _stderr = execute(scan_cmd,
-            ['-clip', '--email', '--url', '--strip-root', '--format', 'json', test_dir, json_file])
-        assert rc == 0
+            ['-clip', '--email', '--url', '--strip-root', test_dir,
+             '--json', json_file])
         result_file = self.get_temp_file('.csv')
         with open(result_file, 'wb') as rf:
             json2csv.json_scan_to_csv(json_file, rf)
         expected_file = self.get_test_loc('livescan/expected.csv')
         check_csvs(result_file, expected_file, regen=False)
+        assert rc == 0