Merge pull request #395 from pyupio/develop

Update master from develop

CHANGELOG.md

@@ -6,6 +6,8 @@ The format is partly based on [Keep a Changelog](https://keepachangelog.com/en/1
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [PEP 440](https://peps.python.org/pep-0440/)
 
 
+## [Unreleased] 2.1.0.dev
+
 ## [2.0.0] - 2022-06-28
 
 ### Summary:

action.yml

@@ -1,9 +1,9 @@
 # action.yml
 name: 'pyupio/safety'
-description: 'Runs the Safety CLI against your project'
+description: 'Runs the Safety CLI dependency scanner against your project'
 inputs:
   api-key:
-    description: 'PyUp API key'
+    description: 'PyUp.io API key'
     required: false
     default: ''
   scan:

appveyor.yml

@@ -1,4 +1,4 @@
-version: 2.0.0-{build}
+version: 2.1.0.dev-{build}
 image:
   - Visual Studio 2019
   - Ubuntu

safety/VERSION

@@ -1 +1 @@
-2.0.0
+2.1.0.dev

safety/cli.py

@@ -5,6 +5,7 @@
 import logging
 import os
 import sys
+import tempfile
 
 import click
 
@@ -13,26 +14,29 @@
 from safety.errors import SafetyException, SafetyError
 from safety.formatter import SafetyFormatter
 from safety.output_utils import should_add_nl
-from safety.safety import get_packages, read_vulnerabilities
+from safety.safety import get_packages, read_vulnerabilities, fetch_policy, post_results
 from safety.util import get_proxy_dict, get_packages_licenses, output_exception, \
     MutuallyExclusiveOption, DependentOption, transform_ignore, SafetyPolicyFile, active_color_if_needed, \
-    get_processed_options, get_safety_version, json_alias, bare_alias
+    get_processed_options, get_safety_version, json_alias, bare_alias, SafetyContext
 
 LOG = logging.getLogger(__name__)
 
 
 @click.group()
 @click.option('--debug/--no-debug', default=False)
-@click.option('--telemetry/--disable-telemetry', default=True)
+@click.option('--telemetry/--disable-telemetry', default=True, hidden=True)
+@click.option('--disable-optional-telemetry-data', default=False, cls=MutuallyExclusiveOption,
+              mutually_exclusive=["telemetry", "disable-telemetry"], is_flag=True, show_default=True)
 @click.version_option(version=get_safety_version())
 @click.pass_context
-def cli(ctx, debug, telemetry):
+def cli(ctx, debug, telemetry, disable_optional_telemetry_data):
     """
     Safety checks Python dependencies for known security vulnerabilities and suggests the proper
     remediations for vulnerabilities detected. Safety can be run on developer machines, in CI/CD pipelines and
     on production systems.
     """
-    ctx.telemetry = telemetry
+    SafetyContext().safety_source = 'cli'
+    ctx.telemetry = telemetry and not disable_optional_telemetry_data
     level = logging.CRITICAL
     if debug:
         level = logging.DEBUG
@@ -80,11 +84,16 @@ def cli(ctx, debug, telemetry):
               help="Output standard exit codes. Default: --exit-code")
 @click.option("--policy-file", type=SafetyPolicyFile(), default='.safety-policy.yml',
               help="Define the policy file to be used")
+@click.option("--audit-and-monitor/--disable-audit-and-monitor", default=True,
+              help="Send results back to pyup.io for viewing on your dashboard. Requires an API key.")
+@click.option("--project", default=None,
+              help="Project to associate this scan with on pyup.io. Defaults to a canonicalized github style name if available, otherwise unknown")
+
 @click.option("--save-json", default="", help="Path to where output file will be placed, if the path is a directory, "
                                               "Safety will use safety-report.json as filename. Default: empty")
 @click.pass_context
 def check(ctx, key, db, full_report, stdin, files, cache, ignore, output, json, bare, proxy_protocol, proxy_host, proxy_port,
-          exit_code, policy_file, save_json):
+          exit_code, policy_file, save_json, audit_and_monitor, project):
     """
     Find vulnerabilities in Python dependencies at the target provided.
 
@@ -100,13 +109,36 @@ def check(ctx, key, db, full_report, stdin, files, cache, ignore, output, json,
             LOG.info('Not local DB used, Getting announcements')
             announcements = safety.get_announcements(key=key, proxy=proxy_dictionary, telemetry=ctx.parent.telemetry)
 
+        if key:
+            server_policies = fetch_policy(key=key, proxy=proxy_dictionary)
+            server_audit_and_monitor = server_policies["audit_and_monitor"]
+            server_safety_policy = server_policies["safety_policy"]
+        else:
+            server_audit_and_monitor = False
+            server_safety_policy = ""
+
+        if server_safety_policy and policy_file:
+            click.secho(
+                "Warning: both a local policy file '{policy_filename}' and a server sent policy are present. "
+                "Continuing with the local policy file.".format(policy_filename=policy_file['filename']),
+                fg="yellow",
+                file=sys.stderr
+            )
+        elif server_safety_policy:
+            with tempfile.NamedTemporaryFile(prefix='server-safety-policy-') as tmp:
+                tmp.write(server_safety_policy.encode('utf-8'))
+                tmp.seek(0)
+
+                policy_file = SafetyPolicyFile().convert(tmp.name, param=None, ctx=None)
+                LOG.info('Using server side policy file')
+
         ignore_severity_rules = None
         ignore, ignore_severity_rules, exit_code = get_processed_options(policy_file, ignore,
                                                                          ignore_severity_rules, exit_code)
 
         is_env_scan = not stdin and not files
         params = {'stdin': stdin, 'files': files, 'policy_file': policy_file, 'continue_on_error': not exit_code,
-                  'ignore_severity_rules': ignore_severity_rules}
+                  'ignore_severity_rules': ignore_severity_rules, 'project': project, 'audit_and_monitor': server_audit_and_monitor and audit_and_monitor}
         LOG.info('Calling the check function')
         vulns, db_full = safety.check(packages=packages, key=key, db_mirror=db, cached=cache, ignore_vulns=ignore,
                                       ignore_severity_rules=ignore_severity_rules, proxy=proxy_dictionary,
@@ -118,8 +150,32 @@ def check(ctx, key, db, full_report, stdin, files, cache, ignore, output, json,
         LOG.info('Safety is going to calculate remediations')
         remediations = safety.calculate_remediations(vulns, db_full)
 
+        json_report = None
+        if save_json or (server_audit_and_monitor and audit_and_monitor):
+            default_name = 'safety-report.json'
+            json_report = SafetyFormatter(output='json').render_vulnerabilities(announcements, vulns, remediations,
+                                                                                full_report, packages)
+
+            if server_audit_and_monitor and audit_and_monitor:
+                policy_contents = ''
+                if policy_file:
+                    policy_contents = policy_file.get('raw', '')
+
+                r = post_results(key=key, proxy=proxy_dictionary, safety_json=json_report, policy_file=policy_contents)
+                SafetyContext().params['audit_and_monitor_url'] = r.get('url')
+
+            if save_json:
+                if os.path.isdir(save_json):
+                    save_json = os.path.join(save_json, default_name)
+
+                with open(save_json, 'w+') as output_json_file:
+                    output_json_file.write(json_report)
+
         LOG.info('Safety is going to render the vulnerabilities report using %s output', output)
-        output_report = SafetyFormatter(output=output).render_vulnerabilities(announcements, vulns, remediations,
+        if json_report and output == 'json':
+            output_report = json_report
+        else:
+            output_report = SafetyFormatter(output=output).render_vulnerabilities(announcements, vulns, remediations,
                                                                               full_report, packages)
 
         # Announcements are send to stderr if not terminal, it doesn't depend on "exit_code" value
@@ -131,19 +187,6 @@ def check(ctx, key, db, full_report, stdin, files, cache, ignore, output, json,
         LOG.info('Vulnerabilities found (Not ignored): %s', len(found_vulns))
         LOG.info('All vulnerabilities found (ignored and Not ignored): %s', len(vulns))
 
-        if save_json:
-            default_name = 'safety-report.json'
-            json_report = output_report
-
-            if output != 'json':
-                json_report = SafetyFormatter(output='json').render_vulnerabilities(announcements, vulns, remediations,
-                                                                                    full_report, packages)
-            if os.path.isdir(save_json):
-                save_json = os.path.join(save_json, default_name)
-
-            with open(save_json, 'w+') as output_json_file:
-                output_json_file.write(json_report)
-
         click.secho(output_report, nl=should_add_nl(output, found_vulns), file=sys.stdout)
 
         if exit_code and found_vulns:
@@ -225,7 +268,6 @@ def license(ctx, key, db, output, cache, files, proxyprotocol, proxyhost, proxyp
     """
     LOG.info('Running license command')
     packages = get_packages(files, False)
-    ctx.obj = packages
 
     proxy_dictionary = get_proxy_dict(proxyprotocol, proxyhost, proxyport)
     announcements = []
@@ -244,7 +286,7 @@ def license(ctx, key, db, output, cache, files, proxyprotocol, proxyhost, proxyp
         exception = e if isinstance(e, SafetyException) else SafetyException(info=e)
         output_exception(exception, exit_code_output=False)
 
-    filtered_packages_licenses = get_packages_licenses(packages, licenses_db)
+    filtered_packages_licenses = get_packages_licenses(packages=packages, licenses_db=licenses_db)
 
     output_report = SafetyFormatter(output=output).render_licenses(announcements, filtered_packages_licenses)
 
@@ -317,6 +359,8 @@ def validate(ctx, name, path):
         click.secho(str(e).lstrip(), fg='red', file=sys.stderr)
         sys.exit(EXIT_CODE_FAILURE)
 
+    del values['raw']
+
     click.secho(f'The Safety policy file was successfully parsed with the following values:', fg='green')
     click.secho(json.dumps(values, indent=4, default=str))
 

safety/errors.py

@@ -28,7 +28,7 @@ def __init__(self, reason=None, fetched_from="server",
                  message="Sorry, something went wrong.\n" +
                          "Safety CLI can not read the data fetched from {fetched_from} because is malformed.\n"):
         info = "Reason, {reason}".format(reason=reason)
-        self.message = message.format(fetched_from=fetched_from) + info if reason else ""
+        self.message = message.format(fetched_from=fetched_from) + (info if reason else "")
         super().__init__(self.message)
 
     def get_exit_code(self):
@@ -58,10 +58,12 @@ def get_exit_code(self):
 
 class InvalidKeyError(DatabaseFetchError):
 
-    def __init__(self, key=None, message="Your API Key '{key}' is invalid. See {link}"):
+    def __init__(self, key=None, message="Your API Key '{key}' is invalid. See {link}.", reason=None):
         self.key = key
         self.link = 'https://bit.ly/3OY2wEI'
         self.message = message.format(key=key, link=self.link) if key else message
+        info = f" Reason: {reason}"
+        self.message = self.message + (info if reason else "")
         super().__init__(self.message)
 
     def get_exit_code(self):
@@ -70,9 +72,10 @@ def get_exit_code(self):
 
 class TooManyRequestsError(DatabaseFetchError):
 
-    def __init__(self,
-                 message="Unable to load database (Too many requests, please wait a while before to make another request)"):
-        self.message = message
+    def __init__(self, reason=None,
+                 message="Too many requests."):
+        info = f" Reason: {reason}"
+        self.message = message + (info if reason else "")
         super().__init__(self.message)
 
     def get_exit_code(self):
@@ -81,7 +84,7 @@ def get_exit_code(self):
 
 class NetworkConnectionError(DatabaseFetchError):
 
-    def __init__(self, message="Check your network connection, unable to reach the server"):
+    def __init__(self, message="Check your network connection, unable to reach the server."):
         self.message = message
         super().__init__(self.message)
 
@@ -98,6 +101,6 @@ class ServerError(DatabaseFetchError):
     def __init__(self, reason=None,
                  message="Sorry, something went wrong.\n" + "Safety CLI can not connect to the server.\n" +
                          "Our engineers are working quickly to resolve the issue."):
-        info = " Reason: {reason}".format(reason=reason)
-        self.message = message + info if reason else ""
+        info = f" Reason: {reason}"
+        self.message = message + (info if reason else "")
         super().__init__(self.message)

safety/output_utils.py

@@ -6,7 +6,7 @@
 import click
 
 from safety.constants import RED, YELLOW
-from safety.util import get_safety_version, Package, get_terminal_size, SafetyContext
+from safety.util import get_safety_version, Package, get_terminal_size, SafetyContext, build_telemetry_data, build_git_data
 
 LOG = logging.getLogger(__name__)
 
@@ -570,13 +570,22 @@ def get_report_brief_info(as_dict=False, report_type=1, **kwargs):
     safety_policy_used = []
 
     brief_data['policy_file'] = policy_file.get('filename', '-') if policy_file else None
+    brief_data['policy_file_source'] = 'server' if brief_data['policy_file'] and 'server-safety-policy' in brief_data['policy_file'] else 'local'
 
     if policy_file and policy_file.get('filename', False):
         safety_policy_used = [
             {'style': False, 'value': '\nScanning using a security policy file'},
             {'style': True, 'value': ' {0}'.format(policy_file.get('filename', '-'))},
         ]
 
+    audit_and_monitor = []
+    if context.params.get('audit_and_monitor'):
+        logged_url = context.params.get('audit_and_monitor_url') if context.params.get('audit_and_monitor_url') else "https://pyup.io"
+        audit_and_monitor = [
+            {'style': False, 'value': '\nLogging scan results to'},
+            {'style': True, 'value': ' {0}'.format(logged_url)},
+        ]
+
     current_time = str(datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
 
     brief_data['api_key'] = bool(key)
@@ -612,6 +621,13 @@ def get_report_brief_info(as_dict=False, report_type=1, **kwargs):
              {'style': True, 'value': f' license {"type" if brief_data["licenses_found"] == 1 else "types"} found'}],
         ]
 
+    brief_data['telemetry'] = build_telemetry_data()
+
+    brief_data['git'] = build_git_data()
+    brief_data['project'] = context.params.get('project', None)
+
+    brief_data['json_version'] = 1
+
     using_sentence = build_using_sentence(key, db)
     scanned_count_sentence = build_scanned_count_sentence(packages)
 
@@ -621,7 +637,7 @@ def get_report_brief_info(as_dict=False, report_type=1, **kwargs):
      {'style': True, 'value': 'v' + get_safety_version()},
      {'style': False, 'value': ' is scanning for '},
      {'style': True, 'value': scanning_types.get(context.command, {}).get('name', '')},
-     {'style': True, 'value': '...'}] + safety_policy_used, action_executed
+     {'style': True, 'value': '...'}] + safety_policy_used + audit_and_monitor, action_executed
      ] + [nl] + scanned_items + [nl] + [using_sentence] + [scanned_count_sentence] + [timestamp]
 
     brief_info.extend(additional_data)