Merge branch 'master' into json-certificate-output

Fixes conflicts due to d27e99c introduced in certbot#8395.

.azure-pipelines/templates/jobs/standard-tests-jobs.yml

@@ -52,7 +52,7 @@ jobs:
           TOXENV: mypy
         linux-integration:
           IMAGE_NAME: ubuntu-18.04
-          PYTHON_VERSION: 2.7
+          PYTHON_VERSION: 3.8
           TOXENV: integration
           ACME_SERVER: pebble
         apache-compat:

certbot-ci/certbot_integration_tests/certbot_tests/test_main.py

@@ -544,7 +544,8 @@ def test_revoke_multiple_lineages(context):
         'revoke', '--cert-path', join(context.config_dir, 'live', cert1, 'cert.pem')
     ])
 
-    assert 'Not deleting revoked certs due to overlapping archive dirs' in output
+    with open(join(context.workspace, 'logs', 'letsencrypt.log'), 'r') as f:
+        assert 'Not deleting revoked certs due to overlapping archive dirs' in f.read()
 
 
 def test_wildcard_certificates(context):

certbot-ci/certbot_integration_tests/utils/acme_server.py

@@ -2,10 +2,12 @@
 """Module to setup an ACME CA server environment able to run multiple tests in parallel"""
 from __future__ import print_function
 
+import argparse
 import errno
 import json
 import os
 from os.path import join
+import re
 import shutil
 import subprocess
 import sys
@@ -32,13 +34,14 @@ class ACMEServer(object):
     ACMEServer is also a context manager, and so can be used to ensure ACME server is started/stopped
     upon context enter/exit.
     """
-    def __init__(self, acme_server, nodes, http_proxy=True, stdout=False):
+    def __init__(self, acme_server, nodes, http_proxy=True, stdout=False, dns_server=None):
         """
         Create an ACMEServer instance.
         :param str acme_server: the type of acme server used (boulder-v1, boulder-v2 or pebble)
         :param list nodes: list of node names that will be setup by pytest xdist
         :param bool http_proxy: if False do not start the HTTP proxy
         :param bool stdout: if True stream all subprocesses stdout to standard stdout
+        :param str dns_server: if set, Pebble/Boulder will use it to resolve domains
         """
         self._construct_acme_xdist(acme_server, nodes)
 
@@ -47,6 +50,7 @@ def __init__(self, acme_server, nodes, http_proxy=True, stdout=False):
         self._workspace = tempfile.mkdtemp()
         self._processes = []
         self._stdout = sys.stdout if stdout else open(os.devnull, 'w')
+        self._dns_server = dns_server
 
     def start(self):
         """Start the test stack"""
@@ -132,13 +136,18 @@ def _prepare_pebble_server(self):
         environ['PEBBLE_AUTHZREUSE'] = '100'
         environ['PEBBLE_ALTERNATE_ROOTS'] = str(PEBBLE_ALTERNATE_ROOTS)
 
-        self._launch_process(
-            [pebble_path, '-config', pebble_config_path, '-dnsserver', '127.0.0.1:8053', '-strict'],
-            env=environ)
+        if self._dns_server:
+            dns_server = self._dns_server
+        else:
+            dns_server = '127.0.0.1:8053'
+            self._launch_process(
+                [challtestsrv_path, '-management', ':{0}'.format(CHALLTESTSRV_PORT),
+                 '-defaultIPv6', '""', '-defaultIPv4', '127.0.0.1', '-http01', '""',
+                 '-tlsalpn01', '""', '-https01', '""'])
 
         self._launch_process(
-            [challtestsrv_path, '-management', ':{0}'.format(CHALLTESTSRV_PORT), '-defaultIPv6', '""',
-             '-defaultIPv4', '127.0.0.1', '-http01', '""', '-tlsalpn01', '""', '-https01', '""'])
+            [pebble_path, '-config', pebble_config_path, '-dnsserver', dns_server, '-strict'],
+            env=environ)
 
         # pebble_ocsp_server is imported here and not at the top of module in order to avoid a useless
         # ImportError, in the case where cryptography dependency is too old to support ocsp, but
@@ -167,6 +176,14 @@ def _prepare_boulder_server(self):
         os.rename(join(instance_path, 'test/rate-limit-policies-b.yml'),
                   join(instance_path, 'test/rate-limit-policies.yml'))
 
+        if self._dns_server:
+            # Change Boulder config to use the provided DNS server
+            with open(join(instance_path, 'test/config/va.json'), 'r') as file_h:
+                config = json.loads(file_h.read())
+            config['va']['dnsResolvers'] = [self._dns_server]
+            with open(join(instance_path, 'test/config/va.json'), 'w') as file_h:
+                file_h.write(json.dumps(config, indent=2, separators=(',', ': ')))
+
         try:
             # Launch the Boulder server
             self._launch_process(['docker-compose', 'up', '--force-recreate'], cwd=instance_path)
@@ -175,10 +192,11 @@ def _prepare_boulder_server(self):
             print('=> Waiting for boulder instance to respond...')
             misc.check_until_timeout(self.acme_xdist['directory_url'], attempts=300)
 
-            # Configure challtestsrv to answer any A record request with ip of the docker host.
-            response = requests.post('http://localhost:{0}/set-default-ipv4'.format(CHALLTESTSRV_PORT),
-                                     json={'ip': '10.77.77.1'})
-            response.raise_for_status()
+            if not self._dns_server:
+                # Configure challtestsrv to answer any A record request with ip of the docker host.
+                response = requests.post('http://localhost:{0}/set-default-ipv4'.format(CHALLTESTSRV_PORT),
+                                         json={'ip': '10.77.77.1'})
+                response.raise_for_status()
         except BaseException:
             # If we failed to set up boulder, print its logs.
             print('=> Boulder setup failed. Boulder logs are:')
@@ -208,14 +226,19 @@ def _launch_process(self, command, cwd=os.getcwd(), env=None, force_stderr=False
 
 
 def main():
-    args = sys.argv[1:]
-    server_type = args[0] if args else 'pebble'
-    possible_values = ('pebble', 'boulder-v1', 'boulder-v2')
-    if server_type not in possible_values:
-        raise ValueError('Invalid server value {0}, should be one of {1}'
-                         .format(server_type, possible_values))
-
-    acme_server = ACMEServer(server_type, [], http_proxy=False, stdout=True)
+    parser = argparse.ArgumentParser(
+        description='CLI tool to start a local instance of Pebble or Boulder CA server.')
+    parser.add_argument('--server-type', '-s',
+                        choices=['pebble', 'boulder-v1', 'boulder-v2'], default='pebble',
+                        help='type of CA server to start: can be Pebble or Boulder '
+                             '(in ACMEv1 or ACMEv2 mode), Pebble is used if not set.')
+    parser.add_argument('--dns-server', '-d',
+                        help='specify the DNS server as `IP:PORT` to use by '
+                             'Pebble; if not specified, a local mock DNS server will be used to '
+                             'resolve domains to localhost.')
+    args = parser.parse_args()
+
+    acme_server = ACMEServer(args.server_type, [], http_proxy=False, stdout=True, dns_server=args.dns_server)
 
     try:
         with acme_server as acme_xdist:

certbot/certbot/_internal/cert_manager.py

@@ -106,8 +106,8 @@ def delete(config):
         return
     for certname in certnames:
         storage.delete_files(config, certname)
-        disp.notification("Deleted all files relating to certificate {0}."
-            .format(certname), pause=False)
+        display_util.notify("Deleted all files relating to certificate {0}."
+                            .format(certname))
 
 ###################
 # Public Helpers

certbot/certbot/_internal/eff.py

@@ -7,6 +7,7 @@
 from acme.magic_typing import Optional  # pylint: disable=unused-import
 
 from certbot import interfaces
+from certbot.display import util as display_util
 from certbot._internal import constants
 from certbot._internal.account import Account  # pylint: disable=unused-import
 from certbot._internal.account import AccountFileStorage
@@ -133,5 +134,4 @@ def _report_failure(reason=None):
         msg.append(' because ')
         msg.append(reason)
     msg.append('. You can try again later by visiting https://act.eff.org.')
-    reporter = zope.component.getUtility(interfaces.IReporter)
-    reporter.add_message(''.join(msg), reporter.LOW_PRIORITY)
+    display_util.notify(''.join(msg))

certbot/certbot/_internal/main.py

@@ -113,12 +113,24 @@ def _get_and_save_cert(le_client, config, domains=None, certname=None, lineage=N
         if lineage is not None:
             # Renewal, where we already know the specific lineage we're
             # interested in
-            logger.info("Renewing an existing certificate")
+            display_util.notify(
+                "{action} for {domains}".format(
+                    action="Simulating renewal of an existing certificate"
+                            if config.dry_run else "Renewing an existing certificate",
+                    domains=display_util.summarize_domain_list(domains or lineage.names())
+                )
+            )
             renewal.renew_cert(config, domains, le_client, lineage)
         else:
             # TREAT AS NEW REQUEST
             assert domains is not None
-            logger.info("Obtaining a new certificate")
+            display_util.notify(
+                "{action} for {domains}".format(
+                    action="Simulating a certificate request" if config.dry_run else
+                           "Requesting a certificate",
+                    domains=display_util.summarize_domain_list(domains)
+                )
+            )
             lineage = le_client.obtain_and_enroll_certificate(domains, certname)
             if lineage is False:
                 raise errors.Error("Certificate could not be obtained")
@@ -163,17 +175,15 @@ def _handle_subset_cert_request(config, domains, cert):
                                        cli_flag="--expand",
                                        force_interactive=True):
         return "renew", cert
-    reporter_util = zope.component.getUtility(interfaces.IReporter)
-    reporter_util.add_message(
+    display_util.notify(
         "To obtain a new certificate that contains these names without "
         "replacing your existing certificate for {0}, you must use the "
         "--duplicate option.{br}{br}"
         "For example:{br}{br}{1} --duplicate {2}".format(
             existing,
             sys.argv[0], " ".join(sys.argv[1:]),
             br=os.linesep
-        ),
-        reporter_util.HIGH_PRIORITY)
+        ))
     raise errors.Error(USER_CANCELLED)
 
 
@@ -542,7 +552,6 @@ def _delete_if_appropriate(config):
         archive dir is found for the specified lineage, etc ...
     """
     display = zope.component.getUtility(interfaces.IDisplay)
-    reporter_util = zope.component.getUtility(interfaces.IReporter)
 
     attempt_deletion = config.delete_after_revoke
     if attempt_deletion is None:
@@ -552,7 +561,6 @@ def _delete_if_appropriate(config):
                 force_interactive=True, default=True)
 
     if not attempt_deletion:
-        reporter_util.add_message("Not deleting revoked certs.", reporter_util.LOW_PRIORITY)
         return
 
     # config.cert_path must have been set
@@ -570,9 +578,8 @@ def _delete_if_appropriate(config):
         cert_manager.match_and_check_overlaps(config, [lambda x: archive_dir],
             lambda x: x.archive_dir, lambda x: x)
     except errors.OverlappingMatchFound:
-        msg = ('Not deleting revoked certs due to overlapping archive dirs. More than '
-                'one lineage is using {0}'.format(archive_dir))
-        reporter_util.add_message(''.join(msg), reporter_util.MEDIUM_PRIORITY)
+        logger.warning("Not deleting revoked certs due to overlapping archive dirs. More than "
+                       "one certificate is using %s", archive_dir)
         return
     except Exception as e:
         msg = ('config.default_archive_dir: {0}, config.live_dir: {1}, archive_dir: {2},'
@@ -625,7 +632,6 @@ def unregister(config, unused_plugins):
     """
     account_storage = account.AccountFileStorage(config)
     accounts = account_storage.find_all()
-    reporter_util = zope.component.getUtility(interfaces.IReporter)
 
     if not accounts:
         return "Could not find existing account to deactivate."
@@ -647,7 +653,7 @@ def unregister(config, unused_plugins):
     # delete local account files
     account_files.delete(config.account)
 
-    reporter_util.add_message("Account deactivated.", reporter_util.MEDIUM_PRIORITY)
+    display_util.notify("Account deactivated.")
     return None
 
 
@@ -698,8 +704,6 @@ def update_account(config, unused_plugins):
     # exist or not.
     account_storage = account.AccountFileStorage(config)
     accounts = account_storage.find_all()
-    reporter_util = zope.component.getUtility(interfaces.IReporter)
-    add_msg = lambda m: reporter_util.add_message(m, reporter_util.MEDIUM_PRIORITY)
 
     if not accounts:
         return "Could not find an existing account to update."
@@ -724,10 +728,11 @@ def update_account(config, unused_plugins):
     account_storage.update_regr(acc, cb_client.acme)
 
     if config.email is None:
-        add_msg("Any contact information associated with this account has been removed.")
+        display_util.notify("Any contact information associated "
+                            "with this account has been removed.")
     else:
         eff.prepare_subscription(config, acc)
-        add_msg("Your e-mail address was updated to {0}.".format(config.email))
+        display_util.notify("Your e-mail address was updated to {0}.".format(config.email))
 
     return None
 

certbot/certbot/_internal/snap_config.py

@@ -49,22 +49,22 @@ def prepare_env(cli_args):
     os.environ['CERTBOT_AUGEAS_PATH'] = '{0}/usr/lib/{1}/libaugeas.so.0'.format(
         os.environ.get('SNAP'), _ARCH_TRIPLET_MAP[snap_arch])
 
-    session = Session()
-    session.mount('http://snapd/', _SnapdAdapter())
-
-    try:
-        response = session.get('http://snapd/v2/connections?snap=certbot&interface=content')
-        response.raise_for_status()
-    except RequestException as e:
-        if isinstance(e, HTTPError) and e.response.status_code == 404:
-            LOGGER.error('An error occurred while fetching Certbot snap plugins: '
-                         'your version of snapd is outdated.')
-            LOGGER.error('Please run "sudo snap install core; sudo snap refresh core" '
-                         'in your terminal and try again.')
-        else:
-            LOGGER.error('An error occurred while fetching Certbot snap plugins: '
-                         'make sure the snapd service is running.')
-        raise e
+    with Session() as session:
+        session.mount('http://snapd/', _SnapdAdapter())
+
+        try:
+            response = session.get('http://snapd/v2/connections?snap=certbot&interface=content')
+            response.raise_for_status()
+        except RequestException as e:
+            if isinstance(e, HTTPError) and e.response.status_code == 404:
+                LOGGER.error('An error occurred while fetching Certbot snap plugins: '
+                             'your version of snapd is outdated.')
+                LOGGER.error('Please run "sudo snap install core; sudo snap refresh core" '
+                             'in your terminal and try again.')
+            else:
+                LOGGER.error('An error occurred while fetching Certbot snap plugins: '
+                             'make sure the snapd service is running.')
+            raise e
 
     data = response.json()
     connections = ['/snap/{0}/current/lib/python3.8/site-packages/'.format(item['slot']['snap'])

certbot/certbot/display/ops.py

@@ -262,17 +262,15 @@ def success_renewal(domains):
 
 
 def success_revocation(cert_path):
-    """Display a box confirming a certificate has been revoked.
+    """Display a message confirming a certificate has been revoked.
 
     :param list cert_path: path to certificate which was revoked.
 
     """
-    z_util(interfaces.IDisplay).notification(
+    display_util.notify(
         "Congratulations! You have successfully revoked the certificate "
-        "that was located at {0}{1}{1}".format(
-            cert_path,
-            os.linesep),
-        pause=False)
+        "that was located at {0}.".format(cert_path)
+    )
 
 
 def _gen_https_names(domains):

certbot/certbot/display/util.py

@@ -16,6 +16,7 @@
 import zope.interface
 import zope.component
 
+from acme.magic_typing import List
 from certbot import errors
 from certbot import interfaces
 from certbot._internal import constants
@@ -97,15 +98,15 @@ def input_with_timeout(prompt=None, timeout=36000.0):
     return line.rstrip('\n')
 
 
-def notify(msg, wrap=True):
+def notify(msg):
     # type: (str, bool) -> None
     """Display a basic status message.
 
     :param str msg: message to display
 
     """
     zope.component.getUtility(interfaces.IDisplay).notification(
-        msg, pause=False, wrap=wrap, decorate=False
+        msg, pause=False, decorate=False, wrap=False
     )
 
 
@@ -633,3 +634,28 @@ def _parens_around_char(label):
 
     """
     return "({first}){rest}".format(first=label[0], rest=label[1:])
+
+
+def summarize_domain_list(domains):
+    # type: (List[str]) -> str
+    """Summarizes a list of domains in the format of:
+        example.com.com and N more domains
+    or if there is are only two domains:
+        example.com and www.example.com
+    or if there is only one domain:
+        example.com
+
+    :param list domains: `str` list of domains
+    :returns: the domain list summary
+    :rtype: str
+    """
+    if not domains:
+        return ""
+
+    l = len(domains)
+    if l == 1:
+        return domains[0]
+    elif l == 2:
+        return " and ".join(domains)
+    else:
+        return "{0} and {1} more domains".format(domains[0], l-1)