Merge branch 'master' into html-table-format

CHANGELOG.md

@@ -6,6 +6,10 @@ The format mostly follows [Keep a Changelog](http://keepachangelog.com/en/1.0.0/
 
 ## UNRELEASED
 
+### Added
+
+- New `enabled` option for all jobs. Set to false to disable a job without needing to remove it or comment it out (Requested in #625 by snowman, contributed in #785 by jamstah)
+
 ### Changed
 
 - Remove EOL'd Python 3.7 (new minimum requirement is Python 3.8), add Python 3.12 testing
@@ -15,6 +19,7 @@ The format mostly follows [Keep a Changelog](http://keepachangelog.com/en/1.0.0/
 - Fix documentation for watching Github tags and releases, again (#723)
 - Fix `--test-reporter` command-line option so `separate` configuration option is no longer ignored when sending test notifications (#772, by marunjar)
 - Fix line height and dark mode regression (#774 reported by kongomongo, PRs #777 and #778 by trevorshannon)
+- Fix compatibility with lxml >= 5 which caused the CSS Selector filter to fail (#783 reported by jamesquilty, PR #786 by jamstah)
 - Fix HTML diff table rendering for long line lengths (#793 by trevorshannon)
 
 ## [2.28] -- 2023-05-03

docs/source/jobs.rst

@@ -169,13 +169,14 @@ Optional keys for all job types
 
 - ``name``: Human-readable name/label of the job
 - ``filter``: :doc:`filters` (if any) to apply to the output (can be tested with ``--test-filter``)
-- ``max_tries``: Number of times to retry fetching the resource
+- ``max_tries``: After this many sequential failed runs, the error will be reported rather than ignored
 - ``diff_tool``: Command to a custom tool for generating diff text
 - ``diff_filter``: :doc:`filters` (if any) to apply to the diff result (can be tested with ``--test-diff-filter``)
 - ``treat_new_as_changed``: Will treat jobs that don't have any historic data as ``CHANGED`` instead of ``NEW`` (and create a diff for new jobs)
 - ``compared_versions``: Number of versions to compare for similarity
 - ``kind`` (redundant): Either ``url``, ``shell`` or ``browser``.  Automatically derived from the unique key (``url``, ``command`` or ``navigate``) of the job type
 - ``user_visible_url``: Different URL to show in reports (e.g. when watched URL is a REST API URL, and you want to show a webpage)
+- ``enabled``: Can be set to false to disable an individual job (default is ``true``)
 
 
 Setting keys for all jobs at once

lib/urlwatch/filters.py

@@ -761,9 +761,9 @@ def _get_filtered_elements(self):
         excluded_elems = None
         if self.filter_kind == 'css':
             selected_elems = CSSSelector(self.expression,
-                                         namespaces=self.namespaces).evaluate(root)
+                                         namespaces=self.namespaces)(root)
             excluded_elems = CSSSelector(self.exclude,
-                                         namespaces=self.namespaces).evaluate(root) if self.exclude else None
+                                         namespaces=self.namespaces)(root) if self.exclude else None
         elif self.filter_kind == 'xpath':
             selected_elems = root.xpath(self.expression, namespaces=self.namespaces)
             excluded_elems = root.xpath(self.exclude, namespaces=self.namespaces) if self.exclude else None

lib/urlwatch/jobs.py

@@ -196,7 +196,7 @@ def ignore_error(self, exception):
 
 class Job(JobBase):
     __required__ = ()
-    __optional__ = ('name', 'filter', 'max_tries', 'diff_tool', 'compared_versions', 'diff_filter', 'treat_new_as_changed', 'user_visible_url')
+    __optional__ = ('name', 'filter', 'max_tries', 'diff_tool', 'compared_versions', 'diff_filter', 'enabled', 'treat_new_as_changed', 'user_visible_url')
 
     # determine if hyperlink "a" tag is used in HtmlReporter
     def location_is_url(self):
@@ -205,6 +205,9 @@ def location_is_url(self):
     def pretty_name(self):
         return self.name if self.name else self.get_location()
 
+    def is_enabled(self):
+        return self.enabled is None or self.enabled
+
 
 class ShellJob(Job):
     """Run a shell command and get its standard output"""

lib/urlwatch/tests/data/disabled-job.yaml

@@ -0,0 +1,6 @@
+name: "1"
+url: "|echo job 1"
+enabled: false
+---
+name: "2"
+url: "|echo job 2"

lib/urlwatch/tests/test_handler.py

@@ -122,6 +122,27 @@ def test_run_watcher():
             cache_storage.close()
 
 
+def test_disabled_job():
+    with teardown_func():
+        urls = os.path.join(here, 'data', 'disabled-job.yaml')
+        config = os.path.join(here, 'data', 'urlwatch.yaml')
+        cache = os.path.join(here, 'data', 'cache.db')
+        hooks = ''
+
+        config_storage = YamlConfigStorage(config)
+        urls_storage = UrlsYaml(urls)
+        cache_storage = CacheMiniDBStorage(cache)
+        try:
+            urlwatch_config = ConfigForTest(config, urls, cache, hooks, True)
+
+            urlwatcher = Urlwatch(urlwatch_config, config_storage, cache_storage, urls_storage)
+            urlwatcher.run_jobs()
+
+            assert len(urlwatcher.report.job_states) == 1
+        finally:
+            cache_storage.close()
+
+
 def test_unserialize_shell_job_without_kind():
     job = JobBase.unserialize({
         'name': 'hoho',

lib/urlwatch/worker.py

@@ -55,7 +55,7 @@ def run_jobs(urlwatcher):
         raise ValueError(f'All job indices must be between 1 and {len(urlwatcher.jobs)}: {urlwatcher.urlwatch_config.joblist}')
     cache_storage = urlwatcher.cache_storage
     jobs = [job.with_defaults(urlwatcher.config_storage.config)
-            for (idx, job) in enumerate(urlwatcher.jobs) if ((idx + 1) in urlwatcher.urlwatch_config.joblist or (not urlwatcher.urlwatch_config.joblist))]
+            for (idx, job) in enumerate(urlwatcher.jobs) if job.is_enabled() and ((idx + 1) in urlwatcher.urlwatch_config.joblist or (not urlwatcher.urlwatch_config.joblist))]
     report = urlwatcher.report
 
     logger.debug('Processing %d jobs (out of %d)', len(jobs), len(urlwatcher.jobs))