Work around requests version conflict (#3469)

PBENCH-1190

The Pbench Agent depends on a version of the `requests` package of at least
2.27 for the `requests.exceptions.JSONDecodeError` exception. We'd like to
require 2.31 as that resolves a CVE, however Pbench Agent supports Python
3.6 which won't work with 2.31. Adding to this mix, the `pquisby` package
requires exactly 2.25.

For now, in order to move forward with Quisby, we remove the version
constraints from `requests` for Pbench and remove the reference to the
`JSONDecodeError` exception.

(Note that a secondary level of mess here is that the Pbench CI installs
*all* our `requirements*.txt` files in the same environment, even though
Pbench Agent and Pbench Server don't really operate in the same environment
and should be completely separate. This currently prevents us from having
incompatible requirements between the two and we ought to change that
going forward.

agent/requirements.txt

@@ -7,6 +7,6 @@ jinja2
 python-daemon
 python-pidfile
 redis
-requests>=2.27.0  # JSONDecodeError, TODO CVE-2023-32681 (2.31)
+requests # TODO CVE-2023-32681 (>=2.31)
 sh
 state-signals>=1.0.1

exec-tests

@@ -164,6 +164,7 @@ if [[ "${subtst:-python}" == "python" ]]; then
     # used which are not compatible with SQLAlchemy 2.x. Since we are not
     # going to switch to 2.x any time soon, we use the big hammer approach
     # to avoid the noise.
+    python3 -m pip list
     SQLALCHEMY_SILENCE_UBER_WARNING=1 PYTHONUNBUFFERED=True _PBENCH_COV_DIR="${_toxenvdir}/cov" ${_ECHO} _time pytest \
         --tb=native \
         ${pytest_jobs_arg} \

lib/pbench/cli/agent/commands/results/move.py

@@ -5,7 +5,6 @@
 from typing import List
 
 import click
-import requests
 
 from pbench.agent.base import BaseCommand
 from pbench.agent.results import CopyResult, MakeResultTb
@@ -103,7 +102,7 @@ def execute(self, single_threaded: bool, delete: bool = True) -> int:
                     if not res.ok:
                         try:
                             msg = res.json()["message"]
-                        except requests.exceptions.JSONDecodeError:
+                        except Exception:
                             msg = res.text if res.text else res.reason
                         raise CopyResult.FileUploadError(msg)
                     if self.context.relay:

lib/pbench/cli/agent/commands/results/push.py

@@ -3,7 +3,6 @@
 from typing import List
 
 import click
-import requests
 
 from pbench.agent.base import BaseCommand
 from pbench.agent.results import CopyResult
@@ -32,7 +31,7 @@ def execute(self) -> int:
 
         try:
             msg = res.json()["message"]
-        except requests.exceptions.JSONDecodeError:
+        except Exception:
             msg = res.text if res.text else res.reason
 
         # dup or other unexpected but non-error status

lib/pbench/test/functional/agent/cli/commands/triggers/test_register_tool_trigger.py

@@ -4,7 +4,7 @@
 def test_pbench_register_tool_trigger_help():
     command = ["pbench-register-tool-trigger", "--help"]
     out, err, exitcode = pytest.helpers.capture(command)
-    assert b"Usage: pbench-register-tool-trigger [OPTIONS]" in out
+    assert b"Usage: pbench-register-tool-trigger" in out
     assert exitcode == 0
 
 

server/requirements.txt

@@ -12,11 +12,12 @@ flask-restful>=0.3.9
 flask-sqlalchemy
 gunicorn
 humanize
+pquisby
 psycopg2
 pyesbulk>=2.0.1
 PyJwt[crypto]
 python-dateutil
-requests>=2.31.0  # CVE-2023-32681
+requests # TODO CVE-2023-32681 (>=2.31.0)
 sdnotify
 sqlalchemy>=1.4.23
 sqlalchemy_utils>=0.37.6