diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 24325e461b..6eb5bce7f2 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -347,7 +347,9 @@ functions: script: | set -o xtrace ${PREPARE_SHELL} - PYTHON_BINARY=${PYTHON_BINARY} MOD_WSGI_VERSION=${MOD_WSGI_VERSION} PROJECT_DIRECTORY=${PROJECT_DIRECTORY} bash ${PROJECT_DIRECTORY}/.evergreen/run-mod-wsgi-tests.sh + PYTHON_BINARY=${PYTHON_BINARY} MOD_WSGI_VERSION=${MOD_WSGI_VERSION} \ + MOD_WSGI_EMBEDDED=${MOD_WSGI_EMBEDDED} PROJECT_DIRECTORY=${PROJECT_DIRECTORY} \ + bash ${PROJECT_DIRECTORY}/.evergreen/run-mod-wsgi-tests.sh "run mockupdb tests": - command: shell.exec @@ -1677,6 +1679,28 @@ tasks: TOPOLOGY: "replica_set" - func: "run mod_wsgi tests" + - name: "mod-wsgi-embedded-mode-standalone" + tags: ["mod_wsgi"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + VERSION: "latest" + TOPOLOGY: "server" + - func: "run mod_wsgi tests" + vars: + MOD_WSGI_EMBEDDED: "1" + + - name: "mod-wsgi-embedded-mode-replica-set" + tags: ["mod_wsgi"] + commands: + - func: "bootstrap mongo-orchestration" + vars: + VERSION: "latest" + TOPOLOGY: "replica_set" + - func: "run mod_wsgi tests" + vars: + MOD_WSGI_EMBEDDED: "1" + - name: "no-server" tags: ["no-server"] commands: @@ -3088,6 +3112,8 @@ buildvariants: tasks: - name: "mod-wsgi-standalone" - name: "mod-wsgi-replica-set" + - name: "mod-wsgi-embedded-mode-standalone" + - name: "mod-wsgi-embedded-mode-replica-set" - matrix_name: "mockupdb-tests" matrix_spec: diff --git a/.evergreen/run-mod-wsgi-tests.sh b/.evergreen/run-mod-wsgi-tests.sh index 9a167895f8..afb3f271ae 100644 --- a/.evergreen/run-mod-wsgi-tests.sh +++ b/.evergreen/run-mod-wsgi-tests.sh @@ -18,25 +18,30 @@ fi PYTHON_VERSION=$(${PYTHON_BINARY} -c "import sys; sys.stdout.write('.'.join(str(val) for val in sys.version_info[:2]))") +# Ensure the C extensions are installed. +${PYTHON_BINARY} setup.py build_ext -i + export MOD_WSGI_SO=/opt/python/mod_wsgi/python_version/$PYTHON_VERSION/mod_wsgi_version/$MOD_WSGI_VERSION/mod_wsgi.so export PYTHONHOME=/opt/python/$PYTHON_VERSION +# If MOD_WSGI_EMBEDDED is set use the default embedded mode behavior instead +# of daemon mode (WSGIDaemonProcess). +if [ -n "$MOD_WSGI_EMBEDDED" ]; then + export MOD_WSGI_CONF=mod_wsgi_test_embedded.conf +else + export MOD_WSGI_CONF=mod_wsgi_test.conf +fi cd .. $APACHE -k start -f ${PROJECT_DIRECTORY}/test/mod_wsgi_test/${APACHE_CONFIG} trap '$APACHE -k stop -f ${PROJECT_DIRECTORY}/test/mod_wsgi_test/${APACHE_CONFIG}' EXIT HUP -set +e -wget -t 1 -T 10 -O - "http://localhost:8080${PROJECT_DIRECTORY}" -STATUS=$? -set -e - -# Debug -cat error_log - -if [ $STATUS != 0 ]; then - exit $STATUS -fi +wget -t 1 -T 10 -O - "http://localhost:8080/interpreter1${PROJECT_DIRECTORY}" || (cat error_log && exit 1) +wget -t 1 -T 10 -O - "http://localhost:8080/interpreter2${PROJECT_DIRECTORY}" || (cat error_log && exit 1) -${PYTHON_BINARY} ${PROJECT_DIRECTORY}/test/mod_wsgi_test/test_client.py -n 25000 -t 100 parallel http://localhost:8080${PROJECT_DIRECTORY} +${PYTHON_BINARY} ${PROJECT_DIRECTORY}/test/mod_wsgi_test/test_client.py -n 25000 -t 100 parallel \ + http://localhost:8080/interpreter1${PROJECT_DIRECTORY} http://localhost:8080/interpreter2${PROJECT_DIRECTORY} || \ + (tail -n 100 error_log && exit 1) -${PYTHON_BINARY} ${PROJECT_DIRECTORY}/test/mod_wsgi_test/test_client.py -n 25000 serial http://localhost:8080${PROJECT_DIRECTORY} +${PYTHON_BINARY} ${PROJECT_DIRECTORY}/test/mod_wsgi_test/test_client.py -n 25000 serial \ + http://localhost:8080/interpreter1${PROJECT_DIRECTORY} http://localhost:8080/interpreter2${PROJECT_DIRECTORY} || \ + (tail -n 100 error_log && exit 1) diff --git a/test/mod_wsgi_test/README.rst b/test/mod_wsgi_test/README.rst index 2ea50c9074..2c204f7ac5 100644 --- a/test/mod_wsgi_test/README.rst +++ b/test/mod_wsgi_test/README.rst @@ -15,7 +15,7 @@ Test Matrix PyMongo should be tested with several versions of mod_wsgi and a selection of Python versions. Each combination of mod_wsgi and Python version should -be tested with a standalone and a replica set. ``mod_wsgi_test.wsgi`` +be tested with a standalone and a replica set. ``mod_wsgi_test.py`` detects if the deployment is a replica set and connects to the whole set. Setup @@ -74,31 +74,37 @@ Run the test Run the included ``test_client.py`` script:: python test/mod_wsgi_test/test_client.py -n 2500 -t 100 parallel \ - http://localhost/${WORKSPACE} + http://localhost/interpreter1${WORKSPACE} http://localhost/interpreter2${WORKSPACE} ...where the "n" argument is the total number of requests to make to Apache, and "t" specifies the number of threads. ``WORKSPACE`` is the location of -the PyMongo checkout. +the PyMongo checkout. Note that multiple URLs are passed, each one corresponds +to a different sub interpreter. Run this script again with different arguments to make serial requests:: python test/mod_wsgi_test/test_client.py -n 25000 serial \ - http://localhost/${WORKSPACE} + http://localhost/interpreter1${WORKSPACE} http://localhost/interpreter2${WORKSPACE} The ``test_client.py`` script merely makes HTTP requests to Apache. Its exit code is non-zero if any of its requests fails, for example with an HTTP 500. -The core of the test is in the WSGI script, ``mod_wsgi_test.wsgi``. +The core of the test is in the WSGI script, ``mod_wsgi_test.py``. This script inserts some documents into MongoDB at startup, then queries documents for each HTTP request. If PyMongo is leaking connections and "n" is much greater than the ulimit, the test will fail when PyMongo exhausts its file descriptors. +The script also encodes and decodes all BSON types to ensure that +multiple sub interpreters in the same process are supported. This tests +the workaround added in `PYTHON-569 `_. + Automation ---------- At MongoDB, Inc. we use a continuous integration job that tests each combination in the matrix. The job starts up Apache, starts a single server or replica set, and runs ``test_client.py`` with the proper arguments. +See `run-mod-wsgi-tests.sh `_ diff --git a/test/mod_wsgi_test/apache22amazon.conf b/test/mod_wsgi_test/apache22amazon.conf index d9d892bbe2..7755336b07 100644 --- a/test/mod_wsgi_test/apache22amazon.conf +++ b/test/mod_wsgi_test/apache22amazon.conf @@ -31,4 +31,4 @@ CustomLog ${PWD}/access_log combined Allow from All -Include ${PROJECT_DIRECTORY}/test/mod_wsgi_test/mod_wsgi_test.conf +Include ${PROJECT_DIRECTORY}/test/mod_wsgi_test/${MOD_WSGI_CONF} diff --git a/test/mod_wsgi_test/apache22ubuntu1204.conf b/test/mod_wsgi_test/apache22ubuntu1204.conf index 281c5862c2..9fa4b2060b 100644 --- a/test/mod_wsgi_test/apache22ubuntu1204.conf +++ b/test/mod_wsgi_test/apache22ubuntu1204.conf @@ -26,4 +26,4 @@ CustomLog ${PWD}/access_log combined Allow from All -Include ${PROJECT_DIRECTORY}/test/mod_wsgi_test/mod_wsgi_test.conf +Include ${PROJECT_DIRECTORY}/test/mod_wsgi_test/${MOD_WSGI_CONF} diff --git a/test/mod_wsgi_test/apache24ubuntu161404.conf b/test/mod_wsgi_test/apache24ubuntu161404.conf index f0a1734375..eb5414f0f7 100644 --- a/test/mod_wsgi_test/apache24ubuntu161404.conf +++ b/test/mod_wsgi_test/apache24ubuntu161404.conf @@ -25,4 +25,4 @@ CustomLog ${PWD}/access_log combined Require all granted -Include ${PROJECT_DIRECTORY}/test/mod_wsgi_test/mod_wsgi_test.conf +Include ${PROJECT_DIRECTORY}/test/mod_wsgi_test/${MOD_WSGI_CONF} diff --git a/test/mod_wsgi_test/mod_wsgi_test.conf b/test/mod_wsgi_test/mod_wsgi_test.conf index 6a77c675d5..a5b09e437f 100644 --- a/test/mod_wsgi_test/mod_wsgi_test.conf +++ b/test/mod_wsgi_test/mod_wsgi_test.conf @@ -1,4 +1,4 @@ -# Copyright 2012-2015 MongoDB, Inc. +# Copyright 2012-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,17 +20,13 @@ LoadModule wsgi_module ${MOD_WSGI_SO} WSGISocketPrefix /tmp/ - ServerName localhost - WSGIDaemonProcess mod_wsgi_test processes=1 threads=15 display-name=mod_wsgi_test - WSGIProcessGroup mod_wsgi_test - + # Mount the script twice so that multiple interpreters are used. # For the convenience of unittests, rather than hard-code the location of - # mod_wsgi_test.wsgi, include it in the URL, so - # http://localhost/location-of-pymongo-checkout will work: - - WSGIScriptAliasMatch ^/(.+) $1/test/mod_wsgi_test/mod_wsgi_test.wsgi - + # mod_wsgi_test.py, include it in the URL, so + # http://localhost/interpreter1/location-of-pymongo-checkout will work: + WSGIScriptAliasMatch ^/interpreter1/(.+) $1/test/mod_wsgi_test/mod_wsgi_test.py + WSGIScriptAliasMatch ^/interpreter2/(.+) $1/test/mod_wsgi_test/mod_wsgi_test.py diff --git a/test/mod_wsgi_test/mod_wsgi_test.py b/test/mod_wsgi_test/mod_wsgi_test.py new file mode 100644 index 0000000000..77b5475ab7 --- /dev/null +++ b/test/mod_wsgi_test/mod_wsgi_test.py @@ -0,0 +1,110 @@ +# Copyright 2012-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Minimal test of PyMongo in a WSGI application, see bug PYTHON-353 +""" + +import datetime +import os +import re +import sys +import uuid + +this_path = os.path.dirname(os.path.join(os.getcwd(), __file__)) + +# Location of PyMongo checkout +repository_path = os.path.normpath(os.path.join(this_path, "..", "..")) +sys.path.insert(0, repository_path) + +import bson +import pymongo +from bson.binary import STANDARD, Binary +from bson.code import Code +from bson.codec_options import CodecOptions +from bson.datetime_ms import DatetimeConversion, DatetimeMS +from bson.dbref import DBRef +from bson.objectid import ObjectId +from bson.regex import Regex +from pymongo.mongo_client import MongoClient + +# Ensure the C extensions are installed. +assert bson.has_c() +assert pymongo.has_c() + +OPTS: "CodecOptions[dict]" = CodecOptions( + uuid_representation=STANDARD, datetime_conversion=DatetimeConversion.DATETIME_AUTO +) +client: "MongoClient[dict]" = MongoClient() +# Use a unique collection name for each process: +coll_name = f"test-{uuid.uuid4()}" +collection = client.test.get_collection(coll_name, codec_options=OPTS) +ndocs = 20 +collection.drop() +doc = { + "int32": 2 << 15, + "int64": 2 << 50, + "null": None, + "bool": True, + "float": 1.5, + "str": "string", + "list": [1, 2, 3], + "dict": {"a": 1, "b": 2, "c": 3}, + "datetime": datetime.datetime.fromtimestamp(1690328577.446), + "datetime_ms_out_of_range": DatetimeMS(-2 << 60), + "regex_native": re.compile("regex*"), + "regex_pymongo": Regex("regex*"), + "binary": Binary(b"bytes", 128), + "oid": ObjectId(), + "dbref": DBRef("test", 1), + "code": Code("function(){ return true; }"), + "code_w_scope": Code("return function(){ return x; }", scope={"x": False}), + "bytes": b"bytes", + "uuid": uuid.uuid4(), +} +collection.insert_many([dict(i=i, **doc) for i in range(ndocs)]) +client.close() # Discard main thread's request socket. +client = MongoClient() +collection = client.test.get_collection(coll_name, codec_options=OPTS) + +try: + from mod_wsgi import version as mod_wsgi_version # type: ignore[import] +except: + mod_wsgi_version = None + + +def application(environ, start_response): + results = list(collection.find().batch_size(10)) + assert len(results) == ndocs, f"n_actual={len(results)} n_expected={ndocs}" + # Test encoding and decoding works (for sub interpreter support). + decoded = bson.decode(bson.encode(doc, codec_options=OPTS), codec_options=OPTS) + for key, value in doc.items(): + # Native regex objects are decoded as bson Regex. + if isinstance(value, re.Pattern): + value = Regex.from_native(value) + assert decoded[key] == value, f"failed on doc[{key!r}]: {decoded[key]!r} != {value!r}" + assert isinstance( + decoded[key], type(value) + ), f"failed on doc[{key}]: {decoded[key]!r} is not an instance of {type(value)}" + + output = ( + f" python {sys.version}, mod_wsgi {mod_wsgi_version}," + f" pymongo {pymongo.version}," + f' mod_wsgi.process_group = {environ["mod_wsgi.process_group"]!r}' + f' mod_wsgi.application_group = {environ["mod_wsgi.application_group"]!r}' + f' wsgi.multithread = {environ["wsgi.multithread"]!r}' + "\n" + ) + response_headers = [("Content-Length", str(len(output)))] + start_response("200 OK", response_headers) + return [output.encode("ascii")] diff --git a/test/mod_wsgi_test/mod_wsgi_test.wsgi b/test/mod_wsgi_test/mod_wsgi_test.wsgi deleted file mode 100644 index 7c7b24cb70..0000000000 --- a/test/mod_wsgi_test/mod_wsgi_test.wsgi +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright 2012-2015 MongoDB, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Minimal test of PyMongo in a WSGI application, see bug PYTHON-353 -""" - -import os -import sys - -this_path = os.path.dirname(os.path.join(os.getcwd(), __file__)) - -# Location of PyMongo checkout -repository_path = os.path.normpath(os.path.join(this_path, '..', '..')) -sys.path.insert(0, repository_path) - -import pymongo -from pymongo.hello import HelloCompat # noqa -from pymongo.mongo_client import MongoClient - -client = MongoClient() -collection = client.test.test -ndocs = 20 -collection.drop() -collection.insert_many([{'i': i} for i in range(ndocs)]) -client.close() # Discard main thread's request socket. -client = MongoClient() -collection = client.test.test - -try: - from mod_wsgi import version as mod_wsgi_version -except: - mod_wsgi_version = None - - -def application(environ, start_response): - results = list(collection.find().batch_size(10)) - assert len(results) == ndocs - output = ' python %s, mod_wsgi %s, pymongo %s ' % ( - sys.version, mod_wsgi_version, pymongo.version) - response_headers = [('Content-Length', str(len(output)))] - start_response('200 OK', response_headers) - return [output.encode('ascii')] diff --git a/test/mod_wsgi_test/mod_wsgi_test_embedded.conf b/test/mod_wsgi_test/mod_wsgi_test_embedded.conf new file mode 100644 index 0000000000..306dab4ab6 --- /dev/null +++ b/test/mod_wsgi_test/mod_wsgi_test_embedded.conf @@ -0,0 +1,30 @@ +# Copyright 2023-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Minimal test of PyMongo in an *Embedded mode* WSGI application. + +LoadModule wsgi_module ${MOD_WSGI_SO} + +# Avoid permissions issues +WSGISocketPrefix /tmp/ + + + ServerName localhost + # Mount the script twice so that multiple interpreters are used. + # For the convenience of unittests, rather than hard-code the location of + # mod_wsgi_test.py, include it in the URL, so + # http://localhost/interpreter1/location-of-pymongo-checkout will work: + WSGIScriptAliasMatch ^/interpreter1/(.+) $1/test/mod_wsgi_test/mod_wsgi_test.py + WSGIScriptAliasMatch ^/interpreter2/(.+) $1/test/mod_wsgi_test/mod_wsgi_test.py + diff --git a/test/mod_wsgi_test/test_client.py b/test/mod_wsgi_test/test_client.py index 6d3b299700..9f6f59a7f9 100644 --- a/test/mod_wsgi_test/test_client.py +++ b/test/mod_wsgi_test/test_client.py @@ -15,6 +15,7 @@ """Test client for mod_wsgi application, see bug PYTHON-353.""" import _thread as thread +import random import sys import threading import time @@ -24,7 +25,7 @@ def parse_args(): parser = OptionParser( - """usage: %prog [options] mode url + """usage: %prog [options] mode url [...] mode:\tparallel or serial""" ) @@ -37,7 +38,7 @@ def parse_args(): type="int", dest="nrequests", default=50 * 1000, - help="Number of times to GET the URL, in total", + help="Number of times to GET the URLs, in total", ) parser.add_option( @@ -68,8 +69,9 @@ def parse_args(): ) try: - options, (mode, url) = parser.parse_args() - except ValueError: + options, args = parser.parse_args() + mode, urls = args[0], args[1:] + except (ValueError, IndexError): parser.print_usage() sys.exit(1) @@ -77,10 +79,11 @@ def parse_args(): parser.print_usage() sys.exit(1) - return options, mode, url + return options, mode, urls -def get(url): +def get(urls): + url = random.choice(urls) urlopen(url).read().strip() @@ -89,17 +92,17 @@ class URLGetterThread(threading.Thread): counter_lock = threading.Lock() counter = 0 - def __init__(self, options, url, nrequests_per_thread): + def __init__(self, options, urls, nrequests_per_thread): super().__init__() self.options = options - self.url = url + self.urls = urls self.nrequests_per_thread = nrequests_per_thread self.errors = 0 def run(self): for _i in range(self.nrequests_per_thread): try: - get(url) + get(urls) except Exception as e: print(e) @@ -119,7 +122,7 @@ def run(self): print(counter) -def main(options, mode, url): +def main(options, mode, urls): start_time = time.time() errors = 0 if mode == "parallel": @@ -129,14 +132,14 @@ def main(options, mode, url): print( "Getting {} {} times total in {} threads, " "{} times per thread".format( - url, + urls, nrequests_per_thread * options.nthreads, options.nthreads, nrequests_per_thread, ) ) threads = [ - URLGetterThread(options, url, nrequests_per_thread) for _ in range(options.nthreads) + URLGetterThread(options, urls, nrequests_per_thread) for _ in range(options.nthreads) ] for t in threads: @@ -152,11 +155,11 @@ def main(options, mode, url): else: assert mode == "serial" if options.verbose: - print(f"Getting {url} {options.nrequests} times in one thread") + print(f"Getting {urls} {options.nrequests} times in one thread") for i in range(1, options.nrequests + 1): try: - get(url) + get(urls) except Exception as e: print(e) if not options.continue_: @@ -179,5 +182,5 @@ def main(options, mode, url): if __name__ == "__main__": - options, mode, url = parse_args() - main(options, mode, url) + options, mode, urls = parse_args() + main(options, mode, urls)