From c2439d2fbe98f5a0ff0bca7fef4671f860b10d57 Mon Sep 17 00:00:00 2001 From: Kathryn Killebrew Date: Fri, 17 Feb 2017 17:25:44 -0500 Subject: [PATCH 01/14] Add docker container for load testing Set up with default new multi-mechanize project. --- docker-compose.yml | 7 +++++++ loadtest/Dockerfile | 24 ++++++++++++++++++++++++ loadtest/climate/config.cfg | 18 ++++++++++++++++++ loadtest/climate/test_scripts/v_user.py | 19 +++++++++++++++++++ loadtest/requirements.txt | 1 + 5 files changed, 69 insertions(+) create mode 100644 loadtest/Dockerfile create mode 100644 loadtest/climate/config.cfg create mode 100644 loadtest/climate/test_scripts/v_user.py create mode 100644 loadtest/requirements.txt diff --git a/docker-compose.yml b/docker-compose.yml index 311a6604..6b26d3e7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -61,3 +61,10 @@ services: - ./nginx/etc/nginx/nginx.conf:/etc/nginx/nginx.conf - ./nginx/etc/nginx/includes/:/etc/nginx/includes/ - ./nginx/etc/nginx/conf.d/app/:/etc/nginx/conf.d/ + + loadtest: + build: + context: ./loadtest + dockerfile: Dockerfile + volumes: + - .:/opt diff --git a/loadtest/Dockerfile b/loadtest/Dockerfile new file mode 100644 index 00000000..b48afdc2 --- /dev/null +++ b/loadtest/Dockerfile @@ -0,0 +1,24 @@ +FROM python:2.7-slim + +MAINTAINER Azavea + +RUN apt-get update && apt-get install -y --no-install-recommends \ + python-dev \ + ipython \ + python-pip \ + python-matplotlib \ + build-essential \ + libfreetype6-dev \ + libpng-dev \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt /tmp/ +RUN pip install --upgrade pip && pip install --no-cache-dir -r /tmp/requirements.txt + +COPY ./ /opt/loadtest + +WORKDIR /opt/loadtest + +ENTRYPOINT ["/bin/bash"] + +CMD ["multimech-run", "climate"] diff --git a/loadtest/climate/config.cfg b/loadtest/climate/config.cfg new file mode 100644 index 00000000..cf4171f0 --- /dev/null +++ b/loadtest/climate/config.cfg @@ -0,0 +1,18 @@ + +[global] +run_time = 30 +rampup = 0 +results_ts_interval = 10 +progress_bar = on +console_logging = off +xml_report = off + + +[user_group-1] +threads = 3 +script = v_user.py + +[user_group-2] +threads = 3 +script = v_user.py + diff --git a/loadtest/climate/test_scripts/v_user.py b/loadtest/climate/test_scripts/v_user.py new file mode 100644 index 00000000..3f805791 --- /dev/null +++ b/loadtest/climate/test_scripts/v_user.py @@ -0,0 +1,19 @@ + +import random +import time + + +class Transaction(object): + def __init__(self): + pass + + def run(self): + r = random.uniform(1, 2) + time.sleep(r) + self.custom_timers['Example_Timer'] = r + + +if __name__ == '__main__': + trans = Transaction() + trans.run() + print trans.custom_timers diff --git a/loadtest/requirements.txt b/loadtest/requirements.txt new file mode 100644 index 00000000..932a5c8c --- /dev/null +++ b/loadtest/requirements.txt @@ -0,0 +1 @@ +multi-mechanize==1.2.0 From 446d58107eb5d5e388e2e1f901ca6afa40f6599a Mon Sep 17 00:00:00 2001 From: Kathryn Killebrew Date: Mon, 20 Feb 2017 10:43:54 -0500 Subject: [PATCH 02/14] Add docker multi-mechanize Add docker container to run multi-mechanize load tests. --- docker-compose.yml | 4 ++++ loadtest/Dockerfile | 5 ++--- loadtest/climate/.gitignore | 1 + 3 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 loadtest/climate/.gitignore diff --git a/docker-compose.yml b/docker-compose.yml index 6b26d3e7..82fba754 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -66,5 +66,9 @@ services: build: context: ./loadtest dockerfile: Dockerfile + environment: + - DISPLAY + - QT_X11_NO_MITSHM=1 volumes: - .:/opt + - /tmp/.X11-unix:/tmp/.X11-unix:rw diff --git a/loadtest/Dockerfile b/loadtest/Dockerfile index b48afdc2..a82cf5ac 100644 --- a/loadtest/Dockerfile +++ b/loadtest/Dockerfile @@ -1,4 +1,4 @@ -FROM python:2.7-slim +FROM python:2.7 MAINTAINER Azavea @@ -10,6 +10,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ libfreetype6-dev \ libpng-dev \ + tcl-dev tk-dev python-tk \ && rm -rf /var/lib/apt/lists/* COPY requirements.txt /tmp/ @@ -20,5 +21,3 @@ COPY ./ /opt/loadtest WORKDIR /opt/loadtest ENTRYPOINT ["/bin/bash"] - -CMD ["multimech-run", "climate"] diff --git a/loadtest/climate/.gitignore b/loadtest/climate/.gitignore new file mode 100644 index 00000000..fbca2253 --- /dev/null +++ b/loadtest/climate/.gitignore @@ -0,0 +1 @@ +results/ From fe95e5dc01a253b84078c4a4bb6fd7a971918a7d Mon Sep 17 00:00:00 2001 From: Kathryn Killebrew Date: Mon, 20 Feb 2017 13:03:32 -0500 Subject: [PATCH 03/14] Switch to locust for load testing Use locust.io load testing library. Set up with docker container. Web UI runs on port 8089. --- docker-compose.yml | 7 +++--- loadtest/Dockerfile | 9 ++++--- loadtest/climate/.gitignore | 1 - loadtest/climate/config.cfg | 18 -------------- loadtest/climate/test_scripts/v_user.py | 19 --------------- loadtest/locustfile.py | 32 +++++++++++++++++++++++++ loadtest/requirements.txt | 2 +- 7 files changed, 41 insertions(+), 47 deletions(-) delete mode 100644 loadtest/climate/.gitignore delete mode 100644 loadtest/climate/config.cfg delete mode 100644 loadtest/climate/test_scripts/v_user.py create mode 100644 loadtest/locustfile.py diff --git a/docker-compose.yml b/docker-compose.yml index 82fba754..79494154 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -66,9 +66,10 @@ services: build: context: ./loadtest dockerfile: Dockerfile + ports: + - "8089:8089" environment: - - DISPLAY - - QT_X11_NO_MITSHM=1 + - API_TOKEN + - API_HOST=https://api.staging.futurefeelslike.com volumes: - .:/opt - - /tmp/.X11-unix:/tmp/.X11-unix:rw diff --git a/loadtest/Dockerfile b/loadtest/Dockerfile index a82cf5ac..d4b380ad 100644 --- a/loadtest/Dockerfile +++ b/loadtest/Dockerfile @@ -3,14 +3,9 @@ FROM python:2.7 MAINTAINER Azavea RUN apt-get update && apt-get install -y --no-install-recommends \ - python-dev \ ipython \ python-pip \ - python-matplotlib \ build-essential \ - libfreetype6-dev \ - libpng-dev \ - tcl-dev tk-dev python-tk \ && rm -rf /var/lib/apt/lists/* COPY requirements.txt /tmp/ @@ -20,4 +15,8 @@ COPY ./ /opt/loadtest WORKDIR /opt/loadtest +EXPOSE 8089 + ENTRYPOINT ["/bin/bash"] + +CMD ["-c", "locust --host=${API_HOST}"] diff --git a/loadtest/climate/.gitignore b/loadtest/climate/.gitignore deleted file mode 100644 index fbca2253..00000000 --- a/loadtest/climate/.gitignore +++ /dev/null @@ -1 +0,0 @@ -results/ diff --git a/loadtest/climate/config.cfg b/loadtest/climate/config.cfg deleted file mode 100644 index cf4171f0..00000000 --- a/loadtest/climate/config.cfg +++ /dev/null @@ -1,18 +0,0 @@ - -[global] -run_time = 30 -rampup = 0 -results_ts_interval = 10 -progress_bar = on -console_logging = off -xml_report = off - - -[user_group-1] -threads = 3 -script = v_user.py - -[user_group-2] -threads = 3 -script = v_user.py - diff --git a/loadtest/climate/test_scripts/v_user.py b/loadtest/climate/test_scripts/v_user.py deleted file mode 100644 index 3f805791..00000000 --- a/loadtest/climate/test_scripts/v_user.py +++ /dev/null @@ -1,19 +0,0 @@ - -import random -import time - - -class Transaction(object): - def __init__(self): - pass - - def run(self): - r = random.uniform(1, 2) - time.sleep(r) - self.custom_timers['Example_Timer'] = r - - -if __name__ == '__main__': - trans = Transaction() - trans.run() - print trans.custom_timers diff --git a/loadtest/locustfile.py b/loadtest/locustfile.py new file mode 100644 index 00000000..57f0471e --- /dev/null +++ b/loadtest/locustfile.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python + +import os + +from locust import HttpLocust, TaskSet, task + + +class UserBehavior(TaskSet): + def on_start(self): + """ on_start is called when a Locust start before any task is scheduled """ + print('starting locust...') + token = os.getenv('API_TOKEN') + if not token: + raise ValueError('Must set API_TOKEN on environment to run load tests.') + self.headers = {'Authorization': 'Token {token}'.format(token=token)} + + @task(2) + def index(self): + resp = self.client.get("/", headers=self.headers) + print(resp) + + @task(1) + def profile(self): + resp = self.client.get("/api/", headers=self.headers) + print(resp) + print(resp.json()) + + +class WebsiteUser(HttpLocust): + task_set = UserBehavior + min_wait = 5000 + max_wait = 9000 diff --git a/loadtest/requirements.txt b/loadtest/requirements.txt index 932a5c8c..e5e00a7a 100644 --- a/loadtest/requirements.txt +++ b/loadtest/requirements.txt @@ -1 +1 @@ -multi-mechanize==1.2.0 +locustio==0.8a2 From 3dcd6080eb245be636f6ee9a8198561be86a91ae Mon Sep 17 00:00:00 2001 From: Kathryn Killebrew Date: Mon, 20 Feb 2017 13:11:57 -0500 Subject: [PATCH 04/14] Add endpoints to load test --- loadtest/locustfile.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/loadtest/locustfile.py b/loadtest/locustfile.py index 57f0471e..7ec681f8 100644 --- a/loadtest/locustfile.py +++ b/loadtest/locustfile.py @@ -14,16 +14,25 @@ def on_start(self): raise ValueError('Must set API_TOKEN on environment to run load tests.') self.headers = {'Authorization': 'Token {token}'.format(token=token)} - @task(2) + @task(1) def index(self): resp = self.client.get("/", headers=self.headers) - print(resp) @task(1) - def profile(self): + def api_main(self): resp = self.client.get("/api/", headers=self.headers) - print(resp) - print(resp.json()) + + @task(2) + def scenarios(self): + resp = self.client.get("/api/scenario/", headers=self.headers) + + @task(2) + def cities(self): + resp = self.client.get("/api/city/", headers=self.headers) + + @task(2) + def projects(self): + resp = self.client.get("/api/project/", headers=self.headers) class WebsiteUser(HttpLocust): From 60c61705e2c32978412c04cfe99912263284c201 Mon Sep 17 00:00:00 2001 From: Kathryn Killebrew Date: Mon, 20 Feb 2017 13:35:19 -0500 Subject: [PATCH 05/14] Add load tests --- loadtest/locustfile.py | 51 +++++++++++++++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 8 deletions(-) diff --git a/loadtest/locustfile.py b/loadtest/locustfile.py index 7ec681f8..02a545bf 100644 --- a/loadtest/locustfile.py +++ b/loadtest/locustfile.py @@ -16,23 +16,58 @@ def on_start(self): @task(1) def index(self): - resp = self.client.get("/", headers=self.headers) + self.client.get('/', headers=self.headers) @task(1) def api_main(self): - resp = self.client.get("/api/", headers=self.headers) + self.client.get('/api/', headers=self.headers) - @task(2) + @task(1) def scenarios(self): - resp = self.client.get("/api/scenario/", headers=self.headers) + self.client.get('/api/scenario/', headers=self.headers) + + @task(1) + def scenario_details(self): + self.client.get('/api/scenario/RCP85/', headers=self.headers) - @task(2) + @task(1) def cities(self): - resp = self.client.get("/api/city/", headers=self.headers) + self.client.get('/api/city/', headers=self.headers) + + @task(1) + def city_data(self): + self.client.get('/api/climate-data/1/RCP85/', headers=self.headers) - @task(2) + @task(1) def projects(self): - resp = self.client.get("/api/project/", headers=self.headers) + self.client.get('/api/project/', headers=self.headers) + + @task(1) + def climate_models(self): + self.client.get('/api/climate-model/', headers=self.headers) + + @task(1) + def climate_model_detail(self): + self.client.get('/api/climate-model/ACCESS1-0/', headers=self.headers) + + @task(1) + def indicator_list(self): + self.client.get('/api/indicator/', headers=self.headers) + + @task(1) + def avg_high_temp(self): + self.client.get('/api/climate-data/14/RCP45/indicator/average_high_temperature/', + headers=self.headers) + + @task(1) + def avg_high_temp_monthly(self): + self.client.get('/api/climate-data/14/RCP45/indicator/average_high_temperature/', + params={'time_aggregation': 'monthly'}, headers=self.headers) + + @task(1) + def avg_high_temp_daily(self): + self.client.get('/api/climate-data/14/RCP45/indicator/average_high_temperature/', + params={'time_aggregation': 'daily'}, headers=self.headers) class WebsiteUser(HttpLocust): From 47c262e64b6db7c17a7b2c823c539308389af411 Mon Sep 17 00:00:00 2001 From: Kathryn Killebrew Date: Mon, 20 Feb 2017 15:33:20 -0500 Subject: [PATCH 06/14] Query for each indicator and time aggregation Query indicators endpoint and use result to build load test queries for each indicator / valid time aggregation. --- loadtest/locustfile.py | 50 ++++++++++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/loadtest/locustfile.py b/loadtest/locustfile.py index 02a545bf..5b7a81f8 100644 --- a/loadtest/locustfile.py +++ b/loadtest/locustfile.py @@ -4,6 +4,32 @@ from locust import HttpLocust, TaskSet, task +CITY_ID = os.getenv('LOAD_TEST_CITY_ID') or 1 +SCENARIO = os.getenv('LOAD_TEST_SCENARIO') or 'RCP85' + + +def general_indicator_query(locust_instance, indicator, params): + locust_instance.client.get('/api/climate-data/{city}/{scenario}/indicator/{indicator}'.format( + city=CITY_ID, scenario=SCENARIO, indicator=indicator), + headers=locust_instance.headers, + params=params) + + +def build_indicator_queries(locust_instance): + """ + Add a load test query for each indciator / time aggregation + """ + indicators = locust_instance.client.get('/api/indicator/', + headers=locust_instance.headers).json() + for indicator in indicators: + indicator_name = indicator['name'] + for agg in indicator['valid_aggregations']: + if agg != 'custom': + locust_instance.schedule_task(general_indicator_query, + kwargs={ + 'indicator': indicator_name, + 'params': {'time_aggregation': agg}}) + class UserBehavior(TaskSet): def on_start(self): @@ -13,6 +39,9 @@ def on_start(self): if not token: raise ValueError('Must set API_TOKEN on environment to run load tests.') self.headers = {'Authorization': 'Token {token}'.format(token=token)} + print('adding indciator queries...') + build_indicator_queries(self) + print('ready to go!') @task(1) def index(self): @@ -28,7 +57,7 @@ def scenarios(self): @task(1) def scenario_details(self): - self.client.get('/api/scenario/RCP85/', headers=self.headers) + self.client.get('/api/scenario/{scenario}/'.format(scenario=SCENARIO), headers=self.headers) @task(1) def cities(self): @@ -36,7 +65,9 @@ def cities(self): @task(1) def city_data(self): - self.client.get('/api/climate-data/1/RCP85/', headers=self.headers) + self.client.get('/api/climate-data/{city}/{scenario}/'.format(city=CITY_ID, + scenario=SCENARIO), + headers=self.headers) @task(1) def projects(self): @@ -54,21 +85,6 @@ def climate_model_detail(self): def indicator_list(self): self.client.get('/api/indicator/', headers=self.headers) - @task(1) - def avg_high_temp(self): - self.client.get('/api/climate-data/14/RCP45/indicator/average_high_temperature/', - headers=self.headers) - - @task(1) - def avg_high_temp_monthly(self): - self.client.get('/api/climate-data/14/RCP45/indicator/average_high_temperature/', - params={'time_aggregation': 'monthly'}, headers=self.headers) - - @task(1) - def avg_high_temp_daily(self): - self.client.get('/api/climate-data/14/RCP45/indicator/average_high_temperature/', - params={'time_aggregation': 'daily'}, headers=self.headers) - class WebsiteUser(HttpLocust): task_set = UserBehavior From 99bae94e5626fd199b24bdcfa1f61f399fff0c58 Mon Sep 17 00:00:00 2001 From: Kathryn Killebrew Date: Mon, 20 Feb 2017 17:08:40 -0500 Subject: [PATCH 07/14] Schedule indicator load test queries repeatably Add tasks to locust load tests for indicator queries such that they may execute more than once. Also switch to python 3 (for functools.partialmethod, though not necessary.) --- loadtest/Dockerfile | 6 +++--- loadtest/locustfile.py | 45 ++++++++++++++++++++------------------- loadtest/requirements.txt | 2 ++ 3 files changed, 28 insertions(+), 25 deletions(-) diff --git a/loadtest/Dockerfile b/loadtest/Dockerfile index d4b380ad..e76b0426 100644 --- a/loadtest/Dockerfile +++ b/loadtest/Dockerfile @@ -1,15 +1,15 @@ -FROM python:2.7 +FROM python:3-slim MAINTAINER Azavea RUN apt-get update && apt-get install -y --no-install-recommends \ ipython \ - python-pip \ + python3-pip \ build-essential \ && rm -rf /var/lib/apt/lists/* COPY requirements.txt /tmp/ -RUN pip install --upgrade pip && pip install --no-cache-dir -r /tmp/requirements.txt +RUN pip3 install --upgrade pip && pip3 install --no-cache-dir -r /tmp/requirements.txt COPY ./ /opt/loadtest diff --git a/loadtest/locustfile.py b/loadtest/locustfile.py index 5b7a81f8..26a4327e 100644 --- a/loadtest/locustfile.py +++ b/loadtest/locustfile.py @@ -1,5 +1,6 @@ -#!/usr/bin/env python +#!/usr/bin/python3 +from functools import partial import os from locust import HttpLocust, TaskSet, task @@ -8,30 +9,29 @@ SCENARIO = os.getenv('LOAD_TEST_SCENARIO') or 'RCP85' -def general_indicator_query(locust_instance, indicator, params): - locust_instance.client.get('/api/climate-data/{city}/{scenario}/indicator/{indicator}'.format( - city=CITY_ID, scenario=SCENARIO, indicator=indicator), - headers=locust_instance.headers, - params=params) +def general_indicator_query(locust_object, indicator, params): + locust_object.client.get('/api/climate-data/{city}/{scenario}/indicator/{indicator}'.format( + city=CITY_ID, scenario=SCENARIO, indicator=indicator), + headers=locust_object.headers, + params=params) -def build_indicator_queries(locust_instance): - """ - Add a load test query for each indciator / time aggregation - """ - indicators = locust_instance.client.get('/api/indicator/', - headers=locust_instance.headers).json() - for indicator in indicators: - indicator_name = indicator['name'] - for agg in indicator['valid_aggregations']: - if agg != 'custom': - locust_instance.schedule_task(general_indicator_query, - kwargs={ - 'indicator': indicator_name, - 'params': {'time_aggregation': agg}}) +class UserBehavior(TaskSet): + def build_indicator_queries(self): + """ + Add a load test query for each indciator / time aggregation + """ + indicators = self.client.get('/api/indicator/', + headers=self.headers).json() + for indicator in indicators: + indicator_name = indicator['name'] + for agg in indicator['valid_aggregations']: + if agg != 'custom': + self.tasks.append(partial(general_indicator_query, + indicator=indicator_name, + params={'time_aggregation': agg})) -class UserBehavior(TaskSet): def on_start(self): """ on_start is called when a Locust start before any task is scheduled """ print('starting locust...') @@ -40,7 +40,8 @@ def on_start(self): raise ValueError('Must set API_TOKEN on environment to run load tests.') self.headers = {'Authorization': 'Token {token}'.format(token=token)} print('adding indciator queries...') - build_indicator_queries(self) + indicator_tasks = TaskSet(self) + self.build_indicator_queries() print('ready to go!') @task(1) diff --git a/loadtest/requirements.txt b/loadtest/requirements.txt index e5e00a7a..8ce828d7 100644 --- a/loadtest/requirements.txt +++ b/loadtest/requirements.txt @@ -1 +1,3 @@ locustio==0.8a2 +ipdb==0.10.2 +ipython==5.2.2 From d8da90a5a51e98a3c343da3b8efc716b5e6fc16f Mon Sep 17 00:00:00 2001 From: Kathryn Killebrew Date: Mon, 20 Feb 2017 17:20:27 -0500 Subject: [PATCH 08/14] Skip threshold indicators Which have additional required parameters. --- loadtest/locustfile.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/loadtest/locustfile.py b/loadtest/locustfile.py index 26a4327e..855284b5 100644 --- a/loadtest/locustfile.py +++ b/loadtest/locustfile.py @@ -26,6 +26,8 @@ def build_indicator_queries(self): headers=self.headers).json() for indicator in indicators: indicator_name = indicator['name'] + if indicator_name.endswith('threshold'): + continue # TODO: threshold indicators have other required parameters; send them for agg in indicator['valid_aggregations']: if agg != 'custom': self.tasks.append(partial(general_indicator_query, From 7370f3740df79e6d1f25bea15476f2cd3eb24dd0 Mon Sep 17 00:00:00 2001 From: Kathryn Killebrew Date: Tue, 21 Feb 2017 13:14:59 -0500 Subject: [PATCH 09/14] Query threshold parameters Do not send in time aggregation, due to errors and likelihood of that parameter being removed. Query other indicators still for each time aggregation. Rename indicator query labels to be indicator name + aggregation level, for readability. --- loadtest/locustfile.py | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/loadtest/locustfile.py b/loadtest/locustfile.py index 855284b5..ba2c80df 100644 --- a/loadtest/locustfile.py +++ b/loadtest/locustfile.py @@ -8,12 +8,20 @@ CITY_ID = os.getenv('LOAD_TEST_CITY_ID') or 1 SCENARIO = os.getenv('LOAD_TEST_SCENARIO') or 'RCP85' +DEFAULT_THRESHOLD_PARAMS = { + 'threshold': 100, + 'threshold_comparator': 'lte', + 'threshold_units': 'F' +} + def general_indicator_query(locust_object, indicator, params): - locust_object.client.get('/api/climate-data/{city}/{scenario}/indicator/{indicator}'.format( - city=CITY_ID, scenario=SCENARIO, indicator=indicator), - headers=locust_object.headers, - params=params) + locust_object.client.get('/api/climate-data/{city}/{scenario}/indicator/{indicator}'.format( + city=CITY_ID, scenario=SCENARIO, indicator=indicator), + headers=locust_object.headers, + params=params, + name='{indicator} {agg}'.format(indicator=indicator, + agg=params.get('time_aggregation', ''))) class UserBehavior(TaskSet): @@ -27,12 +35,16 @@ def build_indicator_queries(self): for indicator in indicators: indicator_name = indicator['name'] if indicator_name.endswith('threshold'): - continue # TODO: threshold indicators have other required parameters; send them - for agg in indicator['valid_aggregations']: - if agg != 'custom': - self.tasks.append(partial(general_indicator_query, - indicator=indicator_name, - params={'time_aggregation': agg})) + self.tasks.append(partial(general_indicator_query, + indicator=indicator_name, + params=DEFAULT_THRESHOLD_PARAMS)) + else: + # for non-threshold indicators, test all non-custom aggregation levels + for agg in indicator['valid_aggregations']: + if agg != 'custom': + self.tasks.append(partial(general_indicator_query, + indicator=indicator_name, + params={'time_aggregation': agg})) def on_start(self): """ on_start is called when a Locust start before any task is scheduled """ From e5fb3b3cb442d4d6dbf92bf1eda9a6dc32736405 Mon Sep 17 00:00:00 2001 From: Kathryn Killebrew Date: Tue, 21 Feb 2017 14:50:01 -0500 Subject: [PATCH 10/14] Use appropriate threshold unit for precipitation Currently only non-temperature threshold indicator. --- loadtest/locustfile.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/loadtest/locustfile.py b/loadtest/locustfile.py index ba2c80df..ad1ab81c 100644 --- a/loadtest/locustfile.py +++ b/loadtest/locustfile.py @@ -35,9 +35,12 @@ def build_indicator_queries(self): for indicator in indicators: indicator_name = indicator['name'] if indicator_name.endswith('threshold'): + params = DEFAULT_THRESHOLD_PARAMS + if indicator.find('precepitation') > -1: + params['threshold_units'] = 'in' self.tasks.append(partial(general_indicator_query, indicator=indicator_name, - params=DEFAULT_THRESHOLD_PARAMS)) + params=params)) else: # for non-threshold indicators, test all non-custom aggregation levels for agg in indicator['valid_aggregations']: From bce667983a288d7bde629326aa18c2f99791476e Mon Sep 17 00:00:00 2001 From: Kathryn Killebrew Date: Tue, 21 Feb 2017 16:24:42 -0500 Subject: [PATCH 11/14] Open locust port on VM So docker container may be run within VM. --- Vagrantfile | 3 +++ loadtest/locustfile.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index b37ee5d3..a18c3c19 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -32,6 +32,9 @@ Vagrant.configure(2) do |config| config.vm.network :forwarded_port, guest: 8080, host: Integer(ENV.fetch("CC_PORT_8080", 8080)) config.vm.network :forwarded_port, guest: 8088, host: Integer(ENV.fetch("CC_PORT_8088", 8088)) + # locust port + config.vm.network :forwarded_port, guest: 8089, host: Integer(ENV.fetch("CC_PORT_8080", 8089)) + # django runserver/debugging config.vm.network :forwarded_port, guest: 8082, host: Integer(ENV.fetch("CC_PORT_8082", 8082)) diff --git a/loadtest/locustfile.py b/loadtest/locustfile.py index ad1ab81c..55a33fb1 100644 --- a/loadtest/locustfile.py +++ b/loadtest/locustfile.py @@ -36,7 +36,7 @@ def build_indicator_queries(self): indicator_name = indicator['name'] if indicator_name.endswith('threshold'): params = DEFAULT_THRESHOLD_PARAMS - if indicator.find('precepitation') > -1: + if indicator_name.find('precepitation') > -1: params['threshold_units'] = 'in' self.tasks.append(partial(general_indicator_query, indicator=indicator_name, From 2d1a18be5a3d71d3f7ac54c2b2c22b8f7ed9a491 Mon Sep 17 00:00:00 2001 From: Kathryn Killebrew Date: Tue, 21 Feb 2017 17:10:16 -0500 Subject: [PATCH 12/14] Append random query parameters during load test To defeat caching, append randomized parameter with UUID to indicator queries. --- loadtest/locustfile.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/loadtest/locustfile.py b/loadtest/locustfile.py index 55a33fb1..b5e51dff 100644 --- a/loadtest/locustfile.py +++ b/loadtest/locustfile.py @@ -2,6 +2,7 @@ from functools import partial import os +import uuid from locust import HttpLocust, TaskSet, task @@ -16,6 +17,8 @@ def general_indicator_query(locust_object, indicator, params): + # append a randomized extra parameter to the query to defeat caching + params['random'] = uuid.uuid4() locust_object.client.get('/api/climate-data/{city}/{scenario}/indicator/{indicator}'.format( city=CITY_ID, scenario=SCENARIO, indicator=indicator), headers=locust_object.headers, From a17ba17427d61f9e01e186fd2a32f3abe4ce1cb3 Mon Sep 17 00:00:00 2001 From: Kathryn Killebrew Date: Tue, 21 Feb 2017 18:04:08 -0500 Subject: [PATCH 13/14] Append randomized parameter to non-indicator queries --- loadtest/locustfile.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/loadtest/locustfile.py b/loadtest/locustfile.py index b5e51dff..f17b4f10 100644 --- a/loadtest/locustfile.py +++ b/loadtest/locustfile.py @@ -29,6 +29,11 @@ def general_indicator_query(locust_object, indicator, params): class UserBehavior(TaskSet): + def get_api_url(self, url): + """ Helper for querying API with authorization header and random parameter to defeat cache + """ + self.client.get(url, headers=self.headers, params={'random': uuid.uuid4()}, name=url) + def build_indicator_queries(self): """ Add a load test query for each indciator / time aggregation @@ -66,45 +71,44 @@ def on_start(self): @task(1) def index(self): - self.client.get('/', headers=self.headers) + self.get_api_url('/') @task(1) def api_main(self): - self.client.get('/api/', headers=self.headers) + self.get_api_url('/api/') @task(1) def scenarios(self): - self.client.get('/api/scenario/', headers=self.headers) + self.get_api_url('/api/scenario/') @task(1) def scenario_details(self): - self.client.get('/api/scenario/{scenario}/'.format(scenario=SCENARIO), headers=self.headers) + self.get_api_url('/api/scenario/{scenario}/'.format(scenario=SCENARIO)) @task(1) def cities(self): - self.client.get('/api/city/', headers=self.headers) + self.get_api_url('/api/city/') @task(1) def city_data(self): - self.client.get('/api/climate-data/{city}/{scenario}/'.format(city=CITY_ID, - scenario=SCENARIO), - headers=self.headers) + self.get_api_url('/api/climate-data/{city}/{scenario}/'.format(city=CITY_ID, + scenario=SCENARIO)) @task(1) def projects(self): - self.client.get('/api/project/', headers=self.headers) + self.get_api_url('/api/project/') @task(1) def climate_models(self): - self.client.get('/api/climate-model/', headers=self.headers) + self.get_api_url('/api/climate-model/') @task(1) def climate_model_detail(self): - self.client.get('/api/climate-model/ACCESS1-0/', headers=self.headers) + self.get_api_url('/api/climate-model/ACCESS1-0/') @task(1) def indicator_list(self): - self.client.get('/api/indicator/', headers=self.headers) + self.get_api_url('/api/indicator/') class WebsiteUser(HttpLocust): From e2926406b3491628521941f780490104a67b89eb Mon Sep 17 00:00:00 2001 From: Kathryn Killebrew Date: Wed, 22 Feb 2017 11:58:37 -0500 Subject: [PATCH 14/14] Document load testing Describe how to run load tests in README. --- README.rst | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/README.rst b/README.rst index 2cc6b200..6800b2de 100644 --- a/README.rst +++ b/README.rst @@ -32,6 +32,28 @@ Run Django tests with:: ./scripts/console django './manage.py test --settings climate_change_api.settings_test' +Load Testing +------------ + +The ``loadtest`` Docker container can be used to test API query response times using `locust `_. + +First set the environment variable ``API_TOKEN`` within the VM to a valid user token with:: + + export API_TOKEN= + +Optionally, the target server to test may be configured to target the local instance with:: + + export API_HOST=http://localhost:8082 + +By default, the staging server will be targeted. + +Then start the Docker container with:: + + docker-compose up loadtest + +Naviagate to http://localhost:8089 and start tests by setting the swarm and hatch rate (1 for each is fine). To stop tests, click the red button in the web UI (or halt the container). + + Documentation -------------