diff --git a/docker/pythonpath_dev/superset_config.py b/docker/pythonpath_dev/superset_config.py index 08dd47a1275d8..84c1dc58ab502 100644 --- a/docker/pythonpath_dev/superset_config.py +++ b/docker/pythonpath_dev/superset_config.py @@ -70,12 +70,12 @@ def get_env_variable(var_name: str, default: Optional[str] = None) -> str: RESULTS_BACKEND = FileSystemCache("/app/superset_home/sqllab") CACHE_CONFIG = { - 'CACHE_TYPE': 'redis', - 'CACHE_DEFAULT_TIMEOUT': 300, - 'CACHE_KEY_PREFIX': 'superset_', - 'CACHE_REDIS_HOST': REDIS_HOST, - 'CACHE_REDIS_PORT': REDIS_PORT, - 'CACHE_REDIS_DB': REDIS_RESULTS_DB, + "CACHE_TYPE": "redis", + "CACHE_DEFAULT_TIMEOUT": 300, + "CACHE_KEY_PREFIX": "superset_", + "CACHE_REDIS_HOST": REDIS_HOST, + "CACHE_REDIS_PORT": REDIS_PORT, + "CACHE_REDIS_DB": REDIS_RESULTS_DB, } DATA_CACHE_CONFIG = CACHE_CONFIG diff --git a/superset/tasks/cache.py b/superset/tasks/cache.py index f20beda9e1279..137ec068e8843 100644 --- a/superset/tasks/cache.py +++ b/superset/tasks/cache.py @@ -14,14 +14,13 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -import json import logging from typing import Any, Dict, List, Optional, Union from urllib import request from urllib.error import URLError -from celery.utils.log import get_task_logger from celery.beat import SchedulingError +from celery.utils.log import get_task_logger from sqlalchemy import and_, func from superset import app, db, security_manager @@ -40,10 +39,8 @@ def get_url(chart: Slice, dashboard: Optional[Dashboard] = None) -> str: """Return external URL for warming up a given chart/table cache.""" with app.test_request_context(): - baseurl = ( - "{WEBDRIVER_BASEURL}".format(**app.config) - ) - url = f"{baseurl}/superset/warm_up_cache/?slice_id={chart.id}" + baseurl = "{WEBDRIVER_BASEURL}".format(**app.config) + url = f"{baseurl}superset/warm_up_cache/?slice_id={chart.id}" if dashboard: url += f"&dashboard_id={dashboard.id}" return url @@ -219,27 +216,26 @@ def get_urls(self) -> List[str]: @celery_app.task(name="fetch_url") -def fetch_url(url, headers): +def fetch_url(url: str, headers: Dict[str, str]) -> Dict[str, str]: """ - Celery job to fetch url + Celery job to fetch url """ result = {} try: logger.info("Fetching %s", url) req = request.Request(url, headers=headers) - response = request.urlopen(req, timeout=600) # pylint: disable=consider-using-with - logger.info(f"Fetching {url} {response.code}") + response = request.urlopen( # pylint: disable=consider-using-with + req, timeout=600 + ) + logger.info("Fetched %s, status code: %s", url, response.code) if response.code == 200: - result = { - "success": url, - "response": response.read().decode('utf-8') - } + result = {"success": url, "response": response.read().decode("utf-8")} else: result = {"error": url, "status_code": response.code} logger.error("Error fetching %s, status code: %s", url, response.code) - except URLError as e: + except URLError as err: logger.exception("Error warming up cache!") - result = {"error": url, "exception": str(e)} + result = {"error": url, "exception": str(err)} return result @@ -272,9 +268,7 @@ def cache_warmup( logger.exception(message) return message - user = security_manager.get_user_by_username( - app.config["THUMBNAIL_SELENIUM_USER"] - ) + user = security_manager.get_user_by_username(app.config["THUMBNAIL_SELENIUM_USER"]) cookies = MachineAuthProvider.get_auth_cookies(user) headers = {"Cookie": f"session={cookies.get('session', '')}"} diff --git a/tests/integration_tests/strategy_tests.py b/tests/integration_tests/strategy_tests.py index aacc681660db6..f31489bb04569 100644 --- a/tests/integration_tests/strategy_tests.py +++ b/tests/integration_tests/strategy_tests.py @@ -40,6 +40,7 @@ DashboardTagsStrategy, TopNDashboardsStrategy, ) +from superset.utils.urls import get_url_host from .base_tests import SupersetTestCase from .dashboard_utils import create_dashboard, create_slice, create_table_metadata @@ -48,7 +49,6 @@ load_unicode_data, ) -URL_PREFIX = "http://0.0.0.0:8081/" mock_positions = { "DASHBOARD_VERSION_KEY": "v2", @@ -68,7 +68,6 @@ class TestCacheWarmUp(SupersetTestCase): - @pytest.mark.usefixtures("load_birth_names_dashboard_with_slices") def test_top_n_dashboards_strategy(self): # create a top visited dashboard @@ -82,7 +81,7 @@ def test_top_n_dashboards_strategy(self): result = sorted(strategy.get_urls()) expected = sorted( [ - f"{URL_PREFIX}superset/warm_up_cache/?slice_id={slc.id}&dashboard_id={dash.id}" + f"{get_url_host()}superset/warm_up_cache/?slice_id={slc.id}&dashboard_id={dash.id}" for slc in dash.slices ] ) @@ -111,7 +110,12 @@ def test_dashboard_tags(self): # tag dashboard 'births' with `tag1` tag1 = get_tag("tag1", db.session, TagTypes.custom) dash = self.get_dash_by_slug("births") - tag1_urls = sorted([f"{URL_PREFIX}superset/warm_up_cache/?slice_id={slc.id}" for slc in dash.slices]) + tag1_urls = sorted( + [ + f"{get_url_host()}superset/warm_up_cache/?slice_id={slc.id}" + for slc in dash.slices + ] + ) tagged_object = TaggedObject( tag_id=tag1.id, object_id=dash.id, object_type=ObjectTypes.dashboard ) @@ -131,7 +135,7 @@ def test_dashboard_tags(self): # tag first slice dash = self.get_dash_by_slug("unicode-test") slc = dash.slices[0] - tag2_urls = [f"{URL_PREFIX}superset/warm_up_cache/?slice_id={slc.id}"] + tag2_urls = [f"{get_url_host()}superset/warm_up_cache/?slice_id={slc.id}"] object_id = slc.id tagged_object = TaggedObject( tag_id=tag2.id, object_id=object_id, object_type=ObjectTypes.chart diff --git a/tests/integration_tests/superset_test_config.py b/tests/integration_tests/superset_test_config.py index 48e72489b0d57..bc8668bb02c87 100644 --- a/tests/integration_tests/superset_test_config.py +++ b/tests/integration_tests/superset_test_config.py @@ -72,7 +72,8 @@ "DASHBOARD_NATIVE_FILTERS": True, } -WEBDRIVER_BASEURL = 'http://0.0.0.0:8081' +WEBDRIVER_BASEURL = "http://0.0.0.0:8081/" + def GET_FEATURE_FLAGS_FUNC(ff): ff_copy = copy(ff)