Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Resolve (#410) Logs are missing query strings #1165

Merged
merged 8 commits into from
Oct 22, 2022
Merged

Resolve (#410) Logs are missing query strings #1165

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
6e6de16
:white_check_mark: migrate utilities based tests from `tests.py` to `…
monkut Aug 16, 2022
35d6b52
:fire: remove `wsgi-request-logger` dependency (appears to no longer …
monkut Aug 16, 2022
9f392e5
Merge branch 'master' into feature/issue-410-add-query-strings-to-logs
monkut Aug 16, 2022
932155e
:wrench: black is no longer in 'beta', changing allow_prereleases to …
monkut Aug 17, 2022
aa6a4d3
Merge branch 'master' into feature/issue-410-add-query-strings-to-logs
monkut Aug 23, 2022
69eed1a
Merge branch 'master' into feature/issue-410-add-query-strings-to-logs
monkut Aug 24, 2022
3f8c21b
Merge branch 'master' into feature/issue-410-add-query-strings-to-logs
monkut Sep 1, 2022
305389b
Merge branch 'master' into feature/issue-410-add-query-strings-to-logs
monkut Oct 17, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,4 +73,7 @@ test-async:
test-general:
nosetests tests/tests.py --with-coverage --cover-package=zappa --with-timer

tests: clean test-docs test-handler test-middleware test-placebo test-async test-general
test-utilities:
nosetests tests/tests_utilities.py --with-coverage --cover-package=zappa --with-timer

tests: clean test-docs test-handler test-middleware test-placebo test-async test-general test-utilities
4 changes: 1 addition & 3 deletions Pipfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,6 @@ tqdm = "*"
troposphere = ">=3.0"
Werkzeug = "*"
wheel = "*"
wsgi-request-logger = "*"

[pipenv]
# Required for 'black' since all of its release tags contain 'b'
allow_prereleases = true
allow_prereleases = false
289 changes: 0 additions & 289 deletions tests/tests.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,7 @@

from zappa.cli import ZappaCLI, disable_click_colors, shamelessly_promote
from zappa.core import ALB_LAMBDA_ALIAS, ASSUME_POLICY, ATTACH_POLICY, Zappa
from zappa.ext.django_zappa import get_django_wsgi
from zappa.letsencrypt import (
cleanup,
create_chained_certificate,
create_domain_csr,
create_domain_key,
Expand All @@ -39,21 +37,6 @@
parse_csr,
register_account,
sign_certificate,
verify_challenge,
)
from zappa.utilities import (
InvalidAwsLambdaName,
conflicts_with_a_neighbouring_module,
contains_python_files_or_subdirs,
detect_django_settings,
detect_flask_apps,
get_venv_from_python_version,
human_size,
is_valid_bucket_name,
parse_s3_url,
string_to_timestamp,
titlecase_keys,
validate_name,
)
from zappa.wsgi import common_log, create_wsgi_request

Expand Down Expand Up @@ -101,61 +84,6 @@ def test_disable_click_colors(self):
disable_click_colors()
assert resolve_color_default() is False

@mock.patch("zappa.core.find_packages")
@mock.patch("os.remove")
def test_copy_editable_packages(self, mock_remove, mock_find_packages):
virtual_env = os.environ.get("VIRTUAL_ENV")
if not virtual_env:
return self.skipTest("test_copy_editable_packages must be run in a virtualenv")

temp_package_dir = tempfile.mkdtemp()
try:
egg_links = [
os.path.join(
virtual_env,
"lib",
get_venv_from_python_version(),
"site-packages",
"test-copy-editable-packages.egg-link",
)
]
egg_path = "/some/other/directory/package"
mock_find_packages.return_value = [
"package",
"package.subpackage",
"package.another",
]
temp_egg_link = os.path.join(temp_package_dir, "package-python.egg-link")

z = Zappa()
mock_open = mock.mock_open(read_data=egg_path.encode("utf-8"))
with mock.patch("zappa.core.open", mock_open), mock.patch("glob.glob") as mock_glob, mock.patch(
"zappa.core.copytree"
) as mock_copytree:
# we use glob.glob to get the egg-links in the temp packages
# directory
mock_glob.return_value = [temp_egg_link]

z.copy_editable_packages(egg_links, temp_package_dir)

# make sure we copied the right directories
mock_copytree.assert_called_with(
os.path.join(egg_path, "package"),
os.path.join(temp_package_dir, "package"),
metadata=False,
symlinks=False,
)
self.assertEqual(mock_copytree.call_count, 1)

# make sure it removes the egg-link from the temp packages
# directory
mock_remove.assert_called_with(temp_egg_link)
self.assertEqual(mock_remove.call_count, 1)
finally:
shutil.rmtree(temp_package_dir)

return

def test_create_lambda_package(self):
# mock the pkg_resources.WorkingSet() to include a known package in lambda_packages so that the code
# for zipping pre-compiled packages gets called
Expand Down Expand Up @@ -2028,119 +1956,6 @@ def test_get_all_zones_two_pages(self, client):
)
self.assertListEqual(zones["HostedZones"], [{"Id": "zone1"}, {"Id": "zone2"}])

##
# Django
##

def test_detect_dj(self):
# Sanity
settings_modules = detect_django_settings()

def test_dj_wsgi(self):
# Sanity
settings_modules = detect_django_settings()

settings = """
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
BASE_DIR = os.path.dirname(os.path.dirname(__file__))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.7/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'alskdfjalsdkf=0*%do-ayvy*m2k=vss*$7)j8q!@u0+d^na7mi2(^!l!d'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

TEMPLATE_DEBUG = True

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = (
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
)

MIDDLEWARE_CLASSES = (
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'blah.urls'
WSGI_APPLICATION = 'hackathon_starter.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.7/ref/settings/#databases

DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
}
}

# Internationalization
# https://docs.djangoproject.com/en/1.7/topics/i18n/

LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'UTC'
USE_I18N = True
USE_L10N = True
USE_TZ = True
"""

djts = open("dj_test_settings.py", "w")
djts.write(settings)
djts.close()

app = get_django_wsgi("dj_test_settings")
try:
os.remove("dj_test_settings.py")
os.remove("dj_test_settings.pyc")
except Exception as e:
pass

##
# Util / Misc
##

def test_human_units(self):
human_size(1)
human_size(9999999999999)

def test_string_to_timestamp(self):
boo = string_to_timestamp("asdf")
self.assertTrue(boo == 0)

yay = string_to_timestamp("1h")
self.assertTrue(type(yay) == int)
self.assertTrue(yay > 0)

yay = string_to_timestamp("4m")
self.assertTrue(type(yay) == int)
self.assertTrue(yay > 0)

yay = string_to_timestamp("1mm")
self.assertTrue(type(yay) == int)
self.assertTrue(yay > 0)

yay = string_to_timestamp("1mm1w1d1h1m1s1ms1us")
self.assertTrue(type(yay) == int)
self.assertTrue(yay > 0)

def test_event_name(self):
zappa = Zappa()
truncated = zappa.get_event_name(
Expand Down Expand Up @@ -2234,42 +2049,9 @@ def test_get_scheduled_event_name__truncated__has_name__has_index(self):
f"{hashed_lambda_name}-{index}-{event['name']}-{function}",
)

def test_detect_dj(self):
# Sanity
settings_modules = detect_django_settings()

def test_detect_flask(self):
# Sanity
settings_modules = detect_flask_apps()

def test_shameless(self):
shamelessly_promote()

def test_s3_url_parser(self):
remote_bucket, remote_file = parse_s3_url("s3://my-project-config-files/filename.json")
self.assertEqual(remote_bucket, "my-project-config-files")
self.assertEqual(remote_file, "filename.json")

remote_bucket, remote_file = parse_s3_url("s3://your-bucket/account.key")
self.assertEqual(remote_bucket, "your-bucket")
self.assertEqual(remote_file, "account.key")

remote_bucket, remote_file = parse_s3_url("s3://my-config-bucket/super-secret-config.json")
self.assertEqual(remote_bucket, "my-config-bucket")
self.assertEqual(remote_file, "super-secret-config.json")

remote_bucket, remote_file = parse_s3_url("s3://your-secure-bucket/account.key")
self.assertEqual(remote_bucket, "your-secure-bucket")
self.assertEqual(remote_file, "account.key")

remote_bucket, remote_file = parse_s3_url("s3://your-bucket/subfolder/account.key")
self.assertEqual(remote_bucket, "your-bucket")
self.assertEqual(remote_file, "subfolder/account.key")

# Sad path
remote_bucket, remote_file = parse_s3_url("/dev/null")
self.assertEqual(remote_bucket, "")

def test_remote_env_package(self):
zappa_cli = ZappaCLI()
zappa_cli.api_stage = "deprecated_remote_env"
Expand Down Expand Up @@ -2395,28 +2177,6 @@ def test_slim_handler(self):

zappa_cli.remove_local_zip()

def test_validate_name(self):
fname = "tests/name_scenarios.json"
with open(fname, "r") as f:
scenarios = json.load(f)
for scenario in scenarios:
value = scenario["value"]
is_valid = scenario["is_valid"]
if is_valid:
assert validate_name(value)
else:
with self.assertRaises(InvalidAwsLambdaName) as exc:
validate_name(value)

def test_contains_python_files_or_subdirs(self):
self.assertTrue(contains_python_files_or_subdirs("tests/data"))
self.assertTrue(contains_python_files_or_subdirs("tests/data/test2"))
self.assertFalse(contains_python_files_or_subdirs("tests/data/test1"))

def test_conflicts_with_a_neighbouring_module(self):
self.assertTrue(conflicts_with_a_neighbouring_module("tests/data/test1"))
self.assertFalse(conflicts_with_a_neighbouring_module("tests/data/test2"))

def test_settings_py_generation(self):
zappa_cli = ZappaCLI()
zappa_cli.api_stage = "ttt888"
Expand Down Expand Up @@ -2449,55 +2209,6 @@ def test_only_ascii_env_var_allowed(self):
zappa_cli.create_package()
self.assertEqual("Environment variable keys must be ascii.", str(context.exception))

def test_titlecase_keys(self):
raw = {
"hOSt": "github.com",
"ConnECtiOn": "keep-alive",
"UpGRAde-InSecuRE-ReQueSts": "1",
"uSer-AGEnT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36",
"cONtENt-TYPe": "text/html; charset=utf-8",
"aCCEpT": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
"ACcePT-encoDInG": "gzip, deflate, br",
"AcCEpT-lAnGUagE": "en-US,en;q=0.9",
}
transformed = titlecase_keys(raw)
expected = {
"Host": "github.com",
"Connection": "keep-alive",
"Upgrade-Insecure-Requests": "1",
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36",
"Content-Type": "text/html; charset=utf-8",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
"Accept-Encoding": "gzip, deflate, br",
"Accept-Language": "en-US,en;q=0.9",
}
self.assertEqual(expected, transformed)

def test_is_valid_bucket_name(self):
# Bucket names must be at least 3 and no more than 63 characters long.
self.assertFalse(is_valid_bucket_name("ab"))
self.assertFalse(is_valid_bucket_name("abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefhijlmn"))
# Bucket names must not contain uppercase characters or underscores.
self.assertFalse(is_valid_bucket_name("aaaBaaa"))
self.assertFalse(is_valid_bucket_name("aaa_aaa"))
# Bucket names must start with a lowercase letter or number.
self.assertFalse(is_valid_bucket_name(".abbbaba"))
self.assertFalse(is_valid_bucket_name("abbaba."))
self.assertFalse(is_valid_bucket_name("-abbaba"))
self.assertFalse(is_valid_bucket_name("ababab-"))
# Bucket names must be a series of one or more labels. Adjacent labels are separated by a single period (.).
# Each label must start and end with a lowercase letter or a number.
self.assertFalse(is_valid_bucket_name("aaa..bbbb"))
self.assertFalse(is_valid_bucket_name("aaa.-bbb.ccc"))
self.assertFalse(is_valid_bucket_name("aaa-.bbb.ccc"))
# Bucket names must not be formatted as an IP address (for example, 192.168.5.4).
self.assertFalse(is_valid_bucket_name("192.168.5.4"))
self.assertFalse(is_valid_bucket_name("127.0.0.1"))
self.assertFalse(is_valid_bucket_name("255.255.255.255"))

self.assertTrue(is_valid_bucket_name("valid-formed-s3-bucket-name"))
self.assertTrue(is_valid_bucket_name("worst.bucket.ever"))

# TODO: encountered error when vpc_config["SubnetIds"] or vpc_config["SecurityGroupIds"] is missing
# We need to make the code more robust in this case and avoid the KeyError
def test_zappa_core_deploy_lambda_alb_missing_cert_arn(self):
Expand Down
Loading