From 9df966eeb02f3de4d8ac2d8943f416d22faeeb73 Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Thu, 22 Aug 2024 13:16:39 +0200 Subject: [PATCH] Further filter fixes. (#547) * Add query string to logging output. * Type checking fix (type->isinstance) * Fix filters for Mreg (HostPolicy outstanding). * Safer version of JSON filter to avoid SQL injections. * Refactor, cleanup. * Hostpolicy filter "fixes", also bump dependencies. - We are stuck on drf 3.14.0 due to https://github.com/encode/django-rest-framework/issues/9358 until https://github.com/encode/django-rest-framework/pull/9483 goes into prod, hopefully 3.15.3. * Skeleton for testing filters. * Add iexact support, add cases. Fix toml. * Move to ruff formater. * More tests. - Add ip support. - Add reverse lookups. * Hostpolicy filter tests. - Also reformat as per ruff. * Add a test for filtering on ?id= for hosts. - This should catch the generic issue of filtering on IDs. Ideally we'd do this for every model that supports ID... * support `__in`. * Support CIDR matching. - Match exact CIDR or IP within a CIDR. --- hostpolicy/api/v1/views.py | 63 ++++- mreg/api/v1/filters.py | 416 +++++++++++++++++++++------- mreg/api/v1/tests/test_filtering.py | 266 ++++++++++++++++++ mreg/api/v1/tests/tests_bacnet.py | 2 +- mreg/middleware/logging_http.py | 2 + pyproject.toml | 23 +- requirements-test.txt | 7 +- requirements.txt | 8 +- 8 files changed, 657 insertions(+), 130 deletions(-) create mode 100644 mreg/api/v1/tests/test_filtering.py diff --git a/hostpolicy/api/v1/views.py b/hostpolicy/api/v1/views.py index b7f970e3..6dc4e715 100644 --- a/hostpolicy/api/v1/views.py +++ b/hostpolicy/api/v1/views.py @@ -2,7 +2,8 @@ from rest_framework import status from rest_framework.response import Response -from django_filters import rest_framework as rest_filters +from django_filters import rest_framework as filters +from rest_framework import filters as rest_filters from hostpolicy.api.permissions import IsSuperOrHostPolicyAdminOrReadOnly from hostpolicy.models import HostPolicyAtom, HostPolicyRole @@ -18,39 +19,71 @@ from mreg.mixins import LowerCaseLookupMixin from mreg.models.host import Host -from . import serializers - -# For some reason the name field for filtersets for HostPolicyAtom and HostPolicyRole does -# not support operators (e.g. __contains, __regex) in the same way as other fields. Yes, -# the name field is a LowerCaseCharField, but the operators work fine in mreg proper. -# To resolve this issue, we create custom fields for the filtersets that use the name field. +from mreg.api.v1.filters import STRING_OPERATORS, INT_OPERATORS -class HostPolicyAtomFilterSet(rest_filters.FilterSet): - name__contains = rest_filters.CharFilter(field_name="name", lookup_expr="contains") - name__regex = rest_filters.CharFilter(field_name="name", lookup_expr="regex") +from . import serializers +# Note that related lookups don't work at the moment, so we need to do them explicitly. +class HostPolicyAtomFilterSet(filters.FilterSet): class Meta: model = HostPolicyAtom - fields = "__all__" + fields = { + "name": STRING_OPERATORS, + "create_date": INT_OPERATORS, + "updated_at": INT_OPERATORS, + "description": STRING_OPERATORS, + "roles": INT_OPERATORS, + } + +class HostPolicyRoleFilterSet(filters.FilterSet): + # This seems to be required due to the many-to-many relationships? + atoms__name__exact = filters.CharFilter(field_name='atoms__name', lookup_expr='exact') + atoms__name__contains = filters.CharFilter(field_name='atoms__name', lookup_expr='contains') + atoms__name__regex = filters.CharFilter(field_name='atoms__name', lookup_expr='regex') + + hosts__name__exact = filters.CharFilter(field_name='hosts__name', lookup_expr='exact') + hosts__name__contains = filters.CharFilter(field_name='hosts__name', lookup_expr='contains') + hosts__name__regex = filters.CharFilter(field_name='hosts__name', lookup_expr='regex') + + labels__name__exact = filters.CharFilter(field_name='labels__name', lookup_expr='exact') + labels__name__contains = filters.CharFilter(field_name='labels__name', lookup_expr='contains') + labels__name__regex = filters.CharFilter(field_name='labels__name', lookup_expr='regex') -class HostPolicyRoleFilterSet(rest_filters.FilterSet): - name__contains = rest_filters.CharFilter(field_name="name", lookup_expr="contains") - name__regex = rest_filters.CharFilter(field_name="name", lookup_expr="regex") class Meta: model = HostPolicyRole - fields = "__all__" + fields = { + "name": STRING_OPERATORS, + "create_date": INT_OPERATORS, + "updated_at": INT_OPERATORS, + "hosts": INT_OPERATORS, + "atoms": INT_OPERATORS, + "labels": INT_OPERATORS, + } class HostPolicyAtomLogMixin(HistoryLog): log_resource = 'hostpolicy_atom' model = HostPolicyAtom + filter_backends = ( + rest_filters.SearchFilter, + filters.DjangoFilterBackend, + rest_filters.OrderingFilter, + ) + ordering_fields = "__all__" + class HostPolicyRoleLogMixin(HistoryLog): log_resource = 'hostpolicy_role' model = HostPolicyRole + filter_backends = ( + rest_filters.SearchFilter, + filters.DjangoFilterBackend, + rest_filters.OrderingFilter, + ) + ordering_fields = "__all__" class HostPolicyPermissionsListCreateAPIView(M2MPermissions, diff --git a/mreg/api/v1/filters.py b/mreg/api/v1/filters.py index 78c06a19..04741c57 100644 --- a/mreg/api/v1/filters.py +++ b/mreg/api/v1/filters.py @@ -1,177 +1,401 @@ +import operator +from functools import reduce +from typing import List + +import structlog +from django.db.models import Q from django_filters import rest_framework as filters from mreg.models.base import History from mreg.models.host import BACnetID, Host, HostGroup, Ipaddress, PtrOverride -from mreg.models.network import ( - Label, - NetGroupRegexPermission, - Network, - NetworkExcludedRange, -) -from mreg.models.resource_records import Cname, Hinfo, Loc, Mx, Naptr, Srv, Sshfp, Txt -from mreg.models.zone import ( - ForwardZone, - ForwardZoneDelegation, - NameServer, - ReverseZone, - ReverseZoneDelegation, -) - -class FilterWithID(filters.FilterSet): - id = filters.NumberFilter(field_name="id") - id__in = filters.BaseInFilter(field_name="id") - id__gt = filters.NumberFilter(field_name="id", lookup_expr="gt") - id__lt = filters.NumberFilter(field_name="id", lookup_expr="lt") - - -class JSONFieldExactFilter(filters.CharFilter): - pass - - -class CIDRFieldExactFilter(filters.CharFilter): - pass - - -class BACnetIDFilterSet(FilterWithID): +from mreg.models.network import (Label, NetGroupRegexPermission, Network, + NetworkExcludedRange) +from mreg.models.resource_records import (Cname, Hinfo, Loc, Mx, Naptr, Srv, + Sshfp, Txt) +from mreg.models.zone import (ForwardZone, ForwardZoneDelegation, NameServer, + ReverseZone, ReverseZoneDelegation) + +from netaddr import IPNetwork, AddrFormatError + +mreg_log = structlog.getLogger(__name__) + +OperatorList = List[str] + +STRING_OPERATORS: OperatorList = [ + "exact", + "iexact", + "regex", + "contains", + "icontains", + "startswith", + "istartswith", + "endswith", + "iendswith", +] +INT_OPERATORS: OperatorList = ["exact", "in", "gt", "lt"] +EXACT_OPERATORS: OperatorList = ["exact"] + +HOST_FIELDS = { + "host": INT_OPERATORS, + "host__comment": STRING_OPERATORS, + "host__contact": STRING_OPERATORS, + "host__name": STRING_OPERATORS, + "host__ttl": INT_OPERATORS, +} + +CREATED_UPDATED = { + "created_at": INT_OPERATORS, + "updated_at": INT_OPERATORS, +} +class CIDRFieldFilter(filters.CharFilter): + def filter(self, qs, value): + if not value: + return qs + + try: + cidr = IPNetwork(value) + return qs.filter(**{f"{self.field_name}__net_contains_or_equals": str(cidr)}) + except AddrFormatError: + return qs.none() + +class BACnetIDFilterSet(filters.FilterSet): class Meta: model = BACnetID - fields = "__all__" + fields = { + "id": INT_OPERATORS, + **HOST_FIELDS, + } -class CnameFilterSet(FilterWithID): +class CnameFilterSet(filters.FilterSet): class Meta: model = Cname - fields = "__all__" + fields = { + "id": INT_OPERATORS, + "name": STRING_OPERATORS, + "ttl": INT_OPERATORS, + **HOST_FIELDS, + **CREATED_UPDATED, + } -class ForwardZoneFilterSet(FilterWithID): +class ForwardZoneFilterSet(filters.FilterSet): class Meta: model = ForwardZone - fields = "__all__" + fields = { + "id": INT_OPERATORS, + "name": STRING_OPERATORS, + **CREATED_UPDATED, + } -class ForwardZoneDelegationFilterSet(FilterWithID): +class ForwardZoneDelegationFilterSet(filters.FilterSet): class Meta: model = ForwardZoneDelegation - fields = "__all__" + fields = { + "id": INT_OPERATORS, + "name": STRING_OPERATORS, + "nameservers": INT_OPERATORS, + "comment": STRING_OPERATORS, + **CREATED_UPDATED, + } -class HinfoFilterSet(FilterWithID): +class HinfoFilterSet(filters.FilterSet): class Meta: model = Hinfo - fields = "__all__" - + fields = { + "cpu": STRING_OPERATORS, + "host": INT_OPERATORS, + "os": STRING_OPERATORS, + } -class HistoryFilterSet(FilterWithID): - data = JSONFieldExactFilter(field_name="data") +class HistoryFilterSet(filters.FilterSet): class Meta: model = History - fields = "__all__" - - -class HostFilterSet(FilterWithID): + fields = { + "id": INT_OPERATORS, + "timestamp": INT_OPERATORS, + "user": STRING_OPERATORS, + "resource": STRING_OPERATORS, + "name": STRING_OPERATORS, + "model_id": INT_OPERATORS, + "model": STRING_OPERATORS, + "action": STRING_OPERATORS, + } + + # This is a fugly hack to make JSON filtering "work" + def filter_queryset(self, queryset): + data_filters = {k: v for k, v in self.data.items() if k.startswith("data__")} + + if data_filters: + queries = [] + for key, value in data_filters.items(): + json_key = key.split("data__")[1] + if "__in" in json_key: + json_key = json_key.split("__in")[0] + values = value.split(",") + queries.append(Q(**{f"data__{json_key}__in": values})) + else: + queries.append(Q(**{f"data__{json_key}": value})) + + queryset = queryset.filter(reduce(operator.and_, queries)) + + return super().filter_queryset(queryset) + + +class HostFilterSet(filters.FilterSet): class Meta: model = Host - fields = "__all__" - - -class HostGroupFilterSet(FilterWithID): + fields = { + "id": INT_OPERATORS, + "name": STRING_OPERATORS, + "contact": STRING_OPERATORS, + "ttl": INT_OPERATORS, + "comment": STRING_OPERATORS, + # These are related fields, ie, inverse relationships + "ipaddresses": INT_OPERATORS, + "ipaddresses__ipaddress": EXACT_OPERATORS, + "ipaddresses__macaddress": STRING_OPERATORS, + "ptr_overrides": INT_OPERATORS, + "ptr_overrides__ipaddress": EXACT_OPERATORS, + "hostgroups": INT_OPERATORS, + "hostgroups__name": STRING_OPERATORS, + "hostgroups__description": STRING_OPERATORS, + "bacnetid": INT_OPERATORS, + "mxs": INT_OPERATORS, + "mxs__priority": INT_OPERATORS, + "mxs__mx": STRING_OPERATORS, + "txts": INT_OPERATORS, + "txts__txt": STRING_OPERATORS, + "cnames": INT_OPERATORS, + "cnames__name": STRING_OPERATORS, + "cnames__ttl": INT_OPERATORS, + "naptrs": INT_OPERATORS, + "naptrs__order": INT_OPERATORS, + "naptrs__preference": INT_OPERATORS, + "naptrs__flag": STRING_OPERATORS, + "naptrs__service": STRING_OPERATORS, + "naptrs__regex": STRING_OPERATORS, + "naptrs__replacement": STRING_OPERATORS, + "srvs": INT_OPERATORS, + "srvs__name": STRING_OPERATORS, + "srvs__priority": INT_OPERATORS, + "srvs__weight": INT_OPERATORS, + "srvs__port": INT_OPERATORS, + "srvs__ttl": INT_OPERATORS, + **CREATED_UPDATED, + } + + +class HostGroupFilterSet(filters.FilterSet): class Meta: model = HostGroup - fields = "__all__" - - -class IpaddressFilterSet(FilterWithID): + fields = { + "id": INT_OPERATORS, + "description": STRING_OPERATORS, + "hosts": INT_OPERATORS, + "name": STRING_OPERATORS, + "owners": INT_OPERATORS, + "parent": INT_OPERATORS, + **CREATED_UPDATED, + } + + +class IpaddressFilterSet(filters.FilterSet): class Meta: model = Ipaddress - fields = "__all__" + fields = { + "id": INT_OPERATORS, + "ipaddress": STRING_OPERATORS, + "macaddress": STRING_OPERATORS, + **HOST_FIELDS, + } -class LabelFilterSet(FilterWithID): +class LabelFilterSet(filters.FilterSet): class Meta: model = Label - fields = "__all__" + fields = { + "id": INT_OPERATORS, + "description": STRING_OPERATORS, + "name": STRING_OPERATORS, + **CREATED_UPDATED, + } -class LocFilterSet(FilterWithID): +class LocFilterSet(filters.FilterSet): class Meta: model = Loc - fields = "__all__" + fields = { + "loc": STRING_OPERATORS, + **HOST_FIELDS, + } -class MxFilterSet(FilterWithID): +class MxFilterSet(filters.FilterSet): class Meta: model = Mx - fields = "__all__" + fields = { + "id": INT_OPERATORS, + "priority": INT_OPERATORS, + "mx": STRING_OPERATORS, + **HOST_FIELDS, + **CREATED_UPDATED + } -class NameServerFilterSet(FilterWithID): +class NameServerFilterSet(filters.FilterSet): class Meta: model = NameServer - fields = "__all__" + fields = { + "id": INT_OPERATORS, + "name": STRING_OPERATORS, + "ttl": INT_OPERATORS, + **CREATED_UPDATED, + } -class NaptrFilterSet(FilterWithID): +class NaptrFilterSet(filters.FilterSet): class Meta: model = Naptr - fields = "__all__" - - -class NetGroupRegexPermissionFilterSet(FilterWithID): - range = CIDRFieldExactFilter(field_name="range") + fields = { + "id": INT_OPERATORS, + "preference": INT_OPERATORS, + "order": INT_OPERATORS, + "flag": STRING_OPERATORS, + "service": STRING_OPERATORS, + "regex": STRING_OPERATORS, + "replacement": STRING_OPERATORS, + **HOST_FIELDS, + **CREATED_UPDATED, + } + + +class NetGroupRegexPermissionFilterSet(filters.FilterSet): + range = CIDRFieldFilter(field_name="range") class Meta: model = NetGroupRegexPermission - fields = "__all__" + fields = { + "id": INT_OPERATORS, + "group": STRING_OPERATORS, + "regex": STRING_OPERATORS, + "labels": INT_OPERATORS, + **CREATED_UPDATED, + } -class NetworkFilterSet(FilterWithID): - network = CIDRFieldExactFilter(field_name="network") +class NetworkFilterSet(filters.FilterSet): + network = CIDRFieldFilter(field_name="network") class Meta: model = Network - fields = "__all__" - - -class NetworkExcludedRangeFilterSet(FilterWithID): + fields = { + "id": INT_OPERATORS, + "description": STRING_OPERATORS, + "vlan": INT_OPERATORS, + "dns_delegated": EXACT_OPERATORS, + "category": STRING_OPERATORS, + "location": STRING_OPERATORS, + "frozen": EXACT_OPERATORS, + "reserved": INT_OPERATORS, + **CREATED_UPDATED, + } + + +class NetworkExcludedRangeFilterSet(filters.FilterSet): class Meta: model = NetworkExcludedRange - fields = "__all__" - - -class PtrOverrideFilterSet(FilterWithID): + fields = { + "id": INT_OPERATORS, + "network": INT_OPERATORS, + "network__description": STRING_OPERATORS, + "network__vlan": INT_OPERATORS, + "network__dns_delegated": EXACT_OPERATORS, + "network__category": STRING_OPERATORS, + "network__location": STRING_OPERATORS, + "network__frozen": EXACT_OPERATORS, + "network__reserved": INT_OPERATORS, + "start_ip": STRING_OPERATORS, + "end_ip": STRING_OPERATORS, + **CREATED_UPDATED, + } + + +class PtrOverrideFilterSet(filters.FilterSet): class Meta: model = PtrOverride - fields = "__all__" + fields = { + "id": INT_OPERATORS, + "ipaddress": EXACT_OPERATORS, + **HOST_FIELDS, + **CREATED_UPDATED, + } + -class ReverseZoneFilterSet(FilterWithID): - network = CIDRFieldExactFilter(field_name="network") +class ReverseZoneFilterSet(filters.FilterSet): + network = CIDRFieldFilter(field_name="network") class Meta: model = ReverseZone - fields = "__all__" + fields = { + "id": INT_OPERATORS, + "name": STRING_OPERATORS, + **CREATED_UPDATED, + } -class ReverseZoneDelegationFilterSet(FilterWithID): +class ReverseZoneDelegationFilterSet(filters.FilterSet): class Meta: model = ReverseZoneDelegation - fields = "__all__" - -class SrvFilterSet(FilterWithID): + fields = { + "id": INT_OPERATORS, + "name": STRING_OPERATORS, + "nameservers": INT_OPERATORS, + "comment": STRING_OPERATORS, + "zone": INT_OPERATORS, + "zone__name": STRING_OPERATORS, + **CREATED_UPDATED, + } + + +class SrvFilterSet(filters.FilterSet): class Meta: model = Srv - fields = "__all__" - - -class SshfpFilterSet(FilterWithID): + fields = { + "id": INT_OPERATORS, + "name": STRING_OPERATORS, + "priority": INT_OPERATORS, + "weight": INT_OPERATORS, + "port": INT_OPERATORS, + "ttl": INT_OPERATORS, + **HOST_FIELDS, + **CREATED_UPDATED, + } + + +class SshfpFilterSet(filters.FilterSet): class Meta: model = Sshfp - fields = "__all__" + fields = { + "id": INT_OPERATORS, + "algorithm": INT_OPERATORS, + "hash_type": INT_OPERATORS, + "fingerprint": STRING_OPERATORS, + **HOST_FIELDS, + **CREATED_UPDATED, + } -class TxtFilterSet(FilterWithID): +class TxtFilterSet(filters.FilterSet): class Meta: model = Txt - fields = "__all__" + fields = { + "id": INT_OPERATORS, + "txt": STRING_OPERATORS, + **HOST_FIELDS, + **CREATED_UPDATED, + } diff --git a/mreg/api/v1/tests/test_filtering.py b/mreg/api/v1/tests/test_filtering.py new file mode 100644 index 00000000..feb0b1b2 --- /dev/null +++ b/mreg/api/v1/tests/test_filtering.py @@ -0,0 +1,266 @@ +from typing import List, Tuple, Union +from itertools import chain + +from unittest_parametrize import ParametrizedTestCase, param, parametrize + +from hostpolicy.models import HostPolicyAtom, HostPolicyRole +from mreg.models.base import Label +from mreg.models.host import Host, Ipaddress +from mreg.models.network import NetGroupRegexPermission +from mreg.models.resource_records import Cname + +from .tests import MregAPITestCase + + +def create_hosts(name: str, count: int) -> List[Host]: + """Create hosts.""" + + hosts: List[Host] = [] + for i in range(count): + hosts.append( + Host.objects.create( + name=f"{name}{i}.example.com".replace("_", ""), + contact="admin@example.com", + ttl=3600, + comment="Test host", + ) + ) + + return hosts + + +def create_cnames(hosts: List[Host]) -> List[Cname]: + """Create cnames.""" + + cnames: List[Cname] = [] + for host in hosts: + cnames.append( + Cname.objects.create( + host=host, + name=f"cname.{host.name}", + ttl=3600, + ) + ) + + return cnames + + +def create_ipaddresses(hosts: List[Host]) -> List[Ipaddress]: + """Create ipaddresses.""" + + ipaddresses: List[Ipaddress] = [] + for i, host in enumerate(hosts): + ipaddresses.append( + Ipaddress.objects.create( + host=host, + ipaddress=f"10.0.0.{i}", + ) + ) + + return ipaddresses + + +def create_labels(name: str, count: int) -> List[Label]: + """Create labels.""" + + labels: List[Label] = [] + for i in range(count): + labels.append( + Label.objects.create( + name=f"{name}{i}".replace("_", ""), + description="Test label", + ) + ) + + return labels + + +def create_atoms(name: str, count: int) -> List[HostPolicyAtom]: + """Create atoms.""" + + atoms: List[HostPolicyAtom] = [] + for i in range(count): + atoms.append( + HostPolicyAtom.objects.create( + name=f"{name}{i}".replace("_", ""), + description=f"Test atom {i}", + ) + ) + + return atoms + + +def create_roles( + name: str, hosts: List[Host], atoms: List[HostPolicyAtom], labels: List[Label] +) -> Tuple[List[HostPolicyRole], List[HostPolicyAtom], List[Label]]: + """Create roles.""" + + if not atoms: + atoms = create_atoms(f"{name}atom", len(hosts)) + + if not labels: + labels = create_labels(f"{name}label", len(hosts)) + + if len(hosts) != len(atoms) or len(hosts) != len(labels): + raise ValueError("Hosts, Atoms, and Labels must be the same length.") + + roles: List[HostPolicyRole] = [] + + for i, h in enumerate(hosts): + policy = HostPolicyRole.objects.create(name=f"{name}host{i}".replace("_", ""), description="Test role") + policy.hosts.add(h) + policy.labels.add(labels[i]) + policy.atoms.add(atoms[i]) + roles.append(policy) + return (roles, atoms, labels) + +def resolve_target(target: Union[str, List[str]]) -> List[int]: + if isinstance(target, str): + return [int(target)] + else: + return [int(t) for t in target] + +class FilterTestCase(ParametrizedTestCase, MregAPITestCase): + """Test filtering.""" + + # endpoint, query_key, target, expected_hits + # + # NOTE: The generated hostnames are UNIQUE across every test case! + # The format is: f"{endpoint}{query_key}{i}.example.com".replace("_", "") + # where i is the index of the hostname (and we make three for each test). + @parametrize( + ("endpoint", "query_key", "target", "expected_hits"), + [ + # Direct host filtering + param("hosts", "id", "1", 1, id="hosts_id"), + param("hosts", "id__in", "2", 1, id="hosts_id__in_2"), + param("hosts", "id__in", "1,2", 2, id="hosts_id__in_12"), + param("hosts", "id__in", "0,1,2", 3, id="hosts_id__in_012"), + param("hosts", "name", "hostsname0.example.com", 1, id="hosts_name"), + param("hosts", "name__contains", "namecontains1", 1, id="hosts_name__contains"), + param("hosts", "name__icontains", "nameicontains2", 1, id="hosts_name__icontains"), + param("hosts", "name__iexact", "hostsnameiexact2.example.com", 1, id="hosts_name__iexact"), + param("hosts", "name__startswith", "hostsnamestartswith1", 1, id="hosts_name__startswith"), + param("hosts", "name__endswith", "endswith0.example.com", 1, id="hosts_name__endswith"), + param("hosts", "name__regex", "nameregex[1-2].example.com", 2, id="hosts_name__regex"), + # Reverse through Ipaddress + param("hosts", "ipaddresses__ipaddress", "10.0.0.1", 1, id="hosts_ipaddresses__ipaddress"), + # Reverse through Cname + param("hosts", "cnames__name", "cname.hostscnamesname0.example.com", 1, id="hosts_cnames__name"), + param("hosts", "cnames__name__regex", "cname.*regex[0-1].example.com", 2, id="host_cnames__regex"), + param("hosts", "cnames__name__endswith", "with0.example.com", 1, id="hosts_cnames__endswith"), + # Indirectly through Ipaddress + param("ipaddresses", "host__name", "ipaddresseshostname0.example.com", 1, id="ipaddresses_host__name"), + param("ipaddresses", "host__name__contains", "contains1", 1, id="ipaddresses_host__contains"), + # Indirectly through Cname + param("cnames", "host__name", "cnameshostname1.example.com", 1, id="cnames_host__name"), + param("cnames", "host__name__icontains", "cnameshostnameicontains", 3, id="cnames_host__icontains"), + param("cnames", "host__name__iexact", "cnameshostnameiexact1.example.com", 1, id="cnames_host__iexact"), + param("cnames", "host__name__startswith", "cnameshostnamestartswith", 3, id="cnames_host__startswith"), + param("cnames", "host__name__endswith", "endswith2.example.com", 1, id="cnames_host__endswith"), + param("cnames", "host__name__regex", "cnameshostnameregex[0-1]", 2, id="cnames_host__regex"), + ], + ) + def test_filtering_for_host(self, endpoint: str, query_key: str, target: str, expected_hits: str) -> None: + """Test filtering on host.""" + + generate_count = 3 + hosts = create_hosts(f"{endpoint}{query_key}", generate_count) + cnames = create_cnames(hosts) + ipadresses = create_ipaddresses(hosts) + + if query_key.startswith("id"): + targets = target.split(",") if query_key.endswith("__in") else [target] + resolved_targets = resolve_target(targets) + target = ",".join(str(hosts[t].id) for t in resolved_targets) # type: ignore + + msg_prefix = f"{endpoint} : {query_key} -> {target} => " + + response = self.client.get(f"/api/v1/{endpoint}/?{query_key}={target}") + self.assertEqual(response.status_code, 200, msg=f"{msg_prefix} {response.content}") + data = response.json() + self.assertEqual(data["count"], expected_hits, msg=f"{msg_prefix} {data}") + + for obj in chain(ipadresses, cnames, hosts): + obj.delete() + + @parametrize( + ("endpoint", "query_key", "target", "expected_hits"), + [ + param("roles", "name", "roleshost0", 1, id="roles_name"), + param("roles", "name__contains", "roleshost1", 1, id="roles_name__contains"), + param("roles", "name__icontains", "roleshost2", 1, id="roles_name__icontains"), + param("roles", "name__iexact", "roleshost2", 1, id="roles_name__iexact"), + param("roles", "name__startswith", "roleshost1", 1, id="roles_name__startswith"), + param("roles", "name__endswith", "host1", 1, id="roles_name__endswith"), + param("roles", "name__regex", "roleshost[0-1]", 2, id="roles_name__regex"), + param("roles", "atoms__name__exact", "rolesatomsnameexact1", 1, id="roles_atoms__name__exact"), + param("roles", "atoms__name__contains", "namecontains1", 1, id="roles_atoms__name__contains"), + param("roles", "atoms__name__regex", "nameregex[0-1]", 2, id="roles_atoms__name__regex"), + param("roles", "hosts__name__exact", "roleshostsnameexact1.example.com", 1, id="roles_hosts__name__exact"), + param("roles", "hosts__name__contains", "namecontains1", 1, id="roles_hosts__name__contains"), + param("roles", "hosts__name__regex", "nameregex[0-1].example.com", 2, id="roles_hosts__name__regex"), + param("roles", "labels__name__exact", "roleslabelsnameexact1", 1, id="roles_labels__name__exact"), + param("roles", "labels__name__contains", "namecontains1", 1, id="roles_labels__name__contains"), + param("roles", "labels__name__regex", "nameregex[0-1]", 2, id="roles_labels__name__regex"), + param("atoms", "name", "atomsname0", 1, id="atoms_name"), + param("atoms", "name__contains", "namecontains1", 1, id="atoms_name__contains"), + param("atoms", "name__icontains", "nameicontains2", 1, id="atoms_name__icontains"), + param("atoms", "name__iexact", "atomsnameiexact2", 1, id="atoms_name__iexact"), + param("atoms", "name__startswith", "atomsnamestartswith1", 1, id="atoms_name__startswith"), + param("atoms", "name__endswith", "endswith0", 1, id="atoms_name__endswith"), + param("atoms", "name__regex", "nameregex[0-1]", 2, id="atoms_name__regex"), + param("atoms", "description", "Test atom 1", 1, id="atoms_description"), + param("atoms", "description__contains", "Test atom", 3, id="atoms_description__contains"), + param("atoms", "description__regex", "Test atom [0-1]", 2, id="atoms_description__regex"), + ], + ) + def test_filtering_for_hostpolicy(self, endpoint: str, query_key: str, target: str, expected_hits: str) -> None: + """Test filtering on hostpolicy.""" + + generate_count = 3 + msg_prefix = f"{endpoint} : {query_key} -> {target} => " + + hosts = create_hosts(f"{endpoint}{query_key}", generate_count) + atoms = create_atoms(f"{endpoint}{query_key}", generate_count) + labels = create_labels(f"{endpoint}{query_key}", generate_count) + + roles, atoms, labels = create_roles(endpoint, hosts, atoms, labels) + + response = self.client.get(f"/api/v1/hostpolicy/{endpoint}/?{query_key}={target}") + self.assertEqual(response.status_code, 200, msg=f"{msg_prefix} {response.content}") + data = response.json() + self.assertEqual(data["count"], expected_hits, msg=f"{msg_prefix} {data}") + + for obj in chain(roles, atoms, labels, hosts): + obj.delete() + + @parametrize(("cidr", "exists"), [ + param("10.0.0.0/24", True, id="cidr_0_true"), + param("10.0.1.0/24", True, id="cidr_1_true"), + param("10.0.2.0/24", True, id="cidr_2_true"), + param("10.0.3.0/24", False, id="cidr_3_false"), + + param("10.0.0.1", True, id="ip_0_1_true"), + param("10.0.0.2", True, id="ip_0_2_true"), + param("10.0.1.1", True, id="ip_1_1_true"), + param("10.0.2.1", True, id="ip_2_1_true"), + + param("10.0.3.1", False, id="ip_3_1_false"), + ], + ) + def test_filter_netgroup_regex_permission(self, cidr: str, exists: bool) -> None: + """Test filtering on netgroup regex permission.""" + + generate_count = 3 + + for i in range(generate_count): + NetGroupRegexPermission.objects.create( + regex=".*", + range=f"10.0.{i}.0/24" + ) + + response = self.client.get(f"/api/v1/permissions/netgroupregex/?range={cidr}") + self.assertEqual(response.status_code, 200) + data = response.json() + self.assertEqual(data["count"], 1 if exists else 0) \ No newline at end of file diff --git a/mreg/api/v1/tests/tests_bacnet.py b/mreg/api/v1/tests/tests_bacnet.py index 346b402f..78ec4147 100644 --- a/mreg/api/v1/tests/tests_bacnet.py +++ b/mreg/api/v1/tests/tests_bacnet.py @@ -10,7 +10,7 @@ class BACnetIDTest(MregAPITestCase): basepath = '/api/v1/bacnet/ids/' def basejoin(self, path): - if type(path) != 'str': + if not isinstance(path, str): path = str(path) return self.basepath + path diff --git a/mreg/middleware/logging_http.py b/mreg/middleware/logging_http.py index 03ff1e27..64597c21 100644 --- a/mreg/middleware/logging_http.py +++ b/mreg/middleware/logging_http.py @@ -106,6 +106,7 @@ def log_request(self, request: HttpRequest) -> None: remote_ip=remote_ip, proxy_ip=proxy_ip, path=request.path_info, + query_string=request.META.get("QUERY_STRING"), request_size=request_size, content=self._get_body(request), ).info("request") @@ -150,6 +151,7 @@ def log_response( status_code=status_code, status_label=status_label, path=request.path_info, + query_string=request.META.get("QUERY_STRING"), content=content, **extra_data, run_time_ms=round(run_time_ms, 2), diff --git a/pyproject.toml b/pyproject.toml index b090b1a0..e06d4672 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,3 @@ -[tool.ruff] -# https://beta.ruff.rs/docs/rules/ -select = ["E", "F"] -line-length = 119 -exclude = [ - "mreg/migrations/", - "hostpolicy/migrations/", - ".tox", -] - [project] name = "mreg" version = "0.0.1" @@ -21,4 +11,15 @@ requires = [ ] [tool.setuptools] -py-modules = ["mreg", "mregsite", "hostpolicy"] \ No newline at end of file +py-modules = ["mreg", "mregsite", "hostpolicy"] + +[tool.ruff] +# https://beta.ruff.rs/docs/rules/ +select = ["E", "F"] +line-length = 119 +exclude = [ + "mreg/migrations/", + "hostpolicy/migrations/", + ".tox", +] + diff --git a/requirements-test.txt b/requirements-test.txt index 1bd77db6..e4ddd128 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -3,19 +3,20 @@ djangorestframework==3.14.0 django-auth-ldap==4.8.0 django-logging-json==1.15 django-netfields==1.3.2 -django-filter==24.2 -structlog==24.1.0 +django-filter==24.3 +structlog==24.4.0 rich==13.7.1 gunicorn==22.0.0 idna==3.7 psycopg2-binary==2.9.9 pika==1.3.2 -sentry-sdk==2.3.1 +sentry-sdk==2.12.0 tzdata==2024.1 # Testing framwork tox pytest pytest-django +unittest_parametrize # These are currently not used, but should be... pylint diff --git a/requirements.txt b/requirements.txt index 190c1b14..b2d47ca9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,15 +1,15 @@ -Django==5.0.6 +Django==5.0.8 djangorestframework==3.14.0 django-auth-ldap==4.8.0 django-netfields==1.3.2 -django-filter==24.2 -structlog==24.1.0 +django-filter==24.3 +structlog==24.4.0 rich==13.7.1 gunicorn==22.0.0 idna==3.7 psycopg2-binary==2.9.9 pika==1.3.2 -sentry-sdk==2.3.1 +sentry-sdk==2.12.0 tzdata==2024.1 # For OpenAPI schema generation. uritemplate