Merge branch 'main' into rel

readthedocs · Dec 18, 2024 · 5ca35e2 · 5ca35e2
2 parents 3848292 + 854a509
commit 5ca35e2
Show file tree

Hide file tree

Showing 16 changed files with 265 additions and 65 deletions.
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -6,6 +6,26 @@ CHANGELOG
 .. This is included by docs/developer/changelog.rst
 
 
+Version v5.12.0
+---------------
+
+This release adds the ability to start reporting on ad campaigns at the domain level.
+The report itself is not yet in this release, but the data is going to start being aggregated
+for an upcoming report.
+There were some dependency updates including a New Relic related dependency fix.
+
+:Date: December 18, 2024
+
+ * @davidfischer: Niche targeting flight status (#965)
+ * @davidfischer: Make paying out via stripe the default (#964)
+ * @davidfischer: Add a 429 template and handler for dev (#963)
+ * @dependabot[bot]: Bump django from 5.0.9 to 5.0.10 in /requirements (#958)
+ * @davidfischer: Update requirements (#957)
+ * @davidfischer: Domain aggregation (#955)
+ * @dependabot[bot]: Bump pyjwt from 2.10.0 to 2.10.1 in /requirements (#952)
+ * @davidfischer: Pin New Relic (#947)
+
+
 Version v5.11.0
 ---------------
 

diff --git a/adserver/admin.py b/adserver/admin.py
@@ -27,6 +27,7 @@
 from .models import Campaign
 from .models import Click
 from .models import CountryRegion
+from .models import DomainImpression
 from .models import Flight
 from .models import GeoImpression
 from .models import Keyword
@@ -880,6 +881,28 @@ class UpliftImpressionAdmin(ImpressionsAdmin):
     pass
 
 
+@admin.register(DomainImpression)
+class DomainImpressionAdmin(ImpressionsAdmin):
+    date_hierarchy = "date"
+    readonly_fields = (
+        "date",
+        "domain",
+        "advertisement",
+        "views",
+        "clicks",
+        "offers",
+        "decisions",
+        "click_to_offer_rate",
+        "view_to_offer_rate",
+        "modified",
+        "created",
+    )
+    list_display = readonly_fields
+    list_filter = ("advertisement__flight__campaign__advertiser",)
+    list_select_related = ("advertisement",)
+    search_fields = ("advertisement__slug", "advertisement__name", "domain")
+
+
 @admin.register(RotationImpression)
 class RotationImpressionAdmin(ImpressionsAdmin):
     date_hierarchy = "date"
@@ -910,7 +933,7 @@ class AdBaseAdmin(RemoveDeleteMixin, admin.ModelAdmin):
         "date",
         "advertisement",
         "publisher",
-        "page_url",
+        "domain",
         "keywords",
         "country",
         "browser_family",
@@ -922,6 +945,7 @@ class AdBaseAdmin(RemoveDeleteMixin, admin.ModelAdmin):
         "ip",
         "div_id",
         "ad_type_slug",
+        "url",
         "client_id",
         "modified",
         "created",
@@ -938,23 +962,14 @@ class AdBaseAdmin(RemoveDeleteMixin, admin.ModelAdmin):
     paginator = EstimatedCountPaginator
     search_fields = (
         "advertisement__name",
-        "url",
+        "domain",
         "ip",
         "country",
         "user_agent",
         "client_id",
     )
     show_full_result_count = False
 
-    def page_url(self, instance):
-        if instance.url:
-            return format_html(
-                '<a href="{}">{}</a>',
-                instance.url,
-                instance.url,
-            )
-        return None
-
     def has_add_permission(self, request):
         """Clicks and views cannot be added through the admin."""
         return False

diff --git a/adserver/migrations/0101_domainimpression_aggregation.py b/adserver/migrations/0101_domainimpression_aggregation.py
@@ -0,0 +1,34 @@
+# Generated by Django 5.0.9 on 2024-12-03 23:42
+
+import django.db.models.deletion
+import django_extensions.db.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('adserver', '0100_add_offer_domain'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='DomainImpression',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
+                ('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
+                ('date', models.DateField(db_index=True, verbose_name='Date')),
+                ('decisions', models.PositiveIntegerField(default=0, help_text="The number of times the Ad Decision API was called. The server might not respond with an ad if there isn't inventory.", verbose_name='Decisions')),
+                ('offers', models.PositiveIntegerField(default=0, help_text='The number of times an ad was proposed by the ad server. The client may not load the ad (a view) for a variety of reasons ', verbose_name='Offers')),
+                ('views', models.PositiveIntegerField(default=0, help_text='Number of times the ad was legitimately viewed', verbose_name='Views')),
+                ('clicks', models.PositiveIntegerField(default=0, help_text='Number of times the ad was legitimately clicked', verbose_name='Clicks')),
+                ('domain', models.CharField(max_length=1000, verbose_name='Domain')),
+                ('advertisement', models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, related_name='domain_impressions', to='adserver.advertisement')),
+            ],
+            options={
+                'ordering': ('-date',),
+                'unique_together': {('advertisement', 'date', 'domain')},
+            },
+        ),
+    ]
diff --git a/adserver/models.py b/adserver/models.py
@@ -1065,6 +1065,11 @@ def get_exclude_countries_display(self):
     def get_days_display(self):
         return [day.capitalize() for day in self.days]
 
+    def get_analyzed_urls_display(self):
+        if "adserver.analyzer" not in settings.INSTALLED_APPS:
+            return []
+        return [aau.url for aau in self.analyzedadvertiserurl_set.all()]
+
     def show_to_geo(self, geo_data):
         """
         Check if a flight is valid for a given country code.
@@ -2498,6 +2503,30 @@ def __str__(self):
         return "Uplift of %s on %s" % (self.advertisement, self.date)
 
 
+class DomainImpression(BaseImpression):
+    """
+    Create an index of domains for each advertisement
+
+    Indexed one per ad/domain per day.
+    """
+
+    domain = models.CharField(_("Domain"), max_length=1000)
+    advertisement = models.ForeignKey(
+        Advertisement,
+        related_name="domain_impressions",
+        on_delete=models.PROTECT,
+        null=True,
+    )
+
+    class Meta:
+        ordering = ("-date",)
+        unique_together = ("advertisement", "date", "domain")
+
+    def __str__(self):
+        """Simple override."""
+        return "Domain %s of %s on %s" % (self.domain, self.advertisement, self.date)
+
+
 class RotationImpression(BaseImpression):
     """
     Create an index of ads that were rotated.

diff --git a/adserver/tasks.py b/adserver/tasks.py
@@ -29,6 +29,7 @@
 from .models import Advertisement
 from .models import Advertiser
 from .models import AdvertiserImpression
+from .models import DomainImpression
 from .models import Flight
 from .models import GeoImpression
 from .models import KeywordImpression
@@ -509,6 +510,55 @@ def daily_update_uplift(day=None):
         )
 
 
+@app.task()
+def daily_update_domains(day=None):
+    """
+    Generate the daily index of DomainImpressions.
+
+    :arg day: An optional datetime object representing a day
+    """
+    start_date, end_date = get_day(day)
+
+    log.info("Updating domains for %s-%s", start_date, end_date)
+
+    queryset = Offer.objects.using(settings.REPLICA_SLUG).filter(
+        date__gte=start_date,
+        date__lt=end_date,  # Things at UTC midnight should count towards tomorrow
+    )
+
+    for values in (
+        queryset.values("advertisement", "domain")
+        .annotate(
+            total_decisions=Count("domain"),
+            total_offers=Count("domain", filter=Q(advertisement__isnull=False)),
+            total_views=Count("domain", filter=Q(viewed=True)),
+            total_clicks=Count("domain", filter=Q(clicked=True)),
+        )
+        .exclude(domain__isnull=True)
+        .order_by("-total_decisions")
+        .values(
+            "advertisement",
+            "domain",
+            "total_decisions",
+            "total_offers",
+            "total_views",
+            "total_clicks",
+        )
+        .iterator()
+    ):
+        impression, _ = DomainImpression.objects.using("default").get_or_create(
+            advertisement_id=values["advertisement"],
+            domain=values["domain"],
+            date=start_date,
+        )
+        DomainImpression.objects.using("default").filter(pk=impression.pk).update(
+            decisions=values["total_decisions"],
+            offers=values["total_offers"],
+            views=values["total_views"],
+            clicks=values["total_clicks"],
+        )
+
+
 @app.task()
 def daily_update_rotations(day=None):
     """
@@ -698,6 +748,7 @@ def update_previous_day_reports(day=None):
     daily_update_publishers(start_date)  # Important: after daily_update_impressions
     daily_update_keywords(start_date)
     daily_update_uplift(start_date)
+    daily_update_domains(start_date)
     daily_update_rotations(start_date)
     daily_update_regiontopic(start_date)
 

diff --git a/adserver/templates/adserver/includes/flight-metadata.html b/adserver/templates/adserver/includes/flight-metadata.html
@@ -134,6 +134,11 @@
         {% if flight.targeting_parameters.days %}
           <li>{% blocktrans with value=flight.get_days_display|join:', ' %}Days: {{ value }}{% endblocktrans %}</li>
         {% endif %}
+        {% if flight.targeting_parameters.niche_targeting %}
+          {% with flight.get_analyzed_urls_display|join:', ' as urls %}
+            <li>{% blocktrans with value=flight.niche_targeting|floatformat:2 %}Niche: Similarity {{ value }} to {{ urls }}{% endblocktrans %}</li>
+          {% endwith %}
+        {% endif %}
       </ul>
     </dd>
   {% endif %}

diff --git a/adserver/templates/adserver/staff/publisher-payout-finish.html b/adserver/templates/adserver/staff/publisher-payout-finish.html
@@ -51,7 +51,7 @@ <h1>{% block heading %}{% trans 'Finish Payout' %}{% endblock heading %}</h1>
         {% if payout.status == 'emailed' and payout.publisher.payout_url %}
           {% if payout.method == "stripe" %}
             <div class="form-group form-check">
-              <input type="checkbox" class="form-check-input" id="stripe-payout-confirm" name="stripe-payout-confirm">
+              <input type="checkbox" class="form-check-input" id="stripe-payout-confirm" name="stripe-payout-confirm" checked="checked">
               <label class="form-check-label" for="stripe-payout-confirm">
                 <span>{% blocktrans with amount=payout.amount|floatformat:2|intcomma %}Send ${{ amount }} via Stripe Connect{% endblocktrans %}</span>
               </label>

diff --git a/adserver/tests/test_tasks.py b/adserver/tests/test_tasks.py
@@ -10,6 +10,7 @@
 from ..constants import HOUSE_CAMPAIGN
 from ..models import AdImpression
 from ..models import AdvertiserImpression
+from ..models import DomainImpression
 from ..models import Flight
 from ..models import GeoImpression
 from ..models import KeywordImpression
@@ -23,6 +24,7 @@
 from ..tasks import calculate_ad_ctrs
 from ..tasks import calculate_publisher_ctrs
 from ..tasks import daily_update_advertisers
+from ..tasks import daily_update_domains
 from ..tasks import daily_update_geos
 from ..tasks import daily_update_impressions
 from ..tasks import daily_update_keywords
@@ -464,6 +466,7 @@ def setUp(self):
             keywords=["backend"],
             div_id="id_1",
             ad_type_slug=self.text_ad_type.slug,
+            domain="example.com",
         )
         get(
             Offer,
@@ -475,6 +478,7 @@ def setUp(self):
             keywords=["backend"],
             div_id="id_1",
             ad_type_slug=self.text_ad_type.slug,
+            domain="example.com",
         )
         get(
             Offer,
@@ -488,6 +492,7 @@ def setUp(self):
             keywords=["backend"],
             div_id="id_1",
             ad_type_slug=self.text_ad_type.slug,
+            domain="example.com",
         )
         get(
             Offer,
@@ -499,6 +504,7 @@ def setUp(self):
             keywords=["backend"],
             div_id="id_2",
             ad_type_slug=self.text_ad_type.slug,
+            domain="sub.example.com",
         )
         get(
             Offer,
@@ -511,6 +517,7 @@ def setUp(self):
             keywords=["security"],
             div_id="id_2",
             ad_type_slug=self.text_ad_type.slug,
+            domain="example.com",
         )
         get(
             Offer,
@@ -523,6 +530,7 @@ def setUp(self):
             keywords=["security"],
             div_id="id_2",
             ad_type_slug=self.text_ad_type.slug,
+            domain="sub.example.com",
         )
 
     def test_daily_update_impressions(self):
@@ -763,6 +771,21 @@ def test_daily_update_uplift(self):
         self.assertEqual(uplift2.views, 2)
         self.assertEqual(uplift2.clicks, 0)
 
+    def test_daily_update_domains(self):
+        daily_update_domains()
+
+        imp1 = DomainImpression.objects.filter(advertisement=self.ad1, domain="example.com").first()
+        self.assertIsNotNone(imp1)
+        self.assertEqual(imp1.offers, 3)
+        self.assertEqual(imp1.views, 2)
+        self.assertEqual(imp1.clicks, 1)
+
+        imp2 = DomainImpression.objects.filter(advertisement=self.ad2, domain="sub.example.com").first()
+        self.assertIsNotNone(imp2)
+        self.assertEqual(imp2.offers, 1)
+        self.assertEqual(imp2.views, 1)
+        self.assertEqual(imp2.clicks, 0)
+
     def test_daily_update_placements(self):
         # Ad1/id_1 - offered/decision=3, views=2, clicks=1
         # Ad1/id_2 - offered/decisions=1, views=1, clicks=0

diff --git a/config/urls.py b/config/urls.py
@@ -13,6 +13,8 @@
 if settings.DEBUG:
     # This allows the error pages to be debugged during development, just visit
     # these url in browser to see how these error pages look like.
+    from allauth.core.ratelimit import _handler429 as handle_429  # noqa
+
     urlpatterns += [
         path(
             "400/",
@@ -29,6 +31,7 @@
             default_views.page_not_found,
             kwargs={"exception": Exception("Page not Found")},
         ),
+        path("429/", handle_429),
         path("500/", default_views.server_error),
     ]
 

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "ethical-ad-server",
-  "version": "5.11.0",
+  "version": "5.12.0",
   "description": "",
   "main": "index.js",
   "engines": {