diff --git a/tracking/managers.py b/tracking/managers.py index deae9ea..eb0321f 100644 --- a/tracking/managers.py +++ b/tracking/managers.py @@ -1,13 +1,16 @@ from __future__ import division +import logging + from datetime import timedelta from django.utils import timezone from django.contrib.auth import get_user_model from django.db import models -from django.db.models import Count, Avg +from django.db.models import Count, Avg, Sum, Max from tracking.settings import TRACK_PAGEVIEWS, TRACK_ANONYMOUS_USERS from tracking.cache import CacheManager +log = logging.getLogger(__file__) class VisitorManager(CacheManager): def active(self, registered_only=True): @@ -163,25 +166,18 @@ def user_stats(self, start_date=None, end_date=None): else: user_kwargs['visit_history__start_time__isnull'] = False visit_kwargs['start_time__isnull'] = False - - users = list(get_user_model().objects.filter(**user_kwargs).annotate( - visit_count=Count('visit_history'), + + users = get_user_model().objects.filter(**user_kwargs).annotate( + visit_count=Count('visit_history', distinct=True), time_on_site=Avg('visit_history__time_on_site'), + page_count=Count('visit_history__pageviews', distinct=True), + pages_per_visit=Count('visit_history__pageviews', distinct=True)/Count('visit_history', distinct=True), + last_pageview=Max('visit_history__pageviews__view_time'), ).filter(visit_count__gt=0).order_by( '-time_on_site', get_user_model().USERNAME_FIELD, - )) - - # Aggregate pageviews per visit - for user in users: - user.pages_per_visit = user.visit_history.filter( - **visit_kwargs - ).annotate( - page_count=Count('pageviews') - ).filter(page_count__gt=0).aggregate( - pages_per_visit=Avg('page_count'))['pages_per_visit'] - # Lop off the floating point, turn into timedelta - user.time_on_site = timedelta(seconds=int(user.time_on_site)) + ) + return users diff --git a/tracking/migrations/0003_auto_20190722_1418.py b/tracking/migrations/0003_auto_20190722_1418.py new file mode 100755 index 0000000..d871770 --- /dev/null +++ b/tracking/migrations/0003_auto_20190722_1418.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('tracking', '0002_auto_20180918_2014'), + ] + + operations = [ + migrations.AlterField( + model_name='pageview', + name='view_time', + field=models.DateTimeField(db_index=True), + ), + migrations.AlterField( + model_name='visitor', + name='start_time', + field=models.DateTimeField(editable=False, default=django.utils.timezone.now, db_index=True), + ), + ] \ No newline at end of file diff --git a/tracking/models.py b/tracking/models.py old mode 100644 new mode 100755 index 9a341c6..a02b3ba --- a/tracking/models.py +++ b/tracking/models.py @@ -38,7 +38,7 @@ class Visitor(models.Model): # Update to GenericIPAddress in Django 1.4 ip_address = models.CharField(max_length=39, editable=False) user_agent = models.TextField(null=True, editable=False) - start_time = models.DateTimeField(default=timezone.now, editable=False) + start_time = models.DateTimeField(default=timezone.now, editable=False, db_index=True) expiry_age = models.IntegerField(null=True, editable=False) expiry_time = models.DateTimeField(null=True, editable=False) time_on_site = models.IntegerField(null=True, editable=False) @@ -93,7 +93,7 @@ class Pageview(models.Model): referer = models.TextField(null=True, editable=False) query_string = models.TextField(null=True, editable=False) method = models.CharField(max_length=20, null=True) - view_time = models.DateTimeField() + view_time = models.DateTimeField(db_index=True) objects = PageviewManager() diff --git a/tracking/settings.py b/tracking/settings.py index 764f295..a64e6d7 100644 --- a/tracking/settings.py +++ b/tracking/settings.py @@ -21,3 +21,5 @@ TRACK_REFERER = getattr(settings, 'TRACK_REFERER', False) TRACK_QUERY_STRING = getattr(settings, 'TRACK_QUERY_STRING', False) + +TRACK_PAGING_SIZE = getattr(settings, 'TRACK_PAGING_SIZE', 100) \ No newline at end of file diff --git a/tracking/templates/tracking/dashboard.html b/tracking/templates/tracking/dashboard.html old mode 100644 new mode 100755 index a4910f9..57532f0 --- a/tracking/templates/tracking/dashboard.html +++ b/tracking/templates/tracking/dashboard.html @@ -5,24 +5,10 @@

Dashboard - django-tracking2

-
-
- {{ form.as_table }} - -
-
-
-

- Visitor tracking began on - {{ track_start_time|date:"Y-m-d H:i:s" }} -

- {% if warn_incomplete %} -

- The start time precedes the oldest tracked visitor, thus - the stats are not complete for the specified range. -

- {% endif %} -
+ {% include "tracking/snippets/tracking_filters.html" %} + {% if has_pageviews %} + Page centric statistics + {% endif %}
{% include "tracking/snippets/stats.html" %}
diff --git a/tracking/templates/tracking/page_detail.html b/tracking/templates/tracking/page_detail.html new file mode 100755 index 0000000..1b9016c --- /dev/null +++ b/tracking/templates/tracking/page_detail.html @@ -0,0 +1,49 @@ + + + +{{ page_url }} Details - django-tracking2 + + +

{{ page_url }} Details - django-tracking2

+ {% include "tracking/snippets/tracking_filters.html" %} +
+
+
Total Views
+
{{ total_views }} +
# of Unique Visitors
+
{{ visitors }} +
+ {% if pageviews.has_previous %} + previous + {% endif %} + {% if pageviews.has_next %} + next + {% endif %} + + + + + + + + + + {% for pv in pageviews %} + + + + + + + {% endfor %} + +
Pageviews
TimeUserMethodURL
{{ pv.view_time|date:"Y-m-d H:i:s" }}{% firstof pv.visitor.user.get_full_name pv.visitor.user %}{{ pv.method }}{{ page_url }}
+ {% if pageviews.has_previous %} + previous + {% endif %} + {% if pageviews.has_next %} + next + {% endif %} +
+ + \ No newline at end of file diff --git a/tracking/templates/tracking/page_overview.html b/tracking/templates/tracking/page_overview.html new file mode 100755 index 0000000..3f8f0e9 --- /dev/null +++ b/tracking/templates/tracking/page_overview.html @@ -0,0 +1,45 @@ + + + +Page Overview - django-tracking2 + + +

Page Overview - django-tracking2

+ {% include "tracking/snippets/tracking_filters.html" %} +
+
+
Total Page Views
+
{{ total_page_views }} +
# of Pages Viewed
+
{{ total_pages }} +
+ {% if pageview_counts.has_previous %} + previous + {% endif %} + {% if pageview_counts.has_next %} + next + {% endif %} + + + + + + + + {% for c in pageview_counts %} + + + + + {% endfor %} + +
Pages
URL# of Views
{{ c.url }}{{ c.views }}
+ {% if pageview_counts.has_previous %} + previous + {% endif %} + {% if pageview_counts.has_next %} + next + {% endif %} +
+ + \ No newline at end of file diff --git a/tracking/templates/tracking/snippets/stats.html b/tracking/templates/tracking/snippets/stats.html old mode 100644 new mode 100755 index 901f34e..00746aa --- a/tracking/templates/tracking/snippets/stats.html +++ b/tracking/templates/tracking/snippets/stats.html @@ -62,15 +62,17 @@

Registered Users

# Visits Avg. Time on Site Avg. Pages/Visit + Last Pageview {% for user in user_stats %} - {% firstof user.get_full_name user %} + {% firstof user.get_full_name user %} {{ user.visit_count }} {{ user.time_on_site|default_if_none:"n/a" }} {{ user.pages_per_visit|floatformat|default:"n/a" }} + {{ user.last_pageview|default:"n/a" }} {% endfor %} diff --git a/tracking/templates/tracking/snippets/tracking_filters.html b/tracking/templates/tracking/snippets/tracking_filters.html new file mode 100644 index 0000000..9c0de13 --- /dev/null +++ b/tracking/templates/tracking/snippets/tracking_filters.html @@ -0,0 +1,21 @@ +
+
+ {{ form.as_table }} + {% if page_url %} + + {% endif %} + +
+
+
+

+ Visitor tracking began on + {{ track_start_time|date:"Y-m-d H:i:s" }} +

+ {% if warn_incomplete %} +

+ The start time precedes the oldest tracked visitor, thus + the stats are not complete for the specified range. +

+ {% endif %} +
\ No newline at end of file diff --git a/tracking/templates/tracking/visitor_detail.html b/tracking/templates/tracking/visitor_detail.html new file mode 100755 index 0000000..7b11161 --- /dev/null +++ b/tracking/templates/tracking/visitor_detail.html @@ -0,0 +1,86 @@ + + + +{% firstof visit.user.get_full_name visit.user %} {{ visit.start_time|date:"Y-m-d H:i:s" }} - {{ visit.end_time|date:"Y-m-d H:i:s" }} Visit Detail - django-tracking2 + + +

{% firstof visit.user.get_full_name visit.user %} {{ visit.start_time|date:"Y-m-d H:i:s" }} - {{ visit.end_time|date:"Y-m-d H:i:s" }} Visit Detail - django-tracking2

+
+
+
Duration
+
{{ visit.time_on_site }} +
# of Page Views
+
{{ pvcount|default:"n/a" }} +
Avg Time/Page
+
{{ avg_time_per_page|default_if_none:"n/a" }} +
IP
+
{{ visit.ip_address }} +
User Agent
+
{{ visit.user_agent }} +
+ {% if pvcount %} +
+ {% if pageview_stats.has_previous %} + previous + {% endif %} + {% if pageview_stats.has_next %} + next + {% endif %} + + + + + + + + {% for stat in pageview_stats %} + + + + + {% endfor %} + +
Views/Page
URL# of views
{{ stat.url }}{{ stat.views }}
+ {% if pageview_stats.has_previous %} + previous + {% endif %} + {% if pageview_stats.has_next %} + next + {% endif %} +
+ +
+ {% if pageviews.has_previous %} + previous + {% endif %} + {% if pageviews.has_next %} + next + {% endif %} + + + + + + + + + {% for view in pageviews %} + + + + + + {% endfor %} + +
Page Views
TimeMethodURL
{{ view.view_time|date:"Y-m-d H:i:s" }}{{ view.method }}{{ view.url }}
+ {% if pageviews.has_previous %} + previous + {% endif %} + {% if pageviews.has_next %} + next + {% endif %} +
+ {% endif %} +
+ + \ No newline at end of file diff --git a/tracking/templates/tracking/visitor_overview.html b/tracking/templates/tracking/visitor_overview.html new file mode 100755 index 0000000..64139b9 --- /dev/null +++ b/tracking/templates/tracking/visitor_overview.html @@ -0,0 +1,56 @@ + + + +{% firstof user.get_full_name user %} Visits Overview - django-tracking2 + + +

{% firstof user.get_full_name user %} Visits Overview - django-tracking2

+ {% include "tracking/snippets/tracking_filters.html" %} +
+
+
# Visits
+
{{ user.visit_count|default:"n/a" }} +
Avg. Time on Site
+
{{ user.time_on_site|default:"n/a" }} +
Avg. Pages/Visit
+
{{ user.pages_per_visit|floatformat|default:"n/a" }} +
+ + {% if visits.has_previous %} + previous + {% endif %} + {% if visits.has_next %} + next + {% endif %} + + + + + + + {% if has_geoip %} + + {% endif %} + + + {% for visit in visits %} + + + + + {% if has_geoip %} + + {% endif %} + + {% endfor %} + +
Visits by {% firstof user.get_full_name user %}
Start TimeEnd TimePage ViewsLocation
{{ visit.start_time|date:"Y-m-d H:i:s" }}{{ visit.end_time|date:"Y-m-d H:i:s" }}{{ visit.pageviews.count }}{{ visit.geoip_data.city }}, {{ visit.geoip_data.region }}, {{ visit.geoip_data.country_code }} {{ visit.geoip_data.postal_code }}
+ {% if visits.has_previous %} + previous + {% endif %} + {% if visits.has_next %} + next + {% endif %} +
+ + diff --git a/tracking/templates/tracking/visitor_page_detail.html b/tracking/templates/tracking/visitor_page_detail.html new file mode 100755 index 0000000..17959bf --- /dev/null +++ b/tracking/templates/tracking/visitor_page_detail.html @@ -0,0 +1,51 @@ + + + +{% firstof user.get_full_name user %}'s Visits involving {{ page_url }} - django-tracking2 + + +

{% firstof user.get_full_name user %}'s Visits involving {{ page_url }} - django-tracking2

+ {% include "tracking/snippets/tracking_filters.html" %} +
+
+
Total Views
+
{{ total_views }} +
Avg Views/Visit
+
{{ avg_views_per_visit }} +
+ {% if visits.has_previous %} + previous + {% endif %} + {% if visits.has_next %} + next + {% endif %} + + + + + + {% if has_geoip %} + + {% endif %} + + + {% for visit in visits %} + + + + {% if has_geoip %} + + {% endif %} + + {% endfor %} + +
Visits by {% firstof user.get_full_name user %} involving {{ page_url }}
Start TimeEnd TimeLocation
{{ visit.start_time|date:"Y-m-d H:i:s" }}{{ visit.end_time|date:"Y-m-d H:i:s" }}{{ visit.geoip_data.city }}, {{ visit.geoip_data.region }}, {{ visit.geoip_data.country_code }} {{ visit.geoip_data.postal_code }}
+ {% if visits.has_previous %} + previous + {% endif %} + {% if visits.has_next %} + next + {% endif %} +
+ + \ No newline at end of file diff --git a/tracking/templates/tracking/visitor_pageview_detail.html b/tracking/templates/tracking/visitor_pageview_detail.html new file mode 100644 index 0000000..6e1c66d --- /dev/null +++ b/tracking/templates/tracking/visitor_pageview_detail.html @@ -0,0 +1,26 @@ + + + +{% firstof pageview.visitor.user.get_full_name pageview.visitor.user.user %}'s View of {{ pageview.url }} at {{ pageview.view_time|date:"Y-m-d H:i:s" }} - django-tracking2 + + +

{% firstof pageview.visitor.user.get_full_name pageview.visitor.user.user %}'s View of {{ pageview.url }} at {{ pageview.view_time|date:"Y-m-d H:i:s" }} - django-tracking2

+
+
+
Method
+
{{ pageview.method }} +
Duration
+
{{ duration }} + {% if pageview.referer %} +
Referrer
+
{{ pageview.referer }} + {% endif %} + {% if pageview.query_string %} +
Query String
+
{{ pageview.query_string }} + {% endif %} +
+
+ Page details across all visitors + + \ No newline at end of file diff --git a/tracking/tests/test_geoip.py b/tracking/tests/test_geoip.py index 19920e3..911b627 100644 --- a/tracking/tests/test_geoip.py +++ b/tracking/tests/test_geoip.py @@ -31,7 +31,7 @@ dj_version = django.get_version() broken_geoip = (dj_version[:3] == '1.5') and (sys.version_info[0] == 3) - +@skipIf(not HAS_GEOIP, "Don't run tests if don't have GEOIP") class GeoIPTestCase(TestCase): def test_geoip_none(self): diff --git a/tracking/tests/test_managers.py b/tracking/tests/test_managers.py index 912375c..4673132 100644 --- a/tracking/tests/test_managers.py +++ b/tracking/tests/test_managers.py @@ -84,7 +84,7 @@ def test_visitor_stats(self): # now expand the end time to include `future` as well end_time = start_time + timedelta(days=3) stats = Visitor.objects.stats(start_time, end_time) - self.assertEqual(stats['pages_per_visit'], 4 / 3) + self.assertAlmostEqual(stats['pages_per_visit'], 4 / 3, places=4) guests = { 'time_on_site': timedelta(seconds=30), 'unique': 1, @@ -156,7 +156,7 @@ def test_user_stats(self): user = stats[0] self.assertEqual(user.username, self.user2.username) self.assertEqual(user.visit_count, 1) - self.assertEqual(user.time_on_site, timedelta(seconds=30)) + self.assertAlmostEqual(user.time_on_site, 30) self.assertEqual(user.pages_per_visit, 1) # no start_time @@ -166,13 +166,13 @@ def test_user_stats(self): user1 = stats[1] self.assertEqual(user1.username, self.user1.username) self.assertEqual(user1.visit_count, 1) - self.assertEqual(user1.time_on_site, timedelta(seconds=30)) + self.assertAlmostEqual(user1.time_on_site, 30) self.assertEqual(user1.pages_per_visit, 2.0) user2 = stats[0] self.assertEqual(user2.username, self.user2.username) self.assertEqual(user2.visit_count, 1) - self.assertEqual(user2.time_on_site, timedelta(seconds=30)) + self.assertAlmostEqual(user2.time_on_site, 30) self.assertEqual(user2.pages_per_visit, 1) def test_pageview_stats(self): diff --git a/tracking/tests/test_views.py b/tracking/tests/test_views.py index 28f9692..eff9e4e 100644 --- a/tracking/tests/test_views.py +++ b/tracking/tests/test_views.py @@ -1,4 +1,4 @@ -from datetime import timedelta +from datetime import timedelta, datetime from django.contrib.admin.sites import AdminSite from django.contrib.auth.models import User @@ -11,16 +11,16 @@ from mock import patch from tracking.admin import VisitorAdmin -from tracking.models import Visitor +from tracking.models import Visitor, Pageview class ViewsTestCase(TestCase): def setUp(self): self.auth = {'username': 'john', 'password': 'smith'} - user = User.objects.create_user(**self.auth) - user.is_superuser = True - user.save() + self.user = User.objects.create_user(**self.auth) + self.user.is_superuser = True + self.user.save() self.assertTrue(self.client.login(**self.auth)) def test_dashboard_default(self): @@ -71,6 +71,237 @@ def test_logout_tracking(self, mock_end): self.assertEqual(visitor.end_time, self.now) self.assertTrue(visitor.time_on_site > 0) + def test_visitor_overview_default(self): + # make a non PAGEVIEW tracking request + Visitor.objects.create( + session_key='skey', + ip_address='127.0.0.1', + user=self.user, + time_on_site = 0, + ) + response = self.client.get('/tracking/visitors/%s/' % self.user.pk) + self.assertEqual(response.status_code, 200) + self.assertEqual( + response.context['user'], + self.user) + + def test_visitor_overview_times(self): + # make a non PAGEVIEW tracking request + Visitor.objects.create( + session_key='skey', + ip_address='127.0.0.1', + user=self.user, + time_on_site = 0, + ) + response = self.client.get( + '/tracking/visitors/%s/?start=2014-11&end=2014-12-01' % self.user.pk) + self.assertEqual(response.status_code, 200) + + def test_visitor_overview_no_records(self): + response = self.client.get( + '/tracking/visitors/%s/?start=2014-11&end=2014-12-01' % self.user.pk) + # Gracefully handle when then there are o records of visits within time range + self.assertEqual(response.status_code, 200) + + def test_visitor_overview_times_bad(self): + # make a non PAGEVIEW tracking request + Visitor.objects.create( + session_key='skey', + ip_address='127.0.0.1', + user=self.user, + time_on_site = 0, + ) + response = self.client.get( + '/tracking/visitors/%s/?start=2014-aa&end=2014-12-01' % self.user.pk) + self.assertEqual(response.status_code, 200) + self.assertContains(response, 'Enter a valid date/time.') + + def test_visitor_detail_visitor_does_not_exist(self): + response = self.client.get( + '/tracking/visits/asdf/') + self.assertEqual(response.status_code, 404) + + def test_visitor_detail_no_pageviews(self): + visitor = Visitor.objects.create( + session_key='skey', + ip_address='127.0.0.1', + user=self.user, + time_on_site = 0, + ) + response = self.client.get( + '/tracking/visits/%s/' % visitor.pk) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.context['pvcount'], 0) + + def test_visitor_detail_has_pageview(self): + visitor = Visitor.objects.create( + session_key='skey', + ip_address='127.0.0.1', + user=self.user, + time_on_site = 0, + ) + Pageview.objects.create( + visitor=visitor, + url='/an/url', + referer='/an/url', + query_string='?a=string', + method='PUT', + view_time=datetime.fromtimestamp(1565033030), + ) + response = self.client.get( + '/tracking/visits/%s/?start=2018&end=2020' % visitor.pk) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.context['pageviews']), 1) + self.assertEqual(len(response.context['pageview_stats']), 1) + self.assertEqual(response.context['pvcount'], 1) + self.assertEqual(response.context['visit'], visitor) + + def test_visitor_page_detail_page_does_not_exist(self): + visitor = Visitor.objects.create( + session_key='skey', + ip_address='127.0.0.1', + user=self.user, + time_on_site = 0, + start_time=datetime.fromtimestamp(1565033030), + ) + response = self.client.get( + '/tracking/visitors/%s/page/?page_url=asdf/' % self.user.pk) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.context['total_views'], 0) + self.assertEqual(response.context['avg_views_per_visit'], 0) + self.assertEqual(len(response.context['visits']), 0) + self.assertEqual(response.context['user'], self.user) + + def test_visitor_page_detail_user_does_not_exist(self): + visitor = Visitor.objects.create( + session_key='skey', + ip_address='127.0.0.1', + user=self.user, + time_on_site = 0, + ) + pv = Pageview.objects.create( + visitor=visitor, + url='/an/url', + referer='/an/url', + query_string='?a=string', + method='PUT', + view_time=datetime.fromtimestamp(1565033030), + ) + response = self.client.get( + '/tracking/visitors/80085/page/?page_url=%s&start=2018&end=2020' % pv.url) + self.assertEqual(response.status_code, 404) + + def test_visitor_page_detail_one_pageview(self): + visitor = Visitor.objects.create( + session_key='skey', + ip_address='127.0.0.1', + user=self.user, + time_on_site = 0, + start_time=datetime.fromtimestamp(1565033030), + ) + pv = Pageview.objects.create( + visitor=visitor, + url='/an/url', + referer='/an/url', + query_string='?a=string', + method='PUT', + view_time=datetime.fromtimestamp(1565033030), + ) + response = self.client.get( + '/tracking/visitors/%s/page/?page_url=%s&start=2018&end=2020' % (self.user.pk, pv.url)) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.context['total_views'], 1) + self.assertEqual(response.context['avg_views_per_visit'], 1) + self.assertEqual(len(response.context['visits']), 1) + self.assertEqual(response.context['user'], self.user) + self.assertEqual(response.context['page_url'], pv.url) + + def test_visitor_pageview_pageview_does_not_exist(self): + response = self.client.get( + '/tracking/visitors/80085/pageview/1337/') + self.assertEqual(response.status_code, 404) + + def test_visitor_pageview_one_pageview_exists(self): + visitor = Visitor.objects.create( + session_key='skey', + ip_address='127.0.0.1', + user=self.user, + time_on_site = 0, + ) + pv = Pageview.objects.create( + visitor=visitor, + url='/an/url', + referer='/an/url', + query_string='?a=string', + method='PUT', + view_time=datetime.fromtimestamp(1565033030), + ) + response = self.client.get( + '/tracking/visitors/%s/pageview/%s/' % (self.user.pk, pv.pk)) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.context['pageview'], pv) + self.assertEqual(response.context['duration'], None) + + def test_visitor_pageview_two_pageviews(self): + visitor = Visitor.objects.create( + session_key='skey', + ip_address='127.0.0.1', + user=self.user, + time_on_site = 0, + ) + pv1_view_time = datetime.fromtimestamp(1565033030) + pv2_view_time = datetime.fromtimestamp(1565034030) + pv = Pageview.objects.create( + visitor=visitor, + url='/an/url', + referer='/an/url', + query_string='?a=string', + method='PUT', + view_time=pv1_view_time, + ) + Pageview.objects.create( + visitor=visitor, + url='/an/url', + referer='/an/url', + query_string='?a=string', + method='PUT', + view_time=pv2_view_time , + ) + response = self.client.get( + '/tracking/visitors/%s/pageview/%s/' % (self.user.pk, pv.pk)) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.context['pageview'], pv) + self.assertEqual(response.context['duration'], pv2_view_time - pv1_view_time) + + def test_visitor_page_overview_no_pageviews(self): + response = self.client.get( + '/tracking/pages/') + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.context['pageview_counts']), 0) + self.assertEqual(response.context['total_page_views'], 0) + self.assertEqual(response.context['total_pages'], 0) + + def test_visitor_page_overview_one_pageviews(self): + visitor = Visitor.objects.create( + session_key='skey', + ip_address='127.0.0.1', + user=self.user, + time_on_site = 0, + ) + Pageview.objects.create( + visitor=visitor, + url='/an/url', + referer='/an/url', + query_string='?a=string', + method='PUT', + view_time=datetime.fromtimestamp(1565033030), + ) + response = self.client.get( + '/tracking/pages/?start=2018&end=2020') + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.context['pageview_counts']), 1) + self.assertEqual(response.context['total_page_views'], 1) + self.assertEqual(response.context['total_pages'], 1) class AdminViewTestCase(TestCase): diff --git a/tracking/urls.py b/tracking/urls.py index ffe3b6a..691e946 100644 --- a/tracking/urls.py +++ b/tracking/urls.py @@ -1,7 +1,21 @@ from django.conf.urls import url -from tracking.views import dashboard +from tracking.views import ( + dashboard, + visitor_overview, + visitor_detail, + visitor_page_detail, + visitor_pageview_detail, + page_overview, + page_detail, +) urlpatterns = [ url(r'^$', dashboard, name='tracking-dashboard'), + url(r'^visitors/(?P.*)/page/$', visitor_page_detail, name='tracking-visitor-page-detail'), + url(r'^visitors/(?P.*)/pageview/(?P.*)/$', visitor_pageview_detail, name='tracking-pageview-detail'), + url(r'^visitors/(?P.*)/$', visitor_overview, name='tracking-visitor-overview'), + url(r'^visits/(?P.*)/$', visitor_detail, name='tracking-visitor-detail'), + url(r'^pages/$', page_overview, name='tracking-page-overview'), + url(r'^page/$', page_detail, name='tracking-page-detail'), ] diff --git a/tracking/utils.py b/tracking/utils.py index bf36af5..61b0507 100644 --- a/tracking/utils.py +++ b/tracking/utils.py @@ -1,14 +1,33 @@ from __future__ import division +from datetime import timedelta + +from django import forms +from django.utils.timezone import now from django.core.exceptions import ValidationError from django.core.validators import validate_ipv46_address +from tracking.models import Visitor + headers = ( 'HTTP_CLIENT_IP', 'HTTP_X_FORWARDED_FOR', 'HTTP_X_FORWARDED', 'HTTP_X_CLUSTERED_CLIENT_IP', 'HTTP_FORWARDED_FOR', 'HTTP_FORWARDED', 'REMOTE_ADDR' ) +# tracking wants to accept more formats than default, here they are +input_formats = [ + '%Y-%m-%d %H:%M:%S', # '2006-10-25 14:30:59' + '%Y-%m-%d %H:%M', # '2006-10-25 14:30' + '%Y-%m-%d', # '2006-10-25' + '%Y-%m', # '2006-10' + '%Y', # '2006' +] + + +class TimeRangeForm(forms.Form): + start = forms.DateTimeField(required=False, input_formats=input_formats) + end = forms.DateTimeField(required=False, input_formats=input_formats) def get_ip_address(request): for header in headers: @@ -25,3 +44,21 @@ def get_ip_address(request): def total_seconds(delta): day_seconds = (delta.days * 24 * 3600) + delta.seconds return (delta.microseconds + day_seconds * 10**6) / 10**6 + +def processTimeRangeForm(request): + end_time = now() + start_time = end_time - timedelta(days=7) + defaults = {'start': start_time, 'end': end_time} + form = TimeRangeForm(data=(request.GET if 'end' in request.GET else None) or defaults) + if form.is_valid(): + start_time = form.cleaned_data['start'] + end_time = form.cleaned_data['end'] + # determine when tracking began + try: + obj = Visitor.objects.order_by('start_time')[0] + track_start_time = obj.start_time + except (IndexError, Visitor.DoesNotExist): + track_start_time = now() + # If the start_date is before tracking began, warn about incomplete data + warn_incomplete = (start_time < track_start_time) + return (start_time, end_time, track_start_time, warn_incomplete, form) \ No newline at end of file diff --git a/tracking/views.py b/tracking/views.py old mode 100644 new mode 100755 index b734c4a..e7a8788 --- a/tracking/views.py +++ b/tracking/views.py @@ -1,57 +1,37 @@ import logging from datetime import timedelta +from functools import reduce -from django import forms -from django.shortcuts import render +from django.shortcuts import ( + render, + get_object_or_404, +) +from django.http import HttpResponseNotFound +from django.contrib.auth import get_user_model from django.contrib.auth.decorators import permission_required -from django.utils.timezone import now +from django.db.models import Count, Avg, Sum +from django.core.paginator import Paginator from tracking.models import Visitor, Pageview -from tracking.settings import TRACK_PAGEVIEWS +from tracking.settings import ( + TRACK_PAGEVIEWS, + TRACK_PAGING_SIZE, + TRACK_USING_GEOIP, +) +from tracking.utils import processTimeRangeForm log = logging.getLogger(__file__) -# tracking wants to accept more formats than default, here they are -input_formats = [ - '%Y-%m-%d %H:%M:%S', # '2006-10-25 14:30:59' - '%Y-%m-%d %H:%M', # '2006-10-25 14:30' - '%Y-%m-%d', # '2006-10-25' - '%Y-%m', # '2006-10' - '%Y', # '2006' -] - - -class DashboardForm(forms.Form): - start = forms.DateTimeField(required=False, input_formats=input_formats) - end = forms.DateTimeField(required=False, input_formats=input_formats) - - @permission_required('tracking.visitor_log') def dashboard(request): - "Counts, aggregations and more!" - end_time = now() - start_time = end_time - timedelta(days=7) - defaults = {'start': start_time, 'end': end_time} - - form = DashboardForm(data=request.GET or defaults) - if form.is_valid(): - start_time = form.cleaned_data['start'] - end_time = form.cleaned_data['end'] - - # determine when tracking began - try: - obj = Visitor.objects.order_by('start_time')[0] - track_start_time = obj.start_time - except (IndexError, Visitor.DoesNotExist): - track_start_time = now() - - # If the start_date is before tracking began, warn about incomplete data - warn_incomplete = (start_time < track_start_time) + (start_time, end_time, track_start_time, warn_incomplete, form) = processTimeRangeForm(request) # queries take `date` objects (for now) user_stats = Visitor.objects.user_stats(start_time, end_time) visitor_stats = Visitor.objects.stats(start_time, end_time) + for us in user_stats: + us.time_on_site = timedelta(seconds=us.time_on_site) if TRACK_PAGEVIEWS: pageview_stats = Pageview.objects.stats(start_time, end_time) else: @@ -64,5 +44,175 @@ def dashboard(request): 'user_stats': user_stats, 'visitor_stats': visitor_stats, 'pageview_stats': pageview_stats, + 'start_time': start_time, + 'end_time': end_time, + 'has_pageviews': TRACK_PAGEVIEWS, } return render(request, 'tracking/dashboard.html', context) + +@permission_required('tracking.visitor_log') +def visitor_overview(request, user_id): + (start_time, end_time, track_start_time, warn_incomplete, form) = processTimeRangeForm(request) + + page = request.GET.get('page', 1) + # queries take `date` objects (for now) + user = Visitor.objects.user_stats(start_time, end_time).filter(pk=user_id).first() + if user: + user.time_on_site = timedelta(seconds=user.time_on_site) + else: + # User did not visit at all during this period. Need name but not stats + user = get_object_or_404(get_user_model(), pk=user_id) + visits = Visitor.objects.filter(user=user, start_time__range=(start_time, end_time)) + paginator = Paginator(visits, TRACK_PAGING_SIZE) + log.critical(visits) + + context = { + 'form': form, + 'track_start_time': track_start_time, + 'warn_incomplete': warn_incomplete, + 'visits': paginator.page(page), + 'user': user, + 'start_time': start_time, + 'end_time': end_time, + 'has_geoip': TRACK_USING_GEOIP, + } + return render(request, 'tracking/visitor_overview.html', context) + +@permission_required('tracking.visitor_log') +def visitor_detail(request, visit_id): + pvpage = request.GET.get('pvpage', 1) + pvspage = request.GET.get('pvspage', 1) + visit = get_object_or_404(Visitor, pk=visit_id) + visit.time_on_site = timedelta(seconds=visit.time_on_site) + pvcount = visit.pageviews.count() + pageviews = visit.pageviews.order_by('-view_time') + pageview_stats = visit.pageviews.values('method', 'url').annotate(views=Count('url')).order_by('-views') + pvspaginator = Paginator(pageview_stats, TRACK_PAGING_SIZE) + pvpaginator = Paginator(pageviews, TRACK_PAGING_SIZE) + + context = { + 'visit': visit, + 'pageviews': pvpaginator.page(pvpage), + 'pageview_stats': pvspaginator.page(pvspage), + 'pvcount': pvcount, + 'avg_time_per_page': visit.time_on_site/pvcount if pvcount else None + } + return render(request, 'tracking/visitor_detail.html', context) + +@permission_required('tracking.visitor_log') +def visitor_page_detail(request, user_id): + try: + page_url = request.GET['page_url'] + except: + return HttpResponseNotFound() + + (start_time, end_time, track_start_time, warn_incomplete, form) = processTimeRangeForm(request) + + page = request.GET.get('page', 1) + user = get_object_or_404(get_user_model(), pk=user_id) + relevant_visits = Visitor.objects.filter( + pageviews__url=page_url, + user__pk=user_id, + start_time__lt=end_time, + ) + if start_time: + relevant_visits = relevant_visits.filter(start_time__gte=start_time) + else: + relevant_visits = relevant_visits.filter(start_time__isnull=False) + + aggs = relevant_visits.values('pk').annotate(views=Count('pageviews')).aggregate( + Avg('views'), + Sum('views') + ) + visits = relevant_visits.distinct().order_by( + 'end_time', + 'start_time' + ) + paginator = Paginator(visits, TRACK_PAGING_SIZE) + + context = { + 'total_views': aggs['views__sum'] if aggs['views__sum'] else 0, + 'avg_views_per_visit': aggs['views__avg'] if aggs['views__avg'] else 0, + 'visits': paginator.page(page), + 'user': user, + 'page_url': page_url, + 'form': form, + 'track_start_time': track_start_time, + 'warn_incomplete': warn_incomplete, + 'start_time': start_time, + 'end_time': end_time, + 'has_geoip': TRACK_USING_GEOIP, + } + return render(request, 'tracking/visitor_page_detail.html', context) + +@permission_required('tracking.visitor_log') +def visitor_pageview_detail(request, user_id, pageview_id): + pageview = get_object_or_404(Pageview, pk=pageview_id, visitor__user_id=user_id) + next_pv = Pageview.objects.filter( + visitor__user_id=user_id, + view_time__gt=pageview.view_time, + ).order_by('view_time').first() + if next_pv: + duration = next_pv.view_time - pageview.view_time + else: + duration = None + + context = { + 'pageview': pageview, + 'duration': duration, + } + return render(request, 'tracking/visitor_pageview_detail.html', context) + +@permission_required('tracking.visitor_log') +def page_overview(request): + (start_time, end_time, track_start_time, warn_incomplete, form) = processTimeRangeForm(request) + + page = request.GET.get('page', 1) + relevant_pvs = Pageview.objects.filter(view_time__lt=end_time) + if start_time: + relevant_pvs = relevant_pvs.filter(view_time__gte=start_time) + pageview_counts = relevant_pvs.values('url').annotate(views=Count('url')).order_by('-views') + paginator = Paginator(pageview_counts, TRACK_PAGING_SIZE) + + context = { + 'pageview_counts': paginator.page(page), + 'total_page_views': reduce(lambda acc, c: acc + c['views'], pageview_counts, 0), + 'total_pages': len(pageview_counts), + 'form': form, + 'track_start_time': track_start_time, + 'warn_incomplete': warn_incomplete, + 'start_time': start_time, + 'end_time': end_time, + } + return render(request, 'tracking/page_overview.html', context) + +@permission_required('tracking.visitor_log') +def page_detail(request): + try: + page_url = request.GET['page_url'] + except: + return HttpResponseNotFound() + + (start_time, end_time, track_start_time, warn_incomplete, form) = processTimeRangeForm(request) + + page = request.GET.get('page', 1) + relevant_pvs = Pageview.objects.filter(view_time__lt=end_time) + if start_time: + relevant_pvs = relevant_pvs.filter(view_time__gte=start_time) + pageviews = relevant_pvs.filter(url=page_url).order_by('-view_time') + pv_count = pageviews.count() + uniqueVisitors = relevant_pvs.values('visitor_id').distinct().count() + paginator = Paginator(pageviews, TRACK_PAGING_SIZE) + + context = { + 'total_views': pv_count, + 'visitors': uniqueVisitors, + 'pageviews': paginator.page(page), + 'page_url': page_url, + 'form': form, + 'track_start_time': track_start_time, + 'warn_incomplete': warn_incomplete, + 'start_time': start_time, + 'end_time': end_time, + } + return render(request, 'tracking/page_detail.html', context)