diff --git a/cl/assets/static-global/css/override.css b/cl/assets/static-global/css/override.css index a9f8668a8f..2764e9cecd 100644 --- a/cl/assets/static-global/css/override.css +++ b/cl/assets/static-global/css/override.css @@ -107,6 +107,9 @@ a.button:active{ /* border:1px solid #6299c5; */ color:#fff; } +.shadow { + box-shadow: 2px 2px 5px #000000; +} #main-query-box { /* margin: 0 auto; */ diff --git a/cl/people_db/admin.py b/cl/people_db/admin.py index f689f260ca..689b472bf2 100644 --- a/cl/people_db/admin.py +++ b/cl/people_db/admin.py @@ -6,8 +6,8 @@ Education, School, Person, Position, RetentionEvent, Race, PoliticalAffiliation, Source, ABARating, PartyType, Party, Role, Attorney, AttorneyOrganization, - AttorneyOrganizationAssociation -) + AttorneyOrganizationAssociation, + FinancialDisclosure) class RetentionEventInline(admin.TabularInline): @@ -94,6 +94,18 @@ class ABARatingInline(admin.TabularInline): extra = 1 +@admin.register(FinancialDisclosure) +class FinancialDisclosureAdmin(admin.ModelAdmin): + raw_id_fields = ( + 'person', + ) + + def save_model(self, request, obj, form, change): + obj.save() + from cl.people_db.tasks import make_png_thumbnail_from_pdf + make_png_thumbnail_from_pdf.delay(obj.pk) + + @admin.register(Person) class PersonAdmin(admin.ModelAdmin, CSSAdminMixin): prepopulated_fields = {"slug": ['name_first', 'name_middle', 'name_last', diff --git a/cl/people_db/migrations/0031_add_financial_disclosures.py b/cl/people_db/migrations/0031_add_financial_disclosures.py new file mode 100644 index 0000000000..08d70b2086 --- /dev/null +++ b/cl/people_db/migrations/0031_add_financial_disclosures.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models +import cl.lib.storage + + +class Migration(migrations.Migration): + + dependencies = [ + ('people_db', '0030_auto_20170714_0700'), + ] + + operations = [ + migrations.CreateModel( + name='FinancialDisclosure', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('year', models.SmallIntegerField(help_text=b'The year that the disclosure corresponds with', db_index=True)), + ('filepath', models.FileField(help_text=b'The disclosure report itself', storage=cl.lib.storage.IncrementingFileSystemStorage(), upload_to=b'financial-disclosures/', db_index=True)), + ('thumbnail', models.FileField(help_text=b'A thumbnail of the first page of the disclosure form', storage=cl.lib.storage.IncrementingFileSystemStorage(), null=True, upload_to=b'financial-disclosures/thumbnails/', blank=True)), + ('thumbnail_status', models.SmallIntegerField(default=0, help_text=b'The status of the thumbnail generation', choices=[(0, b'Thumbnail needed'), (1, b'Thumbnail completed successfully'), (2, b'Unable to generate thumbnail')])), + ('page_count', models.SmallIntegerField(help_text=b'The number of pages in the disclosure report')), + ('person', models.ForeignKey(related_name='financial_disclosures', to='people_db.Person', help_text=b'The person that the document is associated with.')), + ], + options={ + 'ordering': ('-year',), + }, + ), + ] diff --git a/cl/people_db/models.py b/cl/people_db/models.py index 2eacf47c14..52d0932bfc 100644 --- a/cl/people_db/models.py +++ b/cl/people_db/models.py @@ -21,6 +21,7 @@ validate_supervisor, ) from cl.lib.search_index_utils import solr_list, null_map, normalize_search_dicts +from cl.lib.storage import IncrementingFileSystemStorage from cl.lib.string_utils import trunc from cl.search.models import Court @@ -1202,6 +1203,57 @@ def clean_fields(self, *args, **kwargs): super(ABARating, self).clean_fields(*args, **kwargs) +class FinancialDisclosure(models.Model): + """A simple table to hold references to financial disclosure forms""" + THUMBNAIL_NEEDED = 0 + THUMBNAIL_COMPLETE = 1 + THUMBNAIL_FAILED = 2 + THUMBNAIL_STATUSES = ( + (THUMBNAIL_NEEDED, "Thumbnail needed"), + (THUMBNAIL_COMPLETE, "Thumbnail completed successfully"), + (THUMBNAIL_FAILED, 'Unable to generate thumbnail'), + ) + person = models.ForeignKey( + Person, + help_text="The person that the document is associated with.", + related_name='financial_disclosures', + ) + year = models.SmallIntegerField( + help_text="The year that the disclosure corresponds with", + db_index=True, + ) + filepath = models.FileField( + help_text="The disclosure report itself", + upload_to='financial-disclosures/', + storage=IncrementingFileSystemStorage(), + db_index=True, + ) + thumbnail = models.FileField( + help_text="A thumbnail of the first page of the disclosure form", + upload_to="financial-disclosures/thumbnails/", + storage=IncrementingFileSystemStorage(), + null=True, + blank=True, + ) + thumbnail_status = models.SmallIntegerField( + help_text="The status of the thumbnail generation", + choices=THUMBNAIL_STATUSES, + default=0, + ) + page_count = models.SmallIntegerField( + help_text="The number of pages in the disclosure report", + ) + + class Meta: + ordering = ('-year',) + + def save(self, *args, **kwargs): + super(FinancialDisclosure, self).save(*args, **kwargs) + if not self.pk: + from cl.people_db.tasks import make_png_thumbnail_from_pdf + make_png_thumbnail_from_pdf.delay(self.pk) + + class PartyType(models.Model): docket = models.ForeignKey( 'search.Docket', diff --git a/cl/people_db/tasks.py b/cl/people_db/tasks.py new file mode 100644 index 0000000000..b4aae6b67d --- /dev/null +++ b/cl/people_db/tasks.py @@ -0,0 +1,40 @@ +import subprocess +from tempfile import NamedTemporaryFile + +from django.core.files import File + +from cl.celery import app +from cl.people_db.models import FinancialDisclosure + + +@app.task +def make_png_thumbnail_from_pdf(pk, width=350): + """Create a png thumbnail from a financial disclosure PDF""" + fd = FinancialDisclosure.objects.get(pk=pk) + # Use a temporary location for the file, then save it to the model. + with NamedTemporaryFile(prefix='financial_disclosure_', + suffix=".png") as tmp: + convert = [ + 'convert', + # Only do the first page. + '%s[0]' % fd.filepath.path, + '-resize', '%s' % width, + # This and the next line handle transparency problems + '-background', 'white', + '-alpha', 'remove', + tmp.name, + ] + p = subprocess.Popen(convert, close_fds=True, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, universal_newlines=True) + stdout, stderr = p.communicate() + + if p.returncode != 0: + fd.thumbnail_status = fd.THUMBNAIL_FAILED + fd.save() + return fd.pk + + fd.thumbnail_status = fd.THUMBNAIL_COMPLETE + filename = '%s.thumb.%sw.png' % (fd.person.slug, width) + fd.thumbnail.save(filename, File(tmp)) + + return fd diff --git a/cl/people_db/templates/financial_disclosures_for_somebody.html b/cl/people_db/templates/financial_disclosures_for_somebody.html new file mode 100644 index 0000000000..b8ab588611 --- /dev/null +++ b/cl/people_db/templates/financial_disclosures_for_somebody.html @@ -0,0 +1,63 @@ +{% extends "base.html" %} +{% load humanize %} +{% load static %}{% get_static_prefix as STATIC_PREFIX %} + +{% block title %} + Judicial Financial Disclosure Forms for {{ title }} – CourtListener.com +{% endblock %} +{% block og_title %} + Judicial Financial Disclosure Forms for {{ title }} – CourtListener.com +{% endblock %} +{% block description %} + Judicial Financial Disclosure Forms for {{ title }} at the Judicial Financial Disclosures Database. A collaboration of Demand Progress, Fix the Court, Free Law Project, and MuckRock, containing thousands of judicial financial disclosure forms. +{% endblock %} +{% block og_description %} + Judicial Financial Disclosure Forms for {{ title }} at the Judicial Financial Disclosures Database. A collaboration of Demand Progress, Fix the Court, Free Law Project, and MuckRock, containing thousands of judicial financial disclosure forms. +{% endblock %} + +{% block navbar-p %}active{% endblock %} + +{% block sidebar %} + +{% endblock %} + +{% block content %} +
+

Financial Disclosure Forms for {{ title }}

+

View profile

+ + {% for disclosure in person.financial_disclosures.all %} +
+
+

Disclosure for {{ disclosure.year }}

+

{{ disclosure.page_count }} page{{ disclosure.page_count|pluralize }}

+  Download +
+
+ + Thumbnail of disclosure form + +
+ {% if not forloop.last %} +
+

+
+ {% endif %} +
+ {% empty %} +

No disclosure forms found.

+ {% endfor %} +
+{% endblock %} + diff --git a/cl/people_db/templates/financial_disclosures_home.html b/cl/people_db/templates/financial_disclosures_home.html index ea663c4a00..2deba2255d 100644 --- a/cl/people_db/templates/financial_disclosures_home.html +++ b/cl/people_db/templates/financial_disclosures_home.html @@ -1,5 +1,7 @@ {% extends "base.html" %} {% load humanize %} +{% load partition_util %} +{% load cache %} {% block title %} Judicial Financial Disclosures Database – CourtListener.com @@ -8,17 +10,21 @@ Judicial Financial Disclosures Database – CourtListener.com {% endblock %} {% block description %} - The Judicial Financial Disclosures database at CourtListener.com contains {{ disclosure_count|intcomma }} financial disclosure forms, and is the largest collection online. A collaboration of Demand Progress, Fix the Court, Free Law Project, and MuckRock. + The Judicial Financial Disclosures Database at CourtListener.com contains {{ disclosure_count|intcomma }} financial disclosure forms, and is the largest collection online. A collaboration of Demand Progress, Fix the Court, Free Law Project, and MuckRock. {% endblock %} {% block og_description %} - The Judicial Financial Disclosures database at CourtListener.com contains {{ disclosure_count|intcomma }} financial disclosure forms, and is the largest collection online. A collaboration of Demand Progress, Fix the Court, Free Law Project, and MuckRock. + The Judicial Financial Disclosures Database at CourtListener.com contains {{ disclosure_count|intcomma }} financial disclosure forms, and is the largest collection online. A collaboration of Demand Progress, Fix the Court, Free Law Project, and MuckRock. {% endblock %} {% block navbar-p %}active{% endblock %} {% block footer-scripts %} {% endblock %} @@ -84,13 +90,19 @@

Judicial Financial Disclosures Database (Draft)

-

We currently have {{ disclosure_count|intcomma }} financial disclosure forms in our collection.

+

We currently have {{ disclosure_count|intcomma }} financial disclosure forms for {{ people_count|intcomma }} judge{{ people_count|pluralize }}.

-
-
+ {% cache 6000 financial_disclosures_home_listing %} + {% for row in people|rows:4 %} +
+ {% for person in row %} +

{{ person.name_full }} ({{ person.financial_disclosures.count }} form{{ person.financial_disclosures.count|pluralize }}) +

+ {% endfor %} +
+ {% endfor %} + {% endcache %}
- - {% endblock %} diff --git a/cl/people_db/urls.py b/cl/people_db/urls.py index f2615cde3e..def915bce0 100644 --- a/cl/people_db/urls.py +++ b/cl/people_db/urls.py @@ -1,5 +1,6 @@ from cl.people_db.sitemap import people_sitemap_maker -from cl.people_db.views import view_person, financial_disclosures_home +from cl.people_db.views import financial_disclosures_for_somebody, \ + financial_disclosures_fileserver, view_person, financial_disclosures_home from django.conf.urls import url @@ -9,11 +10,28 @@ view_person, name='view_person' ), + url( + r'^person/(?P\d*)/(?P[^/]*)/financial-disclosures/$', + financial_disclosures_for_somebody, + name='financial_disclosures_for_somebody', + ), + # Serve the PDFs, TIFFS, and thumbnails + url( + r'^person/' + r'(?P\d*)/' + r'(?P[^/]*)/' + r'(?Pfinancial-disclosures/' + r'(?:thumbnails/)?' + r'.+\.(?:pdf|tiff|png))$', + financial_disclosures_fileserver, + name='financial_disclosures_fileserver', + ), url( r'^financial-disclosures/$', financial_disclosures_home, name='financial_disclosures_home' ), + # Sitemap url( r'^sitemap-people\.xml', diff --git a/cl/people_db/views.py b/cl/people_db/views.py index e49aee2fb4..f29c6e7fa9 100644 --- a/cl/people_db/views.py +++ b/cl/people_db/views.py @@ -1,10 +1,25 @@ +import os from django.conf import settings from django.core.urlresolvers import reverse -from django.http import HttpResponseRedirect +from django.http import HttpResponseRedirect, HttpResponse from django.shortcuts import get_object_or_404, render +from cl.lib import magic +from cl.lib.bot_detector import is_bot from cl.lib.sunburnt import SolrInterface -from cl.people_db.models import Person +from cl.people_db.models import Person, FinancialDisclosure +from cl.stats.utils import tally_stat + + +def make_title_str(person): + """Make a nice title for somebody.""" + locations = ', '.join( + {p.court.short_name for p in person.positions.all() if p.court} + ) + title = person.name_full + if locations: + title = "%s (%s)" % (title, locations) + return title def view_person(request, pk, slug): @@ -17,12 +32,7 @@ def view_person(request, pk, slug): ])) # Make the title string. - locations = ', '.join( - {p.court.short_name for p in person.positions.all() if p.court} - ) - title = person.name_full - if locations: - title = "Judge %s (%s)" % (title, locations) + title = "Judge %s" % make_title_str(person) # Regroup the positions by whether they're judgeships or other. This allows # us to use the {% ifchanged %} template tags to have two groups in the @@ -86,7 +96,46 @@ def financial_disclosures_home(request): - A list of all the people we have reports for - A simple JS filter to find specific judges """ + people_with_disclosures = Person.objects.filter( + financial_disclosures__isnull=False, + ).distinct() + disclosure_count = FinancialDisclosure.objects.all().count() + people_count = people_with_disclosures.count() return render(request, 'financial_disclosures_home.html', { - 'disclosure_count': 900000, + 'people': people_with_disclosures, + 'disclosure_count': disclosure_count, + 'people_count': people_count, 'private': False, }) + + +def financial_disclosures_for_somebody(request, pk, slug): + """Show the financial disclosures for a particular person""" + person = get_object_or_404(Person, pk=pk) + title = make_title_str(person) + return render(request, 'financial_disclosures_for_somebody.html', { + 'person': person, + 'title': title, + 'private': False, + }) + + +def financial_disclosures_fileserver(request, pk, slug, filepath): + """Serve up the financial disclosure files.""" + response = HttpResponse() + file_loc = os.path.join(settings.MEDIA_ROOT, filepath.encode('utf-8')) + if settings.DEVELOPMENT: + # X-Sendfile will only confuse you in a dev env. + response.content = open(file_loc, 'r').read() + else: + response['X-Sendfile'] = file_loc + filename = filepath.split('/')[-1] + response['Content-Disposition'] = 'inline; filename="%s"' % \ + filename.encode('utf-8') + response['Content-Type'] = magic.from_file(file_loc, mime=True) + if not is_bot(request): + tally_stat('financial_reports.static_file.served') + return response + + +