From 502cdcc5bc262f5554f45b53eca14219f4e50703 Mon Sep 17 00:00:00 2001 From: Forest Gregg Date: Wed, 20 Nov 2019 10:30:31 -0600 Subject: [PATCH 1/8] migrations --- lametro/migrations/0001_initial.py | 62 +++++++++++++++++-- lametro/migrations/0002_lametropost.py | 25 -------- lametro/migrations/0003_lametroperson.py | 25 -------- lametro/migrations/0004_lametroevent.py | 25 -------- .../migrations/0005_lametroorganization.py | 25 -------- lametro/migrations/0006_lametroeventmedia.py | 25 -------- lametro/migrations/0007_lametrosubject.py | 25 -------- lametro/migrations/0008_auto_20190503_1330.py | 25 -------- lametro/migrations/0009_auto_20190516_0842.py | 34 ---------- 9 files changed, 57 insertions(+), 214 deletions(-) delete mode 100644 lametro/migrations/0002_lametropost.py delete mode 100644 lametro/migrations/0003_lametroperson.py delete mode 100644 lametro/migrations/0004_lametroevent.py delete mode 100644 lametro/migrations/0005_lametroorganization.py delete mode 100644 lametro/migrations/0006_lametroeventmedia.py delete mode 100644 lametro/migrations/0007_lametrosubject.py delete mode 100644 lametro/migrations/0008_auto_20190503_1330.py delete mode 100644 lametro/migrations/0009_auto_20190516_0842.py diff --git a/lametro/migrations/0001_initial.py b/lametro/migrations/0001_initial.py index 4de5a7c73..78dc23620 100644 --- a/lametro/migrations/0001_initial.py +++ b/lametro/migrations/0001_initial.py @@ -1,8 +1,7 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.7 on 2016-10-13 20:46 -from __future__ import unicode_literals +# Generated by Django 2.1.13 on 2019-10-11 17:36 -from django.db import migrations +from django.db import migrations, models +import lametro.models class Migration(migrations.Migration): @@ -10,17 +9,70 @@ class Migration(migrations.Migration): initial = True dependencies = [ - ('councilmatic_core', '0010_auto_20160120_1248'), + ('councilmatic_core', '0048_post_shape'), ] operations = [ + migrations.CreateModel( + name='SubjectGuid', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('guid', models.CharField(max_length=256)), + ('name', models.CharField(max_length=256, unique=True)), + ], + ), migrations.CreateModel( name='LAMetroBill', fields=[ ], options={ 'proxy': True, + 'indexes': [], }, bases=('councilmatic_core.bill',), ), + migrations.CreateModel( + name='LAMetroEvent', + fields=[ + ], + options={ + 'proxy': True, + 'indexes': [], + }, + bases=('councilmatic_core.event', lametro.models.LiveMediaMixin), + ), + migrations.CreateModel( + name='LAMetroOrganization', + fields=[ + ], + options={ + 'proxy': True, + 'indexes': [], + }, + bases=('councilmatic_core.organization',), + ), + migrations.CreateModel( + name='LAMetroPerson', + fields=[ + ], + options={ + 'proxy': True, + 'indexes': [], + }, + bases=('councilmatic_core.person',), + ), + migrations.CreateModel( + name='LAMetroPost', + fields=[ + ], + options={ + 'proxy': True, + 'indexes': [], + }, + bases=('councilmatic_core.post',), + ), + migrations.AlterUniqueTogether( + name='subjectguid', + unique_together={('guid', 'name')}, + ), ] diff --git a/lametro/migrations/0002_lametropost.py b/lametro/migrations/0002_lametropost.py deleted file mode 100644 index 0f6693084..000000000 --- a/lametro/migrations/0002_lametropost.py +++ /dev/null @@ -1,25 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.7 on 2016-10-14 15:53 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('councilmatic_core', '0011_auto_20161013_1541'), - ('lametro', '0001_initial'), - ] - - operations = [ - migrations.CreateModel( - name='LAMetroPost', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('councilmatic_core.post',), - ), - ] diff --git a/lametro/migrations/0003_lametroperson.py b/lametro/migrations/0003_lametroperson.py deleted file mode 100644 index c4f7aed18..000000000 --- a/lametro/migrations/0003_lametroperson.py +++ /dev/null @@ -1,25 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.10 on 2016-10-25 17:05 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('councilmatic_core', '0027_merge'), - ('lametro', '0002_lametropost'), - ] - - operations = [ - migrations.CreateModel( - name='LAMetroPerson', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('councilmatic_core.person',), - ), - ] diff --git a/lametro/migrations/0004_lametroevent.py b/lametro/migrations/0004_lametroevent.py deleted file mode 100644 index 36db4c662..000000000 --- a/lametro/migrations/0004_lametroevent.py +++ /dev/null @@ -1,25 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.11 on 2016-11-28 15:20 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('councilmatic_core', '0027_merge'), - ('lametro', '0003_lametroperson'), - ] - - operations = [ - migrations.CreateModel( - name='LAMetroEvent', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('councilmatic_core.event',), - ), - ] diff --git a/lametro/migrations/0005_lametroorganization.py b/lametro/migrations/0005_lametroorganization.py deleted file mode 100644 index 387c89c1d..000000000 --- a/lametro/migrations/0005_lametroorganization.py +++ /dev/null @@ -1,25 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.13 on 2018-04-12 14:42 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('councilmatic_core', '0036_auto_20180302_1247'), - ('lametro', '0004_lametroevent'), - ] - - operations = [ - migrations.CreateModel( - name='LAMetroOrganization', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('councilmatic_core.organization',), - ), - ] diff --git a/lametro/migrations/0006_lametroeventmedia.py b/lametro/migrations/0006_lametroeventmedia.py deleted file mode 100644 index f7c930d96..000000000 --- a/lametro/migrations/0006_lametroeventmedia.py +++ /dev/null @@ -1,25 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.13 on 2018-05-07 20:44 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('councilmatic_core', '0041_event_extras'), - ('lametro', '0005_lametroorganization'), - ] - - operations = [ - migrations.CreateModel( - name='LAMetroEventMedia', - fields=[ - ], - options={ - 'proxy': True, - }, - bases=('councilmatic_core.eventmedia',), - ), - ] diff --git a/lametro/migrations/0007_lametrosubject.py b/lametro/migrations/0007_lametrosubject.py deleted file mode 100644 index 021e9c6f1..000000000 --- a/lametro/migrations/0007_lametrosubject.py +++ /dev/null @@ -1,25 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.13 on 2019-05-02 19:33 -from __future__ import unicode_literals - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('councilmatic_core', '0044_bill_restrict_view'), - ('lametro', '0006_lametroeventmedia'), - ] - - operations = [ - migrations.CreateModel( - name='LAMetroSubject', - fields=[ - ('subject_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='councilmatic_core.Subject')), - ('guid', models.CharField(blank=True, max_length=256, null=True)), - ], - bases=('councilmatic_core.subject',), - ), - ] diff --git a/lametro/migrations/0008_auto_20190503_1330.py b/lametro/migrations/0008_auto_20190503_1330.py deleted file mode 100644 index 46a029530..000000000 --- a/lametro/migrations/0008_auto_20190503_1330.py +++ /dev/null @@ -1,25 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.13 on 2019-05-03 20:30 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('lametro', '0007_lametrosubject'), - ] - - operations = [ - migrations.AddField( - model_name='lametrosubject', - name='name', - field=models.CharField(blank=True, max_length=256, null=True, unique=True), - ), - migrations.AlterField( - model_name='lametrosubject', - name='guid', - field=models.CharField(blank=True, max_length=256, null=True, unique=True), - ), - ] diff --git a/lametro/migrations/0009_auto_20190516_0842.py b/lametro/migrations/0009_auto_20190516_0842.py deleted file mode 100644 index 0bf353ef1..000000000 --- a/lametro/migrations/0009_auto_20190516_0842.py +++ /dev/null @@ -1,34 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.13 on 2019-05-16 15:42 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('lametro', '0008_auto_20190503_1330'), - ] - - operations = [ - migrations.CreateModel( - name='SubjectGuid', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('guid', models.CharField(max_length=256)), - ('name', models.CharField(max_length=256, unique=True)), - ], - ), - migrations.RemoveField( - model_name='lametrosubject', - name='subject_ptr', - ), - migrations.DeleteModel( - name='LAMetroSubject', - ), - migrations.AlterUniqueTogether( - name='subjectguid', - unique_together=set([('guid', 'name')]), - ), - ] From 45256e00c0905899adfd54230bf5bc88c44eda58 Mon Sep 17 00:00:00 2001 From: Forest Gregg Date: Wed, 20 Nov 2019 10:45:02 -0600 Subject: [PATCH 2/8] models, views, templates, oh my --- councilmatic/settings.py | 18 +- councilmatic/settings_jurisdiction.py | 15 +- councilmatic/urls.py | 9 +- lametro/feeds.py | 2 +- lametro/models.py | 369 ++++++++---- lametro/search_indexes.py | 10 +- lametro/templates/lametro/board_members.html | 24 +- lametro/templates/lametro/committee.html | 53 +- lametro/templates/lametro/committees.html | 16 +- lametro/templates/lametro/event.html | 22 +- lametro/templates/lametro/legislation.html | 37 +- lametro/templates/lametro/person.html | 18 +- .../partials/council_member_table.html | 25 +- lametro/templates/partials/map.html | 3 +- .../templates/partials/past_event_day.html | 6 +- lametro/templates/partials/related_bills.html | 10 +- lametro/templatetags/lametro_extras.py | 53 +- lametro/utils.py | 2 +- lametro/views.py | 545 ++++++------------ requirements.txt | 7 +- 20 files changed, 572 insertions(+), 672 deletions(-) diff --git a/councilmatic/settings.py b/councilmatic/settings.py index c440b8f1f..b2ecc5cd5 100644 --- a/councilmatic/settings.py +++ b/councilmatic/settings.py @@ -43,8 +43,10 @@ 'django.contrib.messages', 'django.contrib.staticfiles', 'haystack', - 'lametro', + 'opencivicdata.core', + 'opencivicdata.legislative', 'councilmatic_core', + 'lametro', 'adv_cache_tag', 'debug_toolbar', ) @@ -54,18 +56,15 @@ except NameError: pass -MIDDLEWARE_CLASSES = ( +MIDDLEWARE = ( + 'debug_toolbar.middleware.DebugToolbarMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', - # 'django.middleware.cache.UpdateCacheMiddleware', 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', 'django.middleware.security.SecurityMiddleware', - 'debug_toolbar.middleware.DebugToolbarMiddleware', - # 'django.middleware.cache.FetchFromCacheMiddleware', ) ROOT_URLCONF = 'councilmatic.urls' @@ -73,7 +72,7 @@ TEMPLATES = [ { 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [], + 'DIRS': [os.path.join(BASE_DIR, 'lametro/templates')], 'APP_DIRS': True, 'OPTIONS': { 'context_processors': [ @@ -87,6 +86,7 @@ }, ] + WSGI_APPLICATION = 'councilmatic.wsgi.application' @@ -119,3 +119,7 @@ HAYSTACK_SIGNAL_PROCESSOR = 'haystack.signals.RealtimeSignalProcessor' ADV_CACHE_INCLUDE_PK = True + +import socket +hostname, _, ips = socket.gethostbyname_ex(socket.gethostname()) +INTERNAL_IPS = [ip[:-1] + '1' for ip in ips] + ['127.0.0.1', '10.0.2.2'] diff --git a/councilmatic/settings_jurisdiction.py b/councilmatic/settings_jurisdiction.py index a8a6e6177..a82e9fe3b 100644 --- a/councilmatic/settings_jurisdiction.py +++ b/councilmatic/settings_jurisdiction.py @@ -4,7 +4,7 @@ # These settings are required # ############################### -OCD_CITY_COUNCIL_ID = 'ocd-organization/42e23f04-de78-436a-bec5-ab240c1b977c' +OCD_CITY_COUNCIL_NAME = 'Board of Directors' CITY_COUNCIL_NAME = 'Metro' OCD_JURISDICTION_IDS = ['ocd-jurisdiction/country:us/state:ca/county:los_angeles/transit_authority'] LEGISLATIVE_SESSIONS = ['2014', '2015', '2016', '2017', '2018', '2019'] # the last one in this list should be the current legislative session @@ -47,12 +47,13 @@ BOUNDARY_SET = ['la-metro-supervisory-districts', 'la-metro-committee-districts', 'city-of-la'] -MAP_CONFIG = { - 'center': [34.0522, -118.2437], - 'zoom': 10, - 'color': "#54afe8", - 'highlight_color': '#eb6864' -} +MAP_CONFIG = False +# MAP_CONFIG = { +# 'center': [34.0522, -118.2437], +# 'zoom': 10, +# 'color': "#54afe8", +# 'highlight_color': '#eb6864' +# } # FOOTER_CREDITS = [ # { diff --git a/councilmatic/urls.py b/councilmatic/urls.py index 8caae92af..57a026251 100644 --- a/councilmatic/urls.py +++ b/councilmatic/urls.py @@ -1,6 +1,6 @@ from django.conf.urls import include, url from django.contrib import admin -from django.contrib.staticfiles import views as staticviews +from django.views.static import serve from django.views.generic.base import RedirectView from django.conf import settings from django.views.decorators.cache import never_cache @@ -39,7 +39,7 @@ urlpatterns = [ url(r'', include(patterns)), - url(r'^admin/', include(admin.site.urls)), + url(r'^admin/', admin.site.urls), url(r'^metro-login/$', metro_login, name='metro_login'), url(r'^metro-logout/$', metro_logout, name='metro_logout'), url(r'^delete-submission/(?P[^/]+)/$', delete_submission, name='delete_submission'), @@ -49,6 +49,7 @@ if settings.DEBUG: import debug_toolbar urlpatterns += [ - url(r'^static/(?P.*)/$', staticviews.serve), - url(r'^__debug__/', include(debug_toolbar.urls)), + url(r'^static/(?P.*)/$', serve), + url(r'^images/(?P.*)/$', serve, {'document_root': settings.STATIC_ROOT + '/images/'}), + url(r'^__debug__/', include(debug_toolbar.urls)), ] diff --git a/lametro/feeds.py b/lametro/feeds.py index aaff76c3f..f3aed500c 100644 --- a/lametro/feeds.py +++ b/lametro/feeds.py @@ -1,4 +1,4 @@ -from django.core.urlresolvers import reverse, reverse_lazy +from django.urls import reverse, reverse_lazy from django.utils.feedgenerator import Rss201rev2Feed from django.conf import settings diff --git a/lametro/models.py b/lametro/models.py index 540ef4985..929d57648 100644 --- a/lametro/models.py +++ b/lametro/models.py @@ -9,15 +9,30 @@ from django.db.models.expressions import RawSQL from django.core.exceptions import ObjectDoesNotExist from django.utils import timezone +from django.utils.functional import cached_property from django.contrib.auth.models import User -from django.db.models import Max, Min, Prefetch, Case, When, Value, Q +from django.db.models import Max, Min, Prefetch, Case, When, Value, Q, F +from django.db.models.functions import Now, Cast -from councilmatic_core.models import Bill, Event, Post, Person, Organization, \ - Action, EventMedia, EventDocument, Subject +from councilmatic_core.models import Bill, Event, Post, Person, Organization, EventManager, Membership + +from opencivicdata.legislative.models import EventMedia, EventDocument, EventDocumentLink, EventAgendaItem, EventRelatedEntity, RelatedBill + +from proxy_overrides.related import ProxyForeignKey app_timezone = pytz.timezone(settings.TIME_ZONE) +class SourcesMixin(object): + + @property + def web_source(self): + return self.sources.get(note='web') + + @property + def api_source(self): + return self.sources.get(note='api') + class LAMetroBillManager(models.Manager): def get_queryset(self): @@ -46,16 +61,17 @@ def get_queryset(self): when getting bill querysets. Otherwise restricted view bills may slip through the crevices of Councilmatic display logic. ''' - filtered_qs = super().get_queryset().exclude(restrict_view=True)\ - .filter(Q(related_agenda_items__event__status='passed') | \ - Q(related_agenda_items__event__status='cancelled') | \ - Q(bill_type='Board Box'))\ - .distinct() + filtered_qs = super().get_queryset() + # .exclude(restrict_view=True)\ + # .filter(Q(related_agenda_items__event__status='passed') | \ + # Q(related_agenda_items__event__status='cancelled') | \ + # Q(bill_type='Board Box'))\ + # .distinct() return filtered_qs -class LAMetroBill(Bill): +class LAMetroBill(Bill, SourcesMixin): objects = LAMetroBillManager() class Meta: @@ -71,7 +87,7 @@ def friendly_name(self): @property def inferred_status(self): # Get most recent action. - action = self.actions.all().order_by('-order').first() + action = self.actions.last() # Get description of that action. if action: @@ -79,8 +95,6 @@ def inferred_status(self): else: description = '' - bill_type = self.bill_type - return self._status(description) def _status(self, description): @@ -120,23 +134,29 @@ def get_last_action_date(self): events that have already occurred, so the last action date is not in the future. ''' - actions = Action.objects.filter(_bill_id=self.ocd_id) - last_action_date = '' + try: + return self.actions.last().date_dt + except AttributeError: + return None - if actions: - last_action_date = actions.reverse()[0].date - else: - events = Event.objects.filter(agenda_items__bill_id=self.ocd_id, - start_time__lt=timezone.now()) + @property + def topics(self): + return sorted(self.subject) - if events: - last_action_date = events.latest('start_time').start_time - return last_action_date +class RelatedBill(RelatedBill): - @property - def topics(self): - return [s.subject for s in self.subjects.all()] + class Meta: + proxy = True + + bill = ProxyForeignKey(LAMetroBill, + related_name='related_bills', + on_delete=models.CASCADE) + + related_bill = ProxyForeignKey(LAMetroBill, + related_name='related_bills_reverse', + null=True, + on_delete=models.SET_NULL) class LAMetroPost(Post): @@ -145,38 +165,31 @@ class Meta: proxy = True @property - def current_members(self): - today = timezone.now().date() - return self.memberships.filter(end_date__gte=today) + def acting_label(self): + if self.extras.get('acting'): + return 'Acting ' + self.label + else: + return self.label -class LAMetroPerson(Person): +class LAMetroPerson(Person, SourcesMixin): class Meta: proxy = True @property def latest_council_membership(self): - if hasattr(settings, 'OCD_CITY_COUNCIL_ID'): - filter_kwarg = {'_organization__ocd_id': settings.OCD_CITY_COUNCIL_ID} - else: - filter_kwarg = {'_organization__name': settings.OCD_CITY_COUNCIL_NAME} + filter_kwarg = {'organization__name': settings.OCD_CITY_COUNCIL_NAME,} city_council_memberships = self.memberships.filter(**filter_kwarg) - if city_council_memberships.count(): - return city_council_memberships.order_by('-end_date').first() - return None - @property - def current_council_seat(self): - ''' - current_council_seat operated on assumption that board members - represent a jurisdiction; that's not the case w la metro. just - need to know whether member is current or not... - ''' - m = self.latest_council_membership - if m: - end_date = m.end_date - today = timezone.now().date() - return True if today < end_date else False + # We want to exclude memberships like 1st chair and just + # get the memberships confer membership to the org + # + # see https://github.com/opencivicdata/python-opencivicdata/issues/129 + primary_memberships = city_council_memberships.filter(Q(role='Board Member') | + Q(role='Nonvoting Board Member')) + + if primary_memberships.count(): + return primary_memberships.order_by('-end_date').first() return None @property @@ -191,6 +204,22 @@ def latest_council_seat(self): pass @property + def board_office(self): + + try: + office_membership = self.memberships\ + .filter(organization__name=settings.OCD_CITY_COUNCIL_NAME)\ + .filter(Q(role='Chair') | + Q(role='1st Chair') | + Q(role='2nd Chair') | + Q(role='Vice Chair'))\ + .get(end_date_dt__gt=Now()) + except Membership.DoesNotExist: + office_membership = None + + return office_membership + + @cached_property def committee_sponsorships(self): ''' This property returns a list of ten bills, which have recent actions @@ -198,32 +227,17 @@ def committee_sponsorships(self): Organizations do not include the Board of Directors. ''' - query = ''' - SELECT bill_id - FROM councilmatic_core_bill as bill - JOIN councilmatic_core_action as action - ON bill.ocd_id = action.bill_id - JOIN councilmatic_core_organization as org - ON org.ocd_id = action.organization_id - JOIN councilmatic_core_membership as membership - ON org.ocd_id = membership.organization_id - WHERE membership.person_id='{person}' - AND action.date >= membership.start_date - AND org.classification = 'committee' - ORDER BY action.date DESC - LIMIT 10 - '''.format(person=self.ocd_id) - - with connection.cursor() as cursor: - cursor.execute(query) - bill_ids = [bill_tup[0] for bill_tup in cursor.fetchall()] - - bills = LAMetroBill.objects.filter(ocd_id__in=bill_ids) - - return bills - - -class LAMetroEventManager(models.Manager): + qs = LAMetroBill.objects\ + .defer('extras')\ + .filter(actions__organization__classification='committee')\ + .filter(actions__organization__memberships__in=self.current_memberships)\ + .order_by('-actions__date')\ + .distinct()[:10] + + return qs + + +class LAMetroEventManager(EventManager): def get_queryset(self): ''' If SHOW_TEST_EVENTS is False, omit them from the initial queryset. @@ -232,10 +246,10 @@ def get_queryset(self): when getting event querysets. If a test event slips through, it is likely because we used the default Event to get the queryset. ''' - if not settings.SHOW_TEST_EVENTS: - return super().get_queryset().exclude(location_name='TEST') + if settings.SHOW_TEST_EVENTS: + return super().get_queryset() - return super().get_queryset() + return super().get_queryset().exclude(location__name='TEST') def with_media(self): ''' @@ -248,14 +262,15 @@ def with_media(self): come after links to English audio. 'mediaqueryset' facilitates the ordering of prefetched 'media_urls'. ''' - mediaqueryset = LAMetroEventMedia.objects.annotate( + mediaqueryset = EventMedia.objects.annotate( olabel=Case( When(note__endswith='(SAP)', then=Value(0)), output_field=models.CharField(), ) ).order_by('-olabel') - return self.prefetch_related(Prefetch('media_urls', queryset=mediaqueryset)) + return self.prefetch_related(Prefetch('media', queryset=mediaqueryset))\ + .prefetch_related('media__links') class LiveMediaMixin(object): @@ -314,13 +329,12 @@ def spanish_live_media_url(self): return None -class LAMetroEvent(Event, LiveMediaMixin): +class LAMetroEvent(Event, LiveMediaMixin, SourcesMixin): objects = LAMetroEventManager() class Meta: proxy = True - @classmethod def upcoming_board_meeting(cls): return cls.objects.filter(start_time__gt=datetime.now(app_timezone), name__icontains='Board Meeting')\ @@ -459,50 +473,25 @@ def upcoming_committee_meetings(cls): return meetings - @property - def board_event_minutes(self): - ''' - This method returns the link to an Event's minutes. +class EventAgendaItem(EventAgendaItem): - A small number of Events do not have minutes in - a discoverable, corresponding EventDocument. - For these, we can query board reports - for indicative text, i.e., "minutes of the regular..." - ''' - if 'regular board meeting' in self.name.lower(): - try: - doc = self.documents.get(note__icontains='RBM Minutes') - except EventDocument.DoesNotExist: - try: - date = self.start_time.date().strftime('%B %d, %Y') - content = 'minutes of the regular board meeting held ' + date - board_report = LAMetroBill.objects.get(ocr_full_text__icontains=content, bill_type='Minutes') - except LAMetroBill.DoesNotExist: - return None - else: - return '/board-report/' + board_report.slug - else: - return doc.url + class Meta: + proxy = True + event = ProxyForeignKey(LAMetroEvent, related_name='agenda', on_delete=models.CASCADE) -class LAMetroEventMedia(EventMedia): +class EventRelatedEntity(EventRelatedEntity): class Meta: proxy = True - @property - def label(self): - ''' - EventMedia imported prior to django-councilmatic 0.10.0 may not have - an associated note. - ''' - if self.note and self.note.endswith('(SAP)'): - return 'Ver en Español' - else: - return 'Watch in English' + agenda_item = ProxyForeignKey(EventAgendaItem, + related_name='related_entities', + on_delete=models.CASCADE) + bill = ProxyForeignKey(LAMetroBill, null=True, on_delete=models.SET_NULL) -class LAMetroOrganization(Organization): +class LAMetroOrganization(Organization, SourcesMixin): ''' Overrides use the LAMetroEvent object, rather than the default Event object, so test events are hidden appropriately. @@ -512,8 +501,8 @@ class Meta: @property def recent_events(self): - events = LAMetroEvent.objects.filter(participants__entity_type='organization', participants__entity_name=self.name) - events = events.order_by('-start_time').all() + events = LAMetroEvent.objects.filter(participants__organization=self) + events = events.order_by('-start_time') return events @property @@ -528,6 +517,36 @@ def upcoming_events(self): .all() return events +class Membership(Membership): + class Meta: + proxy = True + + organization = ProxyForeignKey( + LAMetroOrganization, + related_name='memberships', + # memberships will go away if the org does + on_delete=models.CASCADE, + help_text="A link to the Organization in which the Person is a member." + ) + + person = ProxyForeignKey( + LAMetroPerson, + related_name='memberships', + null=True, + # Membership will just unlink if the person goes away + on_delete=models.SET_NULL, + help_text="A link to the Person that is a member of the Organization." + ) + + post = ProxyForeignKey( + LAMetroPost, + related_name='memberships', + null=True, + # Membership will just unlink if the post goes away + on_delete=models.SET_NULL, + help_text="The Post held by the member in the Organization." + ) + class SubjectGuid(models.Model): class Meta: @@ -535,3 +554,117 @@ class Meta: guid = models.CharField(max_length=256) name = models.CharField(max_length=256, unique=True) + + +class BillPacket(models.Model): + + bill = models.OneToOneField(LAMetroBill, + related_name='packet', + on_delete=models.CASCADE) + updated_at = models.DateTimeField(auto_now=True) + url = models.URLField() + ready = models.BooleanField(default=False) + + def save(self, *args, **kwargs): + + self._merge_docs() + + self.url = settings.MERGER_BASE_URL + '/document/' + self.bill.slug + + response = requests.head(self.url) + + super().save(*args, **kwargs) + + def is_ready(self): + + if not self.ready: + response = requests.head(self.url) + if response.status_code == 200: + self.ready = True + super().save() + + return self.ready + + @property + def related_files(self): + board_report = self.bill.versions.get() + + attachments = self.bill.documents\ + .annotate( + index=Case( + When(note__istartswith = '0', then=Value('z')), + default=F('note'), + output_field=models.CharField()))\ + .order_by('index') + + doc_links = [board_report.links.get().url] + + # sometime there are more than url for the same document name + # https://metro.legistar.com/LegislationDetail.aspx?ID=3104422&GUID=C30D3376-7265-477B-AFFA-815270400538%3e%5d%3e + # I'm not sure if this a data problem or not, so we'll just + # add all the doc links + doc_links += [link.url + for doc in attachments + for link in doc.links.all()] + + return doc_links + + def _merge_docs(self): + + merge_url = settings.MERGER_BASE_URL + '/merge_pdfs/' + self.bill.slug + + requests.post(merge_url, json=self.related_files) + + +class EventPacket(models.Model): + + event = models.OneToOneField(LAMetroEvent, + related_name='packet', + on_delete=models.CASCADE) + updated_at = models.DateTimeField(auto_now=True) + url = models.URLField() + ready = models.BooleanField(default=False) + + def save(self, *args, **kwargs): + + self._merge_docs() + + self.url = settings.MERGER_BASE_URL + '/document/' + self.event.slug + + super().save(*args, **kwargs) + + def is_ready(self): + + if not self.ready: + response = requests.head(self.url) + if response.status_code == 200: + self.ready = True + super().save() + + return self.ready + + @property + def related_files(self): + + agenda_doc = self.event.documents.get(note='Agenda') + + related = [agenda_doc.links.get().url] + + agenda_items = self.event.agenda\ + .filter(related_entities__bill__documents__isnull=False)\ + .annotate(int_order=Cast('order', models.IntegerField()))\ + .order_by('int_order')\ + .distinct() + + for item in agenda_items: + for entity in item.related_entities.filter(bill__isnull=False): + bill_packet = BillPacket(bill=entity.bill) + related.extend(bill_packet.related_files) + + return related + + def _merge_docs(self): + + merge_url = settings.MERGER_BASE_URL + '/merge_pdfs/' + self.event.slug + + requests.post(merge_url, json=self.related_files) diff --git a/lametro/search_indexes.py b/lametro/search_indexes.py index 551d0bd9e..c83268ec9 100644 --- a/lametro/search_indexes.py +++ b/lametro/search_indexes.py @@ -2,7 +2,6 @@ from haystack import indexes from councilmatic_core.haystack_indexes import BillIndex -from councilmatic_core.models import Action, EventAgendaItem from lametro.models import LAMetroBill from lametro.utils import format_full_text, parse_subject @@ -19,8 +18,7 @@ def prepare_controlling_body(self, obj): return None def prepare_sponsorships(self, obj): - actions = Action.objects.filter(_bill_id=obj.ocd_id) - return [action.organization for action in actions] + return [action.organization for action in obj.actions.all()] def prepare_last_action_date(self, obj): # Solr seems to be fussy about the time format, and we do not need the time, just the date stamp. @@ -30,7 +28,7 @@ def prepare_last_action_date(self, obj): return last_action_date.date() def prepare_sort_name(self, obj): - full_text = obj.ocr_full_text + full_text = obj.extras.get('plain_text') results = '' if full_text: @@ -41,13 +39,13 @@ def prepare_sort_name(self, obj): return obj.bill_type def prepare_topics(self, obj): - return [s.subject for s in obj.subjects.all()] + return obj.topics def prepare_attachment_text(self, obj): return ' '.join(d.full_text for d in obj.documents.all() if d.full_text) def prepare_legislative_session(self, obj): - start_year = obj._legislative_session.identifier + start_year = obj.legislative_session.identifier end_year = int(start_year) + 1 session = '7/1/{start_year} to 6/30/{end_year}'.format(start_year=start_year, diff --git a/lametro/templates/lametro/board_members.html b/lametro/templates/lametro/board_members.html index b17dea2bc..c84a9e33c 100644 --- a/lametro/templates/lametro/board_members.html +++ b/lametro/templates/lametro/board_members.html @@ -82,7 +82,7 @@

-
+
{% include 'partials/council_member_table.html' %}
@@ -108,11 +108,11 @@

Recent Activity

- {% for action in recent_activity %} + {% for action in recent_activity|slice:":100" %}

- {{action.date|date:'n/d/Y'}} + {{action.date_dt|date:'n/d/Y'}}

{{action.description | remove_action_subj}} @@ -123,7 +123,7 @@

Recent Activity

{{action.bill.friendly_name}}

- {{action.bill.description | short_blurb}} + {{action.bill.title | short_blurb}}

@@ -166,19 +166,12 @@

Board of Directors Meetings

"info": false, "bLengthChange": false, "paging": false, - "aaSorting": [ [3,'asc'] ], + "aaSorting": [ [1,'asc'] ], "aoColumns": [ { "bSortable": false }, null, { "sType": "num-html" } ], - "aoColumnDefs": [ - { - "targets": [ 3 ], - "visible": false, - "searchable": false - }, - ] }); $( document ).ready(function() { @@ -232,7 +225,7 @@

Board of Directors Meetings

if (active_id.includes('city') || controlView) { los_angeles_city.eachLayer(function (layer) { - if (layer.feature.properties.select_id === select_id || select_id.includes("mayor-of-the-city-of-los-angeles")){ + if (layer.feature.properties.select_id === select_id){ layer.fire('tableout'); } }); @@ -261,8 +254,7 @@

Board of Directors Meetings

if (active_id.includes('city') || controlView) { los_angeles_city.eachLayer(function (layer) { - var target_member = layer.feature.properties.council_member.toLowerCase().replace(' ', '-') - if (member_name == target_member) { + if (layer.feature.properties.select_id === select_id){ layer.fire('tableover'); } }); @@ -316,4 +308,4 @@

Board of Directors Meetings

-{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/lametro/templates/lametro/committee.html b/lametro/templates/lametro/committee.html index c46d745ec..20676f838 100644 --- a/lametro/templates/lametro/committee.html +++ b/lametro/templates/lametro/committee.html @@ -61,61 +61,24 @@

Committee Members

- {% if 'Ad-Hoc' in committee.name %} - {% for member in ad_hoc_list %} + {% for membership in non_ceos %} - +
- {{member.name}} + {{membership.person.name}}
- {% if member.label %} - {{ member.name }} - {% else %} - {{ member.name }} - {% endif %} + {{ membership.person.name }} - {% if member.label %} - {{ extras | clean_membership_extras }} {{ member.label.0 | format_string | clean_label | format_label | safe }} - {% else %} - None - {% endif %} + {{ membership.extras | clean_membership_extras }} {{ membership.person.latest_council_membership.post.label | format_label }} - {% if member.committee_role %} - {{ member.committee_role }} - {% else %} - None - {% endif %} - - - - {% endfor %} - - {% else %} - - {% for member in membership_objects %} - - -
- {{member.name}} -
- - - {{ member.name }} - - - {{ member.extras | clean_membership_extras }} {{ member.label.0 | format_string | clean_label | format_label | safe }} - - - {{ member.role }} + {{ membership.role }} {% endfor %} - - {% endif %} @@ -127,7 +90,7 @@

Chief Executive Officer

- {{ceo.name}} + {{ceo.name}}
{{ ceo.name }} @@ -189,7 +152,7 @@

Chief Executive Officer

}); // for reference purposes - console.log( "OCD ID: {{committee.ocd_id}}" ) + console.log( "OCD ID: {{committee.id}}" ) {% endblock %} diff --git a/lametro/templates/lametro/committees.html b/lametro/templates/lametro/committees.html index 14d17c7ff..7972f5b30 100644 --- a/lametro/templates/lametro/committees.html +++ b/lametro/templates/lametro/committees.html @@ -25,25 +25,23 @@

Metro Committees

- {% for committee in committees_list %} + {% for committee in committees %} - - {{ committee.0.name | committee_topic_only }} + + {{ committee.name | committee_topic_only }} - {% for person in committee %} - {% if person.role == 'Chair' %} - {{ person.member_name }} - {% endif %} + {% for membership in committee.chairs %} + {{ membership.person.name }} {% endfor %} - {{ committee | length }} + {{ committee.current_members | length }} {% endfor %} {% for committee in ad_hoc_list %} - {{ committee.0.name | committee_topic_only }}
+ {{ committee.name | committee_topic_only }}
{% for person in committee %} diff --git a/lametro/templates/lametro/event.html b/lametro/templates/lametro/event.html index a33f46fc6..cd7086ba9 100644 --- a/lametro/templates/lametro/event.html +++ b/lametro/templates/lametro/event.html @@ -17,8 +17,8 @@

{{event.name}} {% endif%} - {% for media in event.media_urls.all %} - {{ media.label }} + {% for media in event.media.all %} + {% if media.note == 'Audio (SAP)' %}Ver en Español{% else %}Watch in English{% endif %} {% endfor %}

@@ -32,16 +32,16 @@

{% else %} {{event.start_time | date:"D n/d/Y"}}
{{event.start_time | date:"g:i a"}}
- {{event.location_name}} + {{event.location.name}} {% endif %}

- {% if 'https://metro.legistar.com' in event.source_url %} + {% if event.web_source.url %} {% if not agenda_url and not uploaded_agenda_url and not uploaded_agenda_pdf %} Not seeing an agenda? Please use this link:
{% endif %} - + View on the {{CITY_VOCAB.SOURCE}} website @@ -83,9 +83,9 @@

Agenda

Download Agenda

- {% if packet_url %} + {% if event.packet.is_ready %}

- Download Agenda and Attachments + Download Agenda and Attachments

{% endif %}
@@ -100,13 +100,11 @@

Minutes

{% endif %} - {% with minutes=event.board_event_minutes %} {% if minutes %}

- View Minutes + View Minutes

{% endif %} - {% endwith %} {% endif %} @@ -205,7 +203,7 @@

"@type": "PostalAddress", "addressLocality": "{{ CITY_NAME }}" }, - "name": "{{event.location_name}}" + "name": "{{event.location.name}}" }, "name": "{{event.name}}", "startDate": "{{ event.start_time.isoformat }}", @@ -235,7 +233,7 @@

}); // for reference purposes - console.log( "OCD ID: {{event.ocd_id}}" ) + console.log( "OCD ID: {{event.id}}" ) }); diff --git a/lametro/templates/lametro/legislation.html b/lametro/templates/lametro/legislation.html index 86bfa772d..a53382c51 100644 --- a/lametro/templates/lametro/legislation.html +++ b/lametro/templates/lametro/legislation.html @@ -6,12 +6,13 @@ {% block content %} {% if legislation %} +


- {% if legislation.ocr_full_text|prepare_title %} - {{ legislation.identifier}} - {{ legislation.ocr_full_text|prepare_title }} + {% if legislation.extras.plain_text|prepare_title %} + {{ legislation.identifier}} - {{ legislation.extras.plain_text|prepare_title }} {% else %} {{ legislation.identifier}} {% endif %} @@ -28,7 +29,7 @@

- + View on the {{CITY_VOCAB.SOURCE}} website

@@ -48,9 +49,9 @@

- {% if packet_url and attachments%} + {% if legislation.packet.is_ready and attachments%}

- Download Board Report and Attachments + Download Board Report and Attachments

{% endif %} @@ -61,19 +62,19 @@

seamless="true" width="100%" height="600px" - src="/pdfviewer/?{{board_report.url|full_text_doc_url}}"> + src="/pdfviewer/?{{board_report.links.all.0.url|full_text_doc_url}}">
- {% elif legislation.ocr_full_text != "" %} + {% elif legislation.extras.plain_text != "" %}

Report text

-
- {{ legislation.ocr_full_text|prepare_title}} +
+ {{ legislation.extras.plain_text|prepare_title}}
@@ -98,7 +99,7 @@

History

{% for action in actions %} - {{action.date|date:'n/d/y'}} + {{action.date_dt|date:'n/d/y'}} {{action.description | remove_action_subj}} @@ -130,7 +131,7 @@

@@ -149,19 +150,21 @@

Rel - {% for bill in related_bills %} + {% for relation in related_bills %} + {% with bill=relation.related_bill %} - {{ bill.identifier}} + {{ bill.identifier}} - {% if bill.ocr_full_text|prepare_title %} - - {{ bill.ocr_full_text|prepare_title }} + {% if bill.extras.plain_text|prepare_title %} + - {{ bill.extras.plain_text|prepare_title }} {% endif %} {{bill.inferred_status | inferred_status_label | safe}} {{ bill.get_last_action_date|date:'n/d/Y' }} + {% endwith %} {% endfor %} @@ -232,7 +235,7 @@

Legislation not found

"alternateName": ["{{ legislation.identifier }}", "{{ legislation.identifier.split|join:'' }}"], {% if actions %}"datePublished": "{{actions.0.date|date:'Y-m-d'}}", {% endif %} "description": "{{ legislation.description }}", - "text": "{% firstof legislation.full_text legislation.ocr_full_text %}" + "text": "{% firstof legislation.full_text legislation.extras.plain_text %}" } @@ -277,7 +280,7 @@

Legislation not found

} // for reference purposes - console.log( "OCD ID: {{legislation.ocd_id}}" ) + console.log( "OCD ID: {{legislation.id}}" ) {% endblock %} diff --git a/lametro/templates/lametro/person.html b/lametro/templates/lametro/person.html index 622686249..ef8125863 100644 --- a/lametro/templates/lametro/person.html +++ b/lametro/templates/lametro/person.html @@ -20,7 +20,11 @@

{{ person.name }}
- {{ title }} + {% if person.current_council_seat %} + {{ person.current_council_seat.role }} + {% else %} + Former {{ person.latest_council_seat.role }} + {% endif %}

@@ -36,7 +40,7 @@

{% else %}
- {{person.name}} + {{person.name}}

{% if qualifying_post %} @@ -73,7 +77,7 @@

- {{person.name}} + {{person.name}}

@@ -151,7 +155,7 @@

Committees

{% for membership in memberships_list %} - {{membership.organization}} + {{membership.organization}} {{membership.role}} @@ -170,7 +174,7 @@

Committees

"@context": "http://schema.org", "@type": "Person", "email": "{{person.email}}", - "image": "{{person.headshot_url}}", + "image": "{{person.headshot.url}}", {% if person.current_council_seat %} "jobTitle": "{{ person.current_council_seat }} Representative", {% endif %} @@ -179,7 +183,7 @@

Committees

"name": "{{ CITY_COUNCIL_NAME }}" }, "name": "{{person.name}}", - "url": "{{person.website_url}}" + "url": "{{website_url}}" } @@ -236,4 +240,4 @@

Committees

{% endif %} -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/lametro/templates/partials/council_member_table.html b/lametro/templates/partials/council_member_table.html index 0b5f3cbf1..25b118361 100644 --- a/lametro/templates/partials/council_member_table.html +++ b/lametro/templates/partials/council_member_table.html @@ -15,33 +15,32 @@ - {% for membership in membership_objects %} + {% for membership in posts %} - +
- {{ membership.name }} + {{ membership.name }}
- - {{ membership.ocd_id | call_link_html | safe }} + + {{ membership.person.id | call_link_html | safe }} - - {{ membership.role | clean_role }}
+ + {% firstof membership.person.board_office.role membership.role %} +
- {{ membership.extras | clean_membership_extras }} {{ membership.label|clean_label | format_label | safe }} + {{ membership.post.acting_label | comma_to_line_break | safe }}
- {% if "District" in membership.label|clean_label and "Caltrans District" not in membership.label|clean_label %} + {% if "District" in membership.post.label and "Caltrans District" not in membership.label %} District appointee - {% elif "sector" in membership.label|clean_label %} + {% elif "sector" in membership.post.label %} Sector appointee - {% elif "Mayor" in membership.label|clean_label %} + {% elif "Mayor" in membership.post.label %} Los Angeles {% else %} {% endif %} - - {% endfor %} diff --git a/lametro/templates/partials/map.html b/lametro/templates/partials/map.html index 7634f5e0c..3a605ae57 100644 --- a/lametro/templates/partials/map.html +++ b/lametro/templates/partials/map.html @@ -162,8 +162,7 @@ }); layer.on('mouseover', function(e){ - // Hardcode this for Eric Garcetti, due to overlapping layers - infoBox.update({ council_member: "Eric Garcetti", district: "Mayor of the City of Los Angeles" }); + infoBox.update(e.target.feature.properties); e.target.setStyle({'fillOpacity': 0.8, 'color': cityColor}); }); diff --git a/lametro/templates/partials/past_event_day.html b/lametro/templates/partials/past_event_day.html index c91ace98d..7bade8a27 100644 --- a/lametro/templates/partials/past_event_day.html +++ b/lametro/templates/partials/past_event_day.html @@ -45,9 +45,9 @@
- {% for media in event.media_urls.all %} + {% for media in event.media.all %}

- {{ media.label }} + {% if media.note == 'Audio (SAP)' %}Ver en Español{% else %}Watch in English{% endif %}

{% endfor %} @@ -63,7 +63,7 @@
{% endif %}
- {% with minutes=event.board_event_minutes %} + {% with minutes=event.minutes.0.links.all.0.url %} {% if minutes %}

Minutes diff --git a/lametro/templates/partials/related_bills.html b/lametro/templates/partials/related_bills.html index 9a3aad479..709857ba8 100644 --- a/lametro/templates/partials/related_bills.html +++ b/lametro/templates/partials/related_bills.html @@ -11,14 +11,16 @@

Board Reports

{% for report in related_board_reports %} + {% with associated_bill=report.related_entities.all.0.bill %} - - + + + {% endwith %} {% endfor %}
{{ report.notes | parse_agenda_item }}{{report.identifier}} {{report.description | short_blurb}} {{ report.inferred_status | inferred_status_label | safe }}{{ report.notes.0 | parse_agenda_item }}{{associated_bill.identifier}} {{report.description | short_blurb}} {{ associated_bill.inferred_status | inferred_status_label | safe }} - View + View Download
-
\ No newline at end of file +
diff --git a/lametro/templatetags/lametro_extras.py b/lametro/templatetags/lametro_extras.py index 4e571840c..1b23123b4 100644 --- a/lametro/templatetags/lametro_extras.py +++ b/lametro/templatetags/lametro_extras.py @@ -1,6 +1,6 @@ from django import template from django.template.defaultfilters import stringfilter -from django.utils.html import strip_entities, strip_tags +from django.utils.html import strip_tags from django.utils import timezone from haystack.query import SearchQuerySet @@ -10,32 +10,31 @@ from councilmatic.settings_jurisdiction import * from councilmatic.settings import PIC_BASE_URL -from councilmatic_core.models import Person, EventDocument, Bill +from councilmatic_core.models import Person, Bill from councilmatic_core.utils import ExactHighlighter from lametro.models import LAMetroEvent from lametro.utils import format_full_text, parse_subject -register = template.Library() +from opencivicdata.legislative.models import EventDocument -@register.filter -def call_headshot_url(person_id): - person = Person.objects.get(ocd_id=person_id) - url = person.headshot_url - return url +register = template.Library() @register.filter def call_link_html(person_id): - person = Person.objects.get(ocd_id=person_id) + person = Person.objects.get(id=person_id) url = person.link_html return url @register.filter def format_label(label): - label_parts = label.split(', ') - formatted_label = '
'.join(label_parts) + first_part = label.split(', ')[0] - return formatted_label + return first_part + +@register.filter +def comma_to_line_break(text): + return '
'.join(text.split(', ')) @register.filter def format_district(label): @@ -105,19 +104,6 @@ def clean_role(role_list): return role_list[0] -@register.filter -def clean_label(label_list): - label_list = [ label for label in label_list if 'Chair' not in label ] - label = label_list[0] - - return label - -@register.filter -def format_string(label_list): - label_list = label_list.replace('{', '').replace('}', '').replace('"', '') - - return label_list.split(',') - @register.filter def compare_time(event_date): if event_date < timezone.now(): @@ -147,16 +133,15 @@ def updates_made(event_id): This filter determines if an event had been updated after its related EventDocument (i.e., agenda) was last updated. If the below equates as true, then we render a label with the text "Updated", next to the event, on the meetings page. ''' - try: - # Get the most recent updated agenda, if one of those agendas happens to be manually uploaded - document = EventDocument.objects.filter(event_id=event_id).latest('updated_at') + + event = LAMetroEvent.objects.get(ocd_id=event_id) + + try: + event.documents.get(note__icontains='Agenda') except EventDocument.DoesNotExist: return False - else: - event = LAMetroEvent.objects.get(ocd_id=event_id) - updated = document.updated_at < event.updated_at - return updated + return event.updated_at > document.date @register.filter def find_agenda_url(all_documents): @@ -170,8 +155,8 @@ def find_agenda_url(all_documents): return valid_urls[0] @register.simple_tag(takes_context=True) -def get_highlighted_attachment_text(context, ocd_id): - bill = Bill.objects.get(ocd_id=ocd_id) +def get_highlighted_attachment_text(context, id): + bill = Bill.objects.get(id=id) attachment_text = ' '.join(d.full_text for d in bill.documents.all() if d.full_text) highlight = ExactHighlighter(context['query']) diff --git a/lametro/utils.py b/lametro/utils.py index 7491afd57..2a13e98a9 100644 --- a/lametro/utils.py +++ b/lametro/utils.py @@ -8,7 +8,7 @@ from django.conf import settings from django.utils import timezone -from councilmatic_core.models import EventParticipant, Organization, Event, Action +from councilmatic_core.models import Organization, Event app_timezone = pytz.timezone(settings.TIME_ZONE) diff --git a/lametro/views.py b/lametro/views.py index 77c1b7e1d..9c65adf41 100644 --- a/lametro/views.py +++ b/lametro/views.py @@ -16,13 +16,22 @@ from haystack.inputs import Raw from haystack.query import SearchQuerySet +import pytz + from django.db import transaction, connection, connections from django.conf import settings from django.contrib.auth import authenticate, login, logout from django.contrib.auth.forms import AuthenticationForm from django.shortcuts import render -from django.db.models.functions import Lower -from django.db.models import Max, Min, Prefetch +from django.db.models.functions import Lower, Now, Cast +from django.db.models import (Max, + Min, + Prefetch, + Case, + When, + Value, + IntegerField, + Q) from django.utils import timezone from django.utils.text import slugify from django.views.generic import TemplateView @@ -38,6 +47,8 @@ CouncilmaticSearchForm from councilmatic_core.models import * +from opencivicdata.core.models import PersonLink + from lametro.models import LAMetroBill, LAMetroPost, LAMetroPerson, \ LAMetroEvent, LAMetroOrganization from lametro.forms import AgendaUrlForm, AgendaPdfForm @@ -45,12 +56,16 @@ from councilmatic.settings_jurisdiction import MEMBER_BIOS from councilmatic.settings import MERGER_BASE_URL, PIC_BASE_URL +from opencivicdata.legislative.models import EventDocument + +app_timezone = pytz.timezone(settings.TIME_ZONE) class LAMetroIndexView(IndexView): template_name = 'lametro/index.html' event_model = LAMetroEvent + @property def extra_context(self): extra = {} extra['upcoming_board_meeting'] = self.event_model.upcoming_board_meeting() @@ -66,31 +81,20 @@ class LABillDetail(BillDetailView): def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context['actions'] = self.get_object().actions.all().order_by('-order') - context['attachments'] = self.get_object().attachments.all().order_by(Lower('note')).exclude(note="Board Report") + context['attachments'] = self.get_object().attachments.all().order_by(Lower('note')) - context['board_report'] = self.get_object().attachments.get(note="Board Report") + context['board_report'] = self.get_object().versions.get(note="Board Report") item = context['legislation'] - actions = Action.objects.filter(_bill_id=item.ocd_id) + actions = self.get_object().actions.all() organization_lst = [action.organization for action in actions] context['sponsorships'] = set(organization_lst) - # Create URL for packet download. - packet_slug = item.ocd_id.replace('/', '-') - try: - r = requests.head(MERGER_BASE_URL + '/document/' + packet_slug) - if r.status_code == 200: - context['packet_url'] = MERGER_BASE_URL + '/document/' + packet_slug - except: - context['packet_url'] = None - - # Create list of related board reports, ordered by descending last_action_date. - # Thanks https://stackoverflow.com/a/2179053 for how to handle null last_action_date - if context['legislation'].related_bills.all(): - all_related_bills = context['legislation'].related_bills.all().values('related_bill_identifier') - related_bills = LAMetroBill.objects.filter(identifier__in=all_related_bills) - minimum_date = datetime(MINYEAR, 1, 1, tzinfo=app_timezone) - context['related_bills'] = sorted(related_bills, key=lambda bill: bill.get_last_action_date() or minimum_date, reverse=True) + related_bills = context['legislation']\ + .related_bills\ + .annotate(latest_date=Max('related_bill__actions__date'))\ + .order_by('-latest_date') + context['related_bills'] = related_bills return context @@ -115,11 +119,15 @@ def post(self, request, *args, **kwargs): # Validate forms and redirect. if url_form.is_valid(): agenda_url = url_form['agenda'].value() - document_obj, created = EventDocument.objects.get_or_create(event=event, - url=agenda_url, updated_at= timezone.now()) - document_obj.note = ('Event Document - Manual upload URL') + document_obj, created = EventDocument.objects.get_or_create( + event=event, + note='Event Document - Manual upload URL') + + document_obj.date=timezone.now().date() document_obj.save() + document_obj.links.create(url=agenda_url) + return HttpResponseRedirect('/event/%s' % event_slug) elif pdf_form.is_valid() and 'pdf_form' in request.POST: agenda_pdf = request.FILES['agenda'] @@ -142,110 +150,56 @@ def get_context_data(self, **kwargs): # Metro admins should see a status report if Legistar is down. # GET the calendar page, which contains relevant URL for agendas. - if self.request.user.is_authenticated(): + if self.request.user.is_authenticated: r = requests.get('https://metro.legistar.com/calendar.aspx') context['legistar_ok'] = r.ok - # Create URL for packet download. - packet_slug = event.ocd_id.replace('/', '-') try: - r = requests.head(MERGER_BASE_URL + '/document/' + packet_slug) - if r.status_code == 200: - context['packet_url'] = MERGER_BASE_URL + '/document/' + packet_slug - elif r.status_code == 404: - context['packet_url'] = None - except: - context['packet_url'] = None - - # Logic for getting relevant board report information. - with connection.cursor() as cursor: - query = ''' - SELECT DISTINCT - b.identifier, - b.ocd_id, - b.slug, - b.ocr_full_text, - i.description, - d_bill.url, - i.order, - i.notes - FROM councilmatic_core_billdocument AS d_bill - INNER JOIN councilmatic_core_eventagendaitem as i - ON i.bill_id=d_bill.bill_id - INNER JOIN councilmatic_core_eventdocument as d_event - ON i.event_id=d_event.event_id - INNER JOIN councilmatic_core_bill AS b - ON d_bill.bill_id=b.ocd_id - WHERE d_event.event_id='{}' - AND trim(lower(d_bill.note)) LIKE 'board report%' - ORDER BY i.order - '''.format(event.ocd_id) - - cursor.execute(query) - - # Get field names - columns = [c[0] for c in cursor.description] - columns.append('packet_url') - columns.append('inferred_status') - - # Add field for packet_url - cursor_copy = [] - packet_url = None - - for obj in cursor: - packet_url = None - # Add packet slug - ocd_id = obj[1] - packet_slug = ocd_id.replace('/', '-') - - r = requests.head(MERGER_BASE_URL + '/document/' + packet_slug) - if r.status_code == 200: - packet_url = MERGER_BASE_URL + '/document/' + packet_slug - - obj = obj + (packet_url,) - - # The cursor object potentially includes public and private bills. - # However, the LAMetroBillManager excludes private bills - # from the LAMetroBill queryset. - # Attempting to `get` a private bill from LAMetroBill.objects - # raises a DoesNotExist error. As a precaution, we use filter() - # rather than get(). - board_report = LAMetroBill.objects.filter(ocd_id=ocd_id) - if board_report: - obj = obj + (board_report.first().inferred_status,) - cursor_copy.append(obj) - - # Create a named tuple - board_report_tuple = namedtuple('BoardReportProperties', columns, rename=True) - # Put results inside a list with assigned fields (from namedtuple) - related_board_reports = [board_report_tuple(*r) for r in cursor_copy] - - # Find agenda link. - if event.documents.all(): - for document in event.documents.all(): - if "Agenda" in document.note: - context['agenda_url'] = document.url - context['document_timestamp'] = document.updated_at - elif "Manual upload URL" in document.note: - context['uploaded_agenda_url'] = document.url - context['document_timestamp'] = document.updated_at - elif "Manual upload PDF" in document.note: - context['uploaded_agenda_pdf'] = document.url - context['document_timestamp'] = document.updated_at - ''' - LA Metro Councilmatic uses the adv_cache library to partially cache templates: in the event view, we cache the entire template, except the iframe. (N.B. With this library, the views do not cached, unless explicitly wrapped in a django cache decorator. - Nonetheless, several popular browsers (e.g., Chrome and Firefox) retrieve cached iframe images, regardless of the site's caching specifications. - We use the agenda's "updated_at" timestamp to bust the iframe cache: we save it inside context and then assign it as the "name" of the iframe, preventing the browser from retrieving a cached iframe, when the timestamp changes. - ''' - - context['related_board_reports'] = related_board_reports - context['base_url'] = PIC_BASE_URL # Give JS access to this variable - - # Render forms if not a POST request - if 'url_form' not in context: - context['url_form'] = AgendaUrlForm() - if 'pdf_form' not in context: - context['pdf_form'] = AgendaPdfForm() + context['minutes'] = event.documents.get(note__icontains='minutes') + except EventDocument.DoesNotExist: + pass + + agenda_with_board_reports = event.agenda\ + .filter(related_entities__bill__versions__isnull=False)\ + .annotate(int_order=Cast('order', IntegerField()))\ + .order_by('int_order') + + # Find agenda link. + if event.documents.all(): + for document in event.documents.all(): + if "Agenda" in document.note: + context['agenda_url'] = document.links.first().url + context['document_timestamp'] = document.date + elif "Manual upload URL" in document.note: + context['uploaded_agenda_url'] = document.links.first().url + context['document_timestamp'] = document.date + elif "Manual upload PDF" in document.note: + context['uploaded_agenda_pdf'] = document.links.first().url + context['document_timestamp'] = document.date + ''' + LA Metro Councilmatic uses the adv_cache library + to partially cache templates: in the event view, we cache + the entire template, except the iframe. (N.B. With + this library, the views do not cached, unless + explicitly wrapped in a django cache decorator. + Nonetheless, several popular browsers (e.g., + Chrome and Firefox) retrieve cached iframe images, + regardless of the site's caching specifications. + We use the agenda's "date" timestamp to bust + the iframe cache: we save it inside context and + then assign it as the "name" of the iframe, + preventing the browser from retrieving a cached + iframe, when the timestamp changes. + ''' + + context['related_board_reports'] = agenda_with_board_reports + context['base_url'] = PIC_BASE_URL # Give JS access to this variable + + # Render forms if not a POST request + if 'url_form' not in context: + context['url_form'] = AgendaUrlForm() + if 'pdf_form' not in context: + context['pdf_form'] = AgendaPdfForm() return context @@ -256,9 +210,12 @@ def handle_uploaded_agenda(agenda, event): destination.write(chunk) # Create the document in database - document_obj, created = EventDocument.objects.get_or_create(event=event, - url='pdf/agenda-%s.pdf' % event.slug, updated_at= timezone.now()) - document_obj.note = ('Event Document - Manual upload PDF') + document_obj, created = EventDocument.objects.get_or_create( + event=event, + note='Event Document - Manual upload PDF') + + document_obj.date = timezone.now().date + document_obj.links.create(url='pdf/agenda-%s.pdf' % event.slug) document_obj.save() # Collect static to render PDF on server @@ -267,13 +224,13 @@ def handle_uploaded_agenda(agenda, event): def delete_submission(request, event_slug): event = LAMetroEvent.objects.get(slug=event_slug) - event_doc = EventDocument.objects.filter(event_id=event.ocd_id, note__icontains='Manual upload') + event_doc = EventDocument.objects.filter(event_id=event.id, note__icontains='Manual upload') for e in event_doc: # Remove stored PDF from Metro app. if 'Manual upload PDF' in e.note: try: - os.remove('lametro/static/%s' % e.url ) + os.remove('lametro/static/%s' % e.links.get().url ) except OSError: pass e.delete() @@ -285,13 +242,15 @@ class LAMetroEventsView(EventsView): template_name = 'lametro/events.html' def get_context_data(self, **kwargs): - context = super(LAMetroEventsView, self).get_context_data(**kwargs) + context = {} # Did the user set date boundaries? start_date_str = self.request.GET.get('from') end_date_str = self.request.GET.get('to') day_grouper = lambda x: (x.start_time.year, x.start_time.month, x.start_time.day) + minutes_queryset = EventDocument.objects.filter(note__icontains='minutes') + # If yes... if start_date_str and end_date_str: context['start_date'] = start_date_str @@ -305,6 +264,11 @@ def get_context_data(self, **kwargs): .filter(start_time__lt=end_date_time)\ .order_by('start_time')\ + select_events = select_events.prefetch_related(Prefetch('documents', + minutes_queryset, + to_attr='minutes'))\ + .prefetch_related('minutes__links') + org_select_events = [] for event_date, events in itertools.groupby(select_events, key=day_grouper): @@ -345,8 +309,13 @@ def get_context_data(self, **kwargs): # Past events past_events = LAMetroEvent.objects\ .with_media()\ - .filter(start_time__lt=datetime.now(app_timezone))\ - .order_by('-start_time')\ + .filter(start_time__lt=datetime.datetime.now(app_timezone))\ + .order_by('-start_time') + + past_events = past_events.prefetch_related(Prefetch('documents', + minutes_queryset, + to_attr='minutes'))\ + .prefetch_related('minutes__links') org_past_events = [] @@ -357,7 +326,7 @@ def get_context_data(self, **kwargs): context['past_events'] = org_past_events context['user_subscribed'] = False - if self.request.user.is_authenticated(): + if self.request.user.is_authenticated: user = self.request.user context['user'] = user @@ -370,119 +339,30 @@ def get_context_data(self, **kwargs): class LABoardMembersView(CouncilMembersView): template_name = 'lametro/board_members.html' - model = LAMetroPost + def map(self): + + return {} def get_queryset(self): - posts = LAMetroPost.objects.filter(_organization__ocd_id=settings.OCD_CITY_COUNCIL_ID) + get_kwarg = {'name': settings.OCD_CITY_COUNCIL_NAME} - return posts + return Organization.objects.get(**get_kwarg)\ + .memberships\ + .filter(Q(role='Board Member') | + Q(role='Nonvoting Board Member'))\ + .filter(end_date_dt__gte=Now()) def get_context_data(self, *args, **kwargs): - context = super(LABoardMembersView, self).get_context_data(**kwargs) + context = super(CouncilMembersView, self).get_context_data(**kwargs) + context['seo'] = self.get_seo_blob() - context['map_geojson'] = None + board = LAMetroOrganization.objects.get(name=settings.OCD_CITY_COUNCIL_NAME) + context['recent_activity'] = board.actions.order_by('-date', '-bill__identifier', '-order') + context['recent_events'] = board.recent_events if settings.MAP_CONFIG: - - map_geojson = { - 'type': 'FeatureCollection', - 'features': [] - } - - map_geojson_sectors = { - 'type': 'FeatureCollection', - 'features': [] - } - - map_geojson_city = { - 'type': 'FeatureCollection', - 'features': [] - } - - for post in self.object_list: - if post.shape: - council_member = "Vacant" - detail_link = "" - if post.current_members: - for membership in post.current_members: - council_member = membership.person.name - detail_link = membership.person.slug - - feature = { - 'type': 'Feature', - 'geometry': json.loads(post.shape), - 'properties': { - 'district': post.label, - 'council_member': council_member, - 'detail_link': '/person/' + detail_link, - 'select_id': 'polygon-{}'.format(slugify(post.label)), - }, - } - - if 'council_district' in post.division_ocd_id: - map_geojson['features'].append(feature) - - if 'la_metro_sector' in post.division_ocd_id: - map_geojson_sectors['features'].append(feature) - - if post.division_ocd_id == 'ocd-division/country:us/state:ca/place:los_angeles': - map_geojson_city['features'].append(feature) - - context['map_geojson'] = json.dumps(map_geojson) - context['map_geojson_sectors'] = json.dumps(map_geojson_sectors) - context['map_geojson_city'] = json.dumps(map_geojson_city) - - - with connection.cursor() as cursor: - today = timezone.now().date() - - sql = ''' - SELECT p.ocd_id, p.name, array_agg(pt.label) as label, - m.extras, - array_agg(m.role) as role, - split_part(p.name, ' ', 2) AS last_name - FROM councilmatic_core_membership as m - INNER JOIN councilmatic_core_post as pt - ON pt.ocd_id=m.post_id - INNER JOIN councilmatic_core_person as p - ON m.person_id=p.ocd_id - WHERE m.organization_id='ocd-organization/42e23f04-de78-436a-bec5-ab240c1b977c' - AND m.end_date >= '{0}' - AND m.person_id <> 'ocd-person/912c8ddf-8d04-4f7f-847d-2daf84e096e2' - GROUP BY p.ocd_id, p.name, m.extras - ORDER BY last_name - '''.format(today) - - cursor.execute(sql) - - columns = [c[0] for c in cursor.description] - columns.append('index') - cursor_copy = [] - - # from operator import itemgetter - for obj in cursor: - if '1st Vice Chair' in obj[3]: - obj = obj + ("2",) - elif '2nd Vice Chair' in obj[3]: - obj = obj + ("3",) - elif 'Chair' in obj[3]: - obj = obj + ("1",) - elif 'Nonvoting Board Member' in obj[3]: - obj = obj + ("5",) - else: - obj = obj + ("4",) - cursor_copy.append(obj) - - # Create tuple-like object...iterable and accessible by field names. - membership_tuple = namedtuple('Membership', columns) - membership_objects = [membership_tuple(*r) for r in cursor_copy] - membership_objects = sorted(membership_objects, key=lambda x: x[5]) - context['membership_objects'] = membership_objects - - board = LAMetroOrganization.objects.get(ocd_id=settings.OCD_CITY_COUNCIL_ID) - context['recent_activity'] = board.actions.order_by('-date', '-_bill__identifier', '-order') - context['recent_events'] = board.recent_events + context.update(self.map()) return context @@ -493,7 +373,7 @@ class LAMetroAboutView(AboutView): def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) - context['timestamp'] = datetime.now(app_timezone).strftime('%m%d%Y%s') + context['timestamp'] = datetime.datetime.now(app_timezone).strftime('%m%d%Y%s') return context @@ -501,38 +381,37 @@ def get_context_data(self, **kwargs): class LACommitteesView(CommitteesView): template_name = 'lametro/committees.html' - def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) + def get_queryset(self): + ''' + We only want committees that have at least one member who is not + the CEO. We also want to not count the CEO in the committee + size - with connection.cursor() as cursor: + ''' - sql = (''' - SELECT DISTINCT on (o.ocd_id, m.person_id) o.*, m.person_id, m.role, p.name AS member_name - FROM councilmatic_core_organization AS o - JOIN councilmatic_core_membership AS m - ON o.ocd_id=m.organization_id - JOIN councilmatic_core_person as p - ON p.ocd_id=m.person_id - WHERE o.classification='committee' - AND m.end_date::date > NOW()::date - AND m.role != 'Chief Executive Officer' - ORDER BY o.ocd_id, m.person_id, m.end_date; - ''') + ceo = Membership.objects\ + .select_related('person')\ + .get(post__role='Chief Executive Officer', + end_date_dt__gt=Now())\ + .person - cursor.execute(sql) + memberships = Membership.objects\ + .exclude(person=ceo)\ + .filter(end_date_dt__gt=Now(), + organization__classification='committee') + + qs = LAMetroOrganization.objects\ + .filter(classification='committee')\ + .filter(memberships__in=memberships)\ + .distinct() - columns = [c[0] for c in cursor.description] - committees_tuple = namedtuple('Committee', columns, rename=True) - data = [committees_tuple(*r) for r in cursor] - groups = [] + qs = qs.prefetch_related(Prefetch('memberships', + memberships, + to_attr='current_members')) - for key, group in groupby(data, lambda x: x[1]): - groups.append(list(group)) + return qs + - committees_list = groups - context["committees_list"] = committees_list - - return context class LACommitteeDetailView(CommitteeDetailView): @@ -548,65 +427,29 @@ def get_context_data(self, **kwargs): description = settings.COMMITTEE_DESCRIPTIONS.get(committee.slug) context['committee_description'] = description - with connection.cursor() as cursor: - base_sql = ''' - SELECT - m.role, - p.name, p.slug, p.ocd_id, - m.extras, - array_agg(mm.label::VARCHAR) - FILTER (WHERE mm.label is not Null) as label, - split_part(p.name, ' ', 2) AS last_name - FROM councilmatic_core_membership AS m - LEFT JOIN ( - SELECT - person_id, - array_agg(DISTINCT pt.label) as label - FROM councilmatic_core_membership AS m - JOIN councilmatic_core_post AS pt - ON m.post_id=pt.ocd_id - WHERE m.organization_id = %s - GROUP BY person_id - ) AS mm - USING(person_id) - JOIN councilmatic_core_person AS p - ON m.person_id = p.ocd_id - WHERE m.organization_id = %s - AND m.end_date::date > NOW()::date - AND m.role != 'Chief Executive Officer' - GROUP BY - m.role, p.name, p.slug, p.ocd_id, m.extras - {order_by} - ''' - committee_sql = base_sql.format(order_by="ORDER BY CASE " + - "WHEN m.role='Chair' THEN 0 " + - "WHEN m.role='Vice Chair' THEN 1 " + - "WHEN m.role='Member' THEN 2" + - "END") - - cursor.execute(committee_sql, [settings.OCD_CITY_COUNCIL_ID, committee.ocd_id]) - columns = [c[0] for c in cursor.description] - results_tuple = namedtuple('Member', columns) - objects_list = [results_tuple(*r) for r in cursor] - - context['membership_objects'] = objects_list - - ad_hoc_committee_sql = base_sql.format(order_by="ORDER BY CASE " + - "WHEN m.role='Chair' THEN 0 " + - "WHEN m.role='1st Vice Chair' THEN 1 " + - "WHEN m.role='2nd Vice Chair' THEN 2 " - "WHEN m.role='Member' THEN 3" + - "END") - - cursor.execute(ad_hoc_committee_sql, [settings.OCD_CITY_COUNCIL_ID, committee.ocd_id]) - - columns = [c[0] for c in cursor.description] - results_tuple = namedtuple('Member', columns) - objects_list = [results_tuple(*r) for r in cursor] - - context['ad_hoc_list'] = objects_list - - context['ceo'] = Person.objects.filter(memberships__role='Chief Executive Officer', memberships___organization=committee.ocd_id).first() + try: + ceo = Membership.objects\ + .get(post__role='Chief Executive Officer', + end_date_dt__gt=Now())\ + .person + except Membership.DoesNotExist: + ceo = None + + non_ceos = committee.all_members\ + .annotate(index=Case( + When(role='Chair', then=Value(0)), + When(role='Vice Chair', then=Value(1)), + When(role='1st Vice Chair', then=Value(1)), + When(role='2nd Vice Chair', then=Value(2)), + When(role='Member', then=Value(3)), + default=Value(999), + output_field=IntegerField()))\ + .exclude(person=ceo)\ + .order_by('index', 'person__family_name', 'person__given_name') + + context['non_ceos'] = non_ceos + + context['ceo'] = ceo return context @@ -648,55 +491,33 @@ def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) person = context['person'] - title = '' - qualifying_post = '' # board membership criteria met by person in question - m = person.latest_council_membership - if person.current_council_seat: - title = m.role - if m.post: - qualifying_post = m.post.label - if m.extras.get('acting'): - qualifying_post = 'Acting' + ' ' + qualifying_post - - else: - title = 'Former %s' % m.role - context['title'] = title - context['qualifying_post'] = qualifying_post + context['qualifying_post'] = person.current_council_seat.post.acting_label if person.committee_sponsorships: context['sponsored_legislation'] = person.committee_sponsorships else: context['sponsored_legislation'] = [] - with connection.cursor() as cursor: - - sql = (''' - SELECT o.name as organization, o.slug as org_slug, m.role - FROM councilmatic_core_membership AS m - JOIN councilmatic_core_organization AS o - ON o.ocd_id = m.organization_id - WHERE m.person_id = %s - AND m.end_date::date > NOW()::date - AND m.organization_id != %s - ORDER BY - CASE - WHEN m.role='Chair' THEN 0 - WHEN m.role='Vice Chair' THEN 1 - WHEN m.role='Member' THEN 2 - END - ''') - - cursor.execute(sql, [person.ocd_id, settings.OCD_CITY_COUNCIL_ID]) - - columns = [c[0] for c in cursor.description] - - results_tuple = namedtuple('Member', columns) - memberships_list = [results_tuple(*r) for r in cursor] - context['memberships_list'] = memberships_list + context['memberships_list'] = person.current_memberships\ + .exclude(organization__name='Board of Directors')\ + .annotate(index=Case( + When(role='Chair', then=Value(0)), + When(role='Vice Chair', then=Value(1)), + When(role='1st Vice Chair', then=Value(1)), + When(role='2nd Vice Chair', then=Value(2)), + When(role='Member', then=Value(3)), + default=Value(999), + output_field=IntegerField()))\ + .order_by('index') if person.slug in MEMBER_BIOS: context['member_bio'] = MEMBER_BIOS[person.slug] + try: + context['website_url'] = person.links.get(note='web_site').url + except PersonLink.DoesNotExist: + pass + return context diff --git a/requirements.txt b/requirements.txt index 4d1c3ea36..798ca074d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,11 @@ -psycopg2==2.7.6.1 -django-councilmatic==0.10.16 +https://github.com/opencivicdata/python-opencivicdata/archive/master.zip +django-councilmatic[convert_docs]@https://github.com/datamade/django-councilmatic/archive/la_metro_accomodations.zip django-debug-toolbar==1.9.1 raven==5.30.0 gunicorn==19.6.0 -PyPDF2==1.26.0 -requests==2.20.0 pytest pytest-django pytest-mock==1.6.3 lxml==4.1.1 https://github.com/opencivicdata/python-legistar-scraper/zipball/master +dj_database_url From 6f19fb496c1a417b6d8deb27e45df5935361dac1 Mon Sep 17 00:00:00 2001 From: Forest Gregg Date: Wed, 20 Nov 2019 10:59:03 -0600 Subject: [PATCH 3/8] dockerized --- Dockerfile | 11 +++++++---- docker-compose.yml | 33 ++++++++++++++++++--------------- docker-entrypoint.sh | 8 ++++++++ 3 files changed, 33 insertions(+), 19 deletions(-) create mode 100755 docker-entrypoint.sh diff --git a/Dockerfile b/Dockerfile index 3c0daee7b..88554633d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,14 +6,17 @@ ENV PYTHONUNBUFFERED=1 RUN apt-get update && \ apt-get install -y libxml2-dev libxslt1-dev antiword unrtf poppler-utils \ pstotext tesseract-ocr flac ffmpeg lame libmad0 \ - libsox-fmt-mp3 sox libjpeg-dev swig + libsox-fmt-mp3 sox libjpeg-dev swig gdal-bin RUN mkdir /app WORKDIR /app COPY ./requirements.txt /app/requirements.txt -RUN pip install --upgrade pip -RUN pip install --no-cache-dir -r requirements.txt -RUN pip install textract==1.6.3 +RUN pip install --upgrade pip && \ + pip install --no-cache-dir -r requirements.txt COPY . /app + +RUN DATABASE_URL='' python manage.py collectstatic --noinput + +ENTRYPOINT ["/app/docker-entrypoint.sh"] diff --git a/docker-compose.yml b/docker-compose.yml index 65b923db1..40cc1b3a4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,7 +8,7 @@ services: stdin_open: true tty: true ports: - - 8000:8000 + - 8011:8000 depends_on: postgres: condition: service_healthy @@ -16,26 +16,17 @@ services: condition: service_started volumes: - .:/app - - ${PWD}/councilmatic/settings_deployment.py.example:/app/councilmatic/settings_deployment.py + - ${PWD}/councilmatic/settings_deployment.py:/app/councilmatic/settings_deployment.py + - /Users/fgregg/work/django-councilmatic:/django-councilmatic environment: POSTGRES_HOST: postgres + DJANGO_MANAGEPY_MIGRATE: "on" command: python manage.py runserver 0.0.0.0:8000 - migration: - container_name: lametro-migration - image: lametro:latest - depends_on: - - app - - postgres - volumes: - - .:/app - - ${PWD}/councilmatic/settings_deployment.py.example:/app/councilmatic/settings_deployment.py - command: python manage.py migrate - postgres: container_name: lametro-postgres restart: always - image: postgres:9.6 + image: mdillon/postgis:9.6 healthcheck: test: ["CMD-SHELL", "pg_isready -U postgres"] interval: 10s @@ -46,7 +37,7 @@ services: volumes: - lametro-db-data:/var/lib/postgresql/data ports: - - 32001:5432 + - 32005:5432 solr: image: solr:7.5 @@ -61,6 +52,18 @@ services: SOLR_LOG_LEVEL: INFO restart: on-failure + scrapers: + build: https://github.com/datamade/scrapers-us-municipal.git#la_metro_upgrade + stdin_open: true + tty: true + depends_on: + postgres: + condition: service_healthy + environment: + SHARED_DB: 'True' + DATABASE_URL: 'postgis://postgres:@postgres/lametro' + command: sh -c 'pupa update --rpm=600 lametro people && pupa update --rpm=600 lametro bills window=30 && pupa update --rpm=600 lametro events' + volumes: lametro-solr-data: lametro-db-data: diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100755 index 000000000..da77a0cc6 --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -e + +if [ "$DJANGO_MANAGEPY_MIGRATE" = 'on' ]; then + python manage.py migrate --noinput +fi + +exec "$@" From dc25ef02eeebcab1dcef9854d2afb1187ab89035 Mon Sep 17 00:00:00 2001 From: Forest Gregg Date: Wed, 20 Nov 2019 10:59:19 -0600 Subject: [PATCH 4/8] engine type is postgis not posgresql --- councilmatic/settings_deployment.py.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/councilmatic/settings_deployment.py.example b/councilmatic/settings_deployment.py.example index ede29bcc8..4884226a7 100644 --- a/councilmatic/settings_deployment.py.example +++ b/councilmatic/settings_deployment.py.example @@ -20,7 +20,7 @@ DEBUG = True DATABASES = { 'default': { - 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'ENGINE': 'django.contrib.gis.db.backends.postgis', 'NAME': 'lametro', 'USER': 'postgres', 'HOST': 'postgres', From 677148a15b34f359049ccc7e2cdee1866626d6f4 Mon Sep 17 00:00:00 2001 From: Forest Gregg Date: Wed, 20 Nov 2019 11:19:22 -0600 Subject: [PATCH 5/8] remove packet models for reviewability --- lametro/models.py | 114 ---------------------------------------------- 1 file changed, 114 deletions(-) diff --git a/lametro/models.py b/lametro/models.py index 929d57648..85875cb27 100644 --- a/lametro/models.py +++ b/lametro/models.py @@ -554,117 +554,3 @@ class Meta: guid = models.CharField(max_length=256) name = models.CharField(max_length=256, unique=True) - - -class BillPacket(models.Model): - - bill = models.OneToOneField(LAMetroBill, - related_name='packet', - on_delete=models.CASCADE) - updated_at = models.DateTimeField(auto_now=True) - url = models.URLField() - ready = models.BooleanField(default=False) - - def save(self, *args, **kwargs): - - self._merge_docs() - - self.url = settings.MERGER_BASE_URL + '/document/' + self.bill.slug - - response = requests.head(self.url) - - super().save(*args, **kwargs) - - def is_ready(self): - - if not self.ready: - response = requests.head(self.url) - if response.status_code == 200: - self.ready = True - super().save() - - return self.ready - - @property - def related_files(self): - board_report = self.bill.versions.get() - - attachments = self.bill.documents\ - .annotate( - index=Case( - When(note__istartswith = '0', then=Value('z')), - default=F('note'), - output_field=models.CharField()))\ - .order_by('index') - - doc_links = [board_report.links.get().url] - - # sometime there are more than url for the same document name - # https://metro.legistar.com/LegislationDetail.aspx?ID=3104422&GUID=C30D3376-7265-477B-AFFA-815270400538%3e%5d%3e - # I'm not sure if this a data problem or not, so we'll just - # add all the doc links - doc_links += [link.url - for doc in attachments - for link in doc.links.all()] - - return doc_links - - def _merge_docs(self): - - merge_url = settings.MERGER_BASE_URL + '/merge_pdfs/' + self.bill.slug - - requests.post(merge_url, json=self.related_files) - - -class EventPacket(models.Model): - - event = models.OneToOneField(LAMetroEvent, - related_name='packet', - on_delete=models.CASCADE) - updated_at = models.DateTimeField(auto_now=True) - url = models.URLField() - ready = models.BooleanField(default=False) - - def save(self, *args, **kwargs): - - self._merge_docs() - - self.url = settings.MERGER_BASE_URL + '/document/' + self.event.slug - - super().save(*args, **kwargs) - - def is_ready(self): - - if not self.ready: - response = requests.head(self.url) - if response.status_code == 200: - self.ready = True - super().save() - - return self.ready - - @property - def related_files(self): - - agenda_doc = self.event.documents.get(note='Agenda') - - related = [agenda_doc.links.get().url] - - agenda_items = self.event.agenda\ - .filter(related_entities__bill__documents__isnull=False)\ - .annotate(int_order=Cast('order', models.IntegerField()))\ - .order_by('int_order')\ - .distinct() - - for item in agenda_items: - for entity in item.related_entities.filter(bill__isnull=False): - bill_packet = BillPacket(bill=entity.bill) - related.extend(bill_packet.related_files) - - return related - - def _merge_docs(self): - - merge_url = settings.MERGER_BASE_URL + '/merge_pdfs/' + self.event.slug - - requests.post(merge_url, json=self.related_files) From 58ae40dc24cdb371163e7282db3a2297f49bbfc5 Mon Sep 17 00:00:00 2001 From: Forest Gregg Date: Mon, 25 Nov 2019 13:33:01 -0500 Subject: [PATCH 6/8] Update docker-compose.yml Co-Authored-By: Hannah Cushman --- docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 40cc1b3a4..b92dbf529 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,7 +17,6 @@ services: volumes: - .:/app - ${PWD}/councilmatic/settings_deployment.py:/app/councilmatic/settings_deployment.py - - /Users/fgregg/work/django-councilmatic:/django-councilmatic environment: POSTGRES_HOST: postgres DJANGO_MANAGEPY_MIGRATE: "on" From 2feeec1862c77cb28a093d40fd6b4b8f11d9d055 Mon Sep 17 00:00:00 2001 From: Forest Gregg Date: Mon, 25 Nov 2019 14:31:15 -0500 Subject: [PATCH 7/8] respond to some of @hancush's review --- councilmatic/settings.py | 4 --- councilmatic/settings_deployment.py.example | 4 +++ lametro/models.py | 27 ++++++++++++++------- lametro/views.py | 15 ++---------- 4 files changed, 24 insertions(+), 26 deletions(-) diff --git a/councilmatic/settings.py b/councilmatic/settings.py index b2ecc5cd5..18c45c6ef 100644 --- a/councilmatic/settings.py +++ b/councilmatic/settings.py @@ -119,7 +119,3 @@ HAYSTACK_SIGNAL_PROCESSOR = 'haystack.signals.RealtimeSignalProcessor' ADV_CACHE_INCLUDE_PK = True - -import socket -hostname, _, ips = socket.gethostbyname_ex(socket.gethostname()) -INTERNAL_IPS = [ip[:-1] + '1' for ip in ips] + ['127.0.0.1', '10.0.2.2'] diff --git a/councilmatic/settings_deployment.py.example b/councilmatic/settings_deployment.py.example index 4884226a7..3df4e3f4d 100644 --- a/councilmatic/settings_deployment.py.example +++ b/councilmatic/settings_deployment.py.example @@ -85,3 +85,7 @@ LOGGING = { # Set to False in production! SHOW_TEST_EVENTS = True + +import socket +hostname, _, ips = socket.gethostbyname_ex(socket.gethostname()) +INTERNAL_IPS = [ip[:-1] + '1' for ip in ips] + ['127.0.0.1', '10.0.2.2'] diff --git a/lametro/models.py b/lametro/models.py index 85875cb27..3c800775b 100644 --- a/lametro/models.py +++ b/lametro/models.py @@ -16,6 +16,8 @@ from councilmatic_core.models import Bill, Event, Post, Person, Organization, EventManager, Membership +import councilmatic_core.models + from opencivicdata.legislative.models import EventMedia, EventDocument, EventDocumentLink, EventAgendaItem, EventRelatedEntity, RelatedBill from proxy_overrides.related import ProxyForeignKey @@ -62,11 +64,6 @@ def get_queryset(self): may slip through the crevices of Councilmatic display logic. ''' filtered_qs = super().get_queryset() - # .exclude(restrict_view=True)\ - # .filter(Q(related_agenda_items__event__status='passed') | \ - # Q(related_agenda_items__event__status='cancelled') | \ - # Q(bill_type='Board Box'))\ - # .distinct() return filtered_qs @@ -181,14 +178,14 @@ def latest_council_membership(self): filter_kwarg = {'organization__name': settings.OCD_CITY_COUNCIL_NAME,} city_council_memberships = self.memberships.filter(**filter_kwarg) - # We want to exclude memberships like 1st chair and just - # get the memberships confer membership to the org + # Select posts denoting membership, i.e., exclude leadership + # posts, like 1st Chair # # see https://github.com/opencivicdata/python-opencivicdata/issues/129 primary_memberships = city_council_memberships.filter(Q(role='Board Member') | Q(role='Nonvoting Board Member')) - if primary_memberships.count(): + if primary_memberships.exists(): return primary_memberships.order_by('-end_date').first() return None @@ -236,6 +233,18 @@ def committee_sponsorships(self): return qs + @classmethod + def ceo(cls): + try: + ceo = Membership.objects\ + .get(post__role='Chief Executive Officer', + end_date_dt__gt=Now())\ + .person + except Membership.DoesNotExist: + ceo = None + + return ceo + class LAMetroEventManager(EventManager): def get_queryset(self): @@ -517,7 +526,7 @@ def upcoming_events(self): .all() return events -class Membership(Membership): +class Membership(councilmatic_core.models.Membership): class Meta: proxy = True diff --git a/lametro/views.py b/lametro/views.py index 9c65adf41..42d5005a7 100644 --- a/lametro/views.py +++ b/lametro/views.py @@ -388,12 +388,7 @@ def get_queryset(self): size ''' - - ceo = Membership.objects\ - .select_related('person')\ - .get(post__role='Chief Executive Officer', - end_date_dt__gt=Now())\ - .person + ceo = LAMetroPerson.ceo() memberships = Membership.objects\ .exclude(person=ceo)\ @@ -427,13 +422,7 @@ def get_context_data(self, **kwargs): description = settings.COMMITTEE_DESCRIPTIONS.get(committee.slug) context['committee_description'] = description - try: - ceo = Membership.objects\ - .get(post__role='Chief Executive Officer', - end_date_dt__gt=Now())\ - .person - except Membership.DoesNotExist: - ceo = None + ceo = LAMetroPerson.ceo() non_ceos = committee.all_members\ .annotate(index=Case( From 6fc94c3ad0bfc94f96488c6b0dd176608065f295 Mon Sep 17 00:00:00 2001 From: Forest Gregg Date: Mon, 25 Nov 2019 16:34:27 -0500 Subject: [PATCH 8/8] combine filter conditions --- lametro/models.py | 4 ++-- lametro/templatetags/lametro_extras.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lametro/models.py b/lametro/models.py index 3c800775b..811dc2d85 100644 --- a/lametro/models.py +++ b/lametro/models.py @@ -226,8 +226,8 @@ def committee_sponsorships(self): ''' qs = LAMetroBill.objects\ .defer('extras')\ - .filter(actions__organization__classification='committee')\ - .filter(actions__organization__memberships__in=self.current_memberships)\ + .filter(actions__organization__classification='committee', + actions__organization__memberships__in=self.current_memberships)\ .order_by('-actions__date')\ .distinct()[:10] diff --git a/lametro/templatetags/lametro_extras.py b/lametro/templatetags/lametro_extras.py index 1b23123b4..535ffdf9d 100644 --- a/lametro/templatetags/lametro_extras.py +++ b/lametro/templatetags/lametro_extras.py @@ -148,8 +148,8 @@ def find_agenda_url(all_documents): ''' This filter determines how to format the URL link, particularly, in the case of manually uploaded agenda. ''' - valid_urls = [x.url for x in all_documents if (x.note == 'Agenda' or x.note == 'Event Document - Manual upload URL')] - pdf_url = [('static/' + x.url) for x in all_documents if x.note == 'Event Document - Manual upload PDF'] + valid_urls = [url for x in all_documents if (x.note == 'Agenda' or x.note == 'Event Document - Manual upload URL') for url in x.links.all()] + pdf_url = [('static/' + url) for x in all_documents if x.note == 'Event Document - Manual upload PDF' for url in x.links.all()] valid_urls += pdf_url return valid_urls[0]