From d89d44aecf8183efa15a1da4f796a1611cba0fa5 Mon Sep 17 00:00:00 2001 From: James Turk Date: Thu, 27 Jun 2019 03:12:25 -0400 Subject: [PATCH 1/6] add SearchableBill model --- .../migrations/0009_searchablebill.py | 27 +++++++++++++++++++ opencivicdata/legislative/models/bill.py | 21 +++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 opencivicdata/legislative/migrations/0009_searchablebill.py diff --git a/opencivicdata/legislative/migrations/0009_searchablebill.py b/opencivicdata/legislative/migrations/0009_searchablebill.py new file mode 100644 index 0000000..17a0b90 --- /dev/null +++ b/opencivicdata/legislative/migrations/0009_searchablebill.py @@ -0,0 +1,27 @@ +# Generated by Django 2.0.13 on 2019-06-27 02:10 + +import django.contrib.postgres.search +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('legislative', '0008_longer_event_name'), + ] + + operations = [ + migrations.CreateModel( + name='SearchableBill', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('search_vector', django.contrib.postgres.search.SearchVectorField()), + ('raw_text', models.TextField()), + ('is_error', models.BooleanField(default=False)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('bill', models.OneToOneField(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='searchable', to='legislative.Bill')), + ('version', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='searchable', to='legislative.BillVersionLink')), + ], + ), + ] diff --git a/opencivicdata/legislative/models/bill.py b/opencivicdata/legislative/models/bill.py index e2e7878..568e375 100644 --- a/opencivicdata/legislative/models/bill.py +++ b/opencivicdata/legislative/models/bill.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals from django.db import models from django.contrib.postgres.fields import ArrayField, JSONField +from django.contrib.postgres.search import SearchVectorField from django.utils.encoding import python_2_unicode_compatible from opencivicdata.core.models.base import (OCDBase, LinkBase, OCDIDField, @@ -208,3 +209,23 @@ class BillSource(LinkBase): class Meta: db_table = 'opencivicdata_billsource' + + +class SearchableBill(models.Model): + """ + This model associates a single version's text with a given bill. + + This is done for a few reasons: + * bills with multiple versions aren't weighted more heavily than others + * this makes querying quite a bit more efficient (no need to deduplicate results) + + We'll also store error results, assuming that they're somewhat persistent. + """ + bill = models.OneToOneField(Bill, related_name="searchable", null=True, on_delete=models.CASCADE) + version = models.OneToOneField(BillVersionLink, related_name="searchable", on_delete=models.CASCADE) + + search_vector = SearchVectorField() + raw_text = models.TextField() + is_error = models.BooleanField(default=False) + + created_at = models.DateTimeField(auto_now_add=True) From 23a5175ef5f4b0dc8686f7ae61211379b0fdf6b0 Mon Sep 17 00:00:00 2001 From: James Turk Date: Thu, 27 Jun 2019 03:17:46 -0400 Subject: [PATCH 2/6] fix flake8 line length errors --- opencivicdata/legislative/models/bill.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/opencivicdata/legislative/models/bill.py b/opencivicdata/legislative/models/bill.py index 568e375..dafcac2 100644 --- a/opencivicdata/legislative/models/bill.py +++ b/opencivicdata/legislative/models/bill.py @@ -221,8 +221,17 @@ class SearchableBill(models.Model): We'll also store error results, assuming that they're somewhat persistent. """ - bill = models.OneToOneField(Bill, related_name="searchable", null=True, on_delete=models.CASCADE) - version = models.OneToOneField(BillVersionLink, related_name="searchable", on_delete=models.CASCADE) + bill = models.OneToOneField( + Bill, + related_name="searchable", + null=True, + on_delete=models.CASCADE + ) + version = models.OneToOneField( + BillVersionLink, + related_name="searchable", + on_delete=models.CASCADE + ) search_vector = SearchVectorField() raw_text = models.TextField() From d1f0fe9c4dec6623720a919af589065e86657ea3 Mon Sep 17 00:00:00 2001 From: James Turk Date: Thu, 27 Jun 2019 15:02:30 -0400 Subject: [PATCH 3/6] make SearchableBill importable --- opencivicdata/legislative/models/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opencivicdata/legislative/models/__init__.py b/opencivicdata/legislative/models/__init__.py index 370500a..4be4383 100644 --- a/opencivicdata/legislative/models/__init__.py +++ b/opencivicdata/legislative/models/__init__.py @@ -2,7 +2,7 @@ from .session import LegislativeSession from .bill import (Bill, BillAbstract, BillTitle, BillIdentifier, RelatedBill, BillSponsorship, BillDocument, BillVersion, BillDocumentLink, BillVersionLink, BillSource, - BillActionRelatedEntity, BillAction) + BillActionRelatedEntity, BillAction, SearchableBill) from .vote import (VoteEvent, VoteCount, PersonVote, VoteSource) from .event import (Event, EventLocation, EventMedia, EventMediaLink, EventDocument, EventLink, EventSource, EventParticipant, EventAgendaItem, EventRelatedEntity, From 66e5d9356379ba6aab0721d76815cde13ccdef16 Mon Sep 17 00:00:00 2001 From: James Turk Date: Mon, 8 Jul 2019 20:37:37 -0400 Subject: [PATCH 4/6] fix table name and field names --- .../legislative/migrations/0009_searchablebill.py | 12 ++++++++---- opencivicdata/legislative/models/bill.py | 10 +++++++--- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/opencivicdata/legislative/migrations/0009_searchablebill.py b/opencivicdata/legislative/migrations/0009_searchablebill.py index 17a0b90..0d3c9a2 100644 --- a/opencivicdata/legislative/migrations/0009_searchablebill.py +++ b/opencivicdata/legislative/migrations/0009_searchablebill.py @@ -1,4 +1,4 @@ -# Generated by Django 2.0.13 on 2019-06-27 02:10 +# Generated by Django 2.0.13 on 2019-06-27 14:32 import django.contrib.postgres.search from django.db import migrations, models @@ -16,12 +16,16 @@ class Migration(migrations.Migration): name='SearchableBill', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('search_vector', django.contrib.postgres.search.SearchVectorField()), - ('raw_text', models.TextField()), + ('search_vector', django.contrib.postgres.search.SearchVectorField(default=None)), + ('all_titles', models.TextField(default='')), + ('raw_text', models.TextField(default='')), ('is_error', models.BooleanField(default=False)), ('created_at', models.DateTimeField(auto_now_add=True)), ('bill', models.OneToOneField(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='searchable', to='legislative.Bill')), - ('version', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='searchable', to='legislative.BillVersionLink')), + ('version_link', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='searchable', to='legislative.BillVersionLink')), ], + options={ + 'db_table': 'opencivicdata_searchablebill', + }, ), ] diff --git a/opencivicdata/legislative/models/bill.py b/opencivicdata/legislative/models/bill.py index dafcac2..8956690 100644 --- a/opencivicdata/legislative/models/bill.py +++ b/opencivicdata/legislative/models/bill.py @@ -227,14 +227,18 @@ class SearchableBill(models.Model): null=True, on_delete=models.CASCADE ) - version = models.OneToOneField( + version_link = models.OneToOneField( BillVersionLink, related_name="searchable", on_delete=models.CASCADE ) - search_vector = SearchVectorField() - raw_text = models.TextField() + search_vector = SearchVectorField(default=None) + all_titles = models.TextField(default="") + raw_text = models.TextField(default="") is_error = models.BooleanField(default=False) created_at = models.DateTimeField(auto_now_add=True) + + class Meta: + db_table = 'opencivicdata_searchablebill' From d67b6ece73487f6d31a065eaf4eb62ce03257151 Mon Sep 17 00:00:00 2001 From: James Turk Date: Mon, 8 Jul 2019 20:43:33 -0400 Subject: [PATCH 5/6] drop django 2.0, EOL --- .travis.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index e98f35d..3300438 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,6 @@ python: - "3.7" env: - DJANGO_VERSION=1.11 - - DJANGO_VERSION=2.0 - DJANGO_VERSION=2.1 - DJANGO_VERSION=2.2 install: @@ -29,8 +28,6 @@ after_success: - coveralls matrix: exclude: - - python: "2.7" - env: DJANGO_VERSION=2.0 - python: "2.7" env: DJANGO_VERSION=2.1 - python: "2.7" From a8946b6d35bd2415384d7f9ec542cde31beac40c Mon Sep 17 00:00:00 2001 From: James Turk Date: Mon, 8 Jul 2019 20:46:19 -0400 Subject: [PATCH 6/6] bump to 2.4.0 --- CHANGELOG.md | 6 ++++++ setup.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index be8c1f8..4b2e3fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 2.4.0 + +Improvements requiring migrations: + +* added experimental SearchableBill model + ## 2.3.0 (2018-12-04) Improvements requiring migrations: diff --git a/setup.py b/setup.py index 198b0e6..1f39bbf 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ extras_require[":python_version<='2.7'"] = ["backports.csv"] setup(name="opencivicdata", - version='2.3.0', + version='2.4.0', author="James Turk", author_email='james@openstates.org', license="BSD",