Merge pull request #539 from WebarchivCZ/updates

1.0.1 + 1.0.2 + 1.0.3

.gitignore

@@ -3,6 +3,7 @@
 # Virtualenv
 # http://iamzed.com/2009/05/07/a-primer-on-virtualenv/
 .Python
+.python-version
 [Bb]in
 [Ii]nclude
 [Ll]ib

Seeder/api/serializers.py

@@ -23,6 +23,7 @@ class Meta:
 class ContactPersonSerializer(ModelSerializer):
     class Meta:
         model = publishers.models.ContactPerson
+        fields = '__all__'
 
 
 class PublisherSerializer(ModelSerializer):

Seeder/core/dashboard_data.py

@@ -1,5 +1,6 @@
 from django.urls import reverse
 from django.utils.translation import ugettext_lazy as _
+from django.utils import timezone
 from django.db.models import Count, Q
 from django.core.paginator import Paginator
 
@@ -120,6 +121,11 @@ def get_queryset(self):
             state=voting_models.constants.VOTE_INITIAL
         ).annotate(Count('vote')).order_by('vote__count')
 
+    def get_color(self, element):
+        # User has cast a vote in the voting round
+        has_voted = element.vote_set.filter(author=self.user).exists()
+        return 'success' if has_voted else ''
+
 
 class OpenToVoteRounds(VoteCard):
     """
@@ -203,7 +209,29 @@ def get_url(self, element):
         return reverse('qa:create', args=[str(element.id)])
 
     def get_queryset(self):
-        return source_models.Source.objects.needs_qa().filter(owner=self.user)
+        # Seed to today's date & user: each user has different & persists
+        import random
+        random.seed(str(self.user) + str(timezone.now().date()))
+        # Include all sources, both owned by user and not
+        qa_sources = source_models.Source.objects.needs_qa()
+        # Randomly select up to N sources for QA with at most M tries
+        random_qa = set()  # collect pks
+        tries = 0
+        while (
+            len(random_qa) < source_models.constants.RANDOM_QA_MAX_SOURCES and
+            tries < source_models.constants.RANDOM_QA_MAX_TRIES
+        ):
+            tries += 1
+            s = qa_sources[random.randint(0, qa_sources.count()-1)]
+            random_qa.add(s.pk)
+        # Order these by date so that the first item is the oldest
+        return source_models.Source.objects.filter(
+            pk__in=random_qa).order_by('created')
+
+    def get_count(self):
+        # Return number of performed QAs by the user today
+        return self.user.qualityassurancecheck_set.filter(
+            created__date=timezone.now()).count()
 
     def get_color(self, element):
         return element.css_class()

Seeder/harvests/forms.py

@@ -60,6 +60,7 @@ class Meta:
             'custom_sources',
             'topic_collections',
             'topic_collection_frequency',
+            'seeds_not_harvested',
         ]
         widgets = autocomplete_widgets
 

Seeder/harvests/migrations/0013_auto_20201103_0630.py

@@ -0,0 +1,19 @@
+# Generated by Django 2.2.12 on 2020-11-03 06:30
+
+import core.models
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('harvests', '0012_auto_20200507_1345'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='topiccollection',
+            name='date_to',
+            field=core.models.DatePickerField(blank=True, null=True, verbose_name='Date to'),
+        ),
+    ]

Seeder/harvests/migrations/0014_harvest_seeds_not_harvested.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.12 on 2020-11-03 08:11
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('harvests', '0013_auto_20201103_0630'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='harvest',
+            name='seeds_not_harvested',
+            field=models.TextField(blank=True, null=True, verbose_name='Seeds not harvested'),
+        ),
+    ]

Seeder/harvests/models.py

@@ -96,7 +96,8 @@ def get_custom_seeds(self):
         return set(self.custom_seeds.splitlines())
 
     def get_custom_sources_seeds(self):
-        seeds = Seed.archiving.filter(source__in=self.custom_sources.all())
+        seeds = Seed.objects.archiving().filter(
+            source__in=self.custom_sources.all())
         return set(seeds.values_list('url', flat=True)) - self.get_blacklisted()
 
     def get_seeds(self, blacklisted=None):
@@ -180,6 +181,11 @@ class Harvest(HarvestAbstractModel):
         null=True
     )
 
+    seeds_not_harvested = models.TextField(
+        _("Seeds not harvested"),
+        blank=True, null=True,
+    )
+
     auto_created = models.BooleanField(default=False)
 
     scheduled_on = DatePickerField(
@@ -228,7 +234,7 @@ def get_previously_harvested_seeds(self):
     def get_seeds_by_frequency(self, blacklisted=None):
         if not self.target_frequency:
             return set()
-        seeds = Seed.archiving.filter(
+        seeds = Seed.objects.archiving().filter(
             source__frequency__in=self.target_frequency)
         # Compute blacklisted only if not provided
         if blacklisted is None:
@@ -238,7 +244,7 @@ def get_seeds_by_frequency(self, blacklisted=None):
     def get_tests_seeds(self, blacklisted=None):
         if not self.tests:
             return set()
-        seeds = Seed.archiving.filter(
+        seeds = Seed.objects.filter(
             source__state=source_constants.STATE_TECHNICAL_REVIEW)
         # Compute blacklisted only if not provided
         if blacklisted is None:
@@ -250,7 +256,7 @@ def get_oneshot_seeds(self, blacklisted=None, previously_harvested=None):
         if not self.is_oneshot:
             return set()
         # Get all potential OneShot seeds
-        oneshot = Seed.archiving.filter(source__frequency=0)
+        oneshot = Seed.objects.archiving().filter(source__frequency=0)
         oneshot = set(oneshot.values_list('url', flat=True))
         # Get all harvested seeds up to this Harvest's scheduled date
         if previously_harvested is None:  # only if not supplied
@@ -265,7 +271,8 @@ def get_archiveit_seeds(self, blacklisted=None, previously_harvested=None):
         if not self.archive_it:
             return set()
         # Get all potential ArchiveIt seeds
-        archiveit = Seed.archiving.filter(source__frequency__in=[1, 2, 4])
+        archiveit = Seed.objects.archiving().filter(
+            source__frequency__in=[1, 2, 4, 6])
         archiveit = set(archiveit.values_list('url', flat=True))
         # Get all harvested seeds up to this Harvest's scheduled date
         if previously_harvested is None:  # only if not supplied
@@ -335,6 +342,7 @@ def freeze_seeds(self):
         Freezes the seeds to preserve them for later use
         """
         self.seeds_frozen = '\n'.join(self.get_seeds())
+        self.save()
 
     @property
     def is_oneshot(self):
@@ -437,7 +445,7 @@ class TopicCollection(HarvestAbstractModel, OrderedModel):
     )
 
     date_from = DatePickerField(_('Date from'), null=True)
-    date_to = DatePickerField(_('Date to'), null=True)
+    date_to = DatePickerField(_('Date to'), null=True, blank=True)
 
     def get_www_url(self):
         return reverse('www:collection_detail', kwargs={"slug": self.slug})

Seeder/harvests/templates/topic_collection.html

@@ -52,7 +52,7 @@ <h4>{% trans 'Info' %}</h4>
 
                 <tr>
                     <td>{% trans 'Harvested till' %}</td>
-                    <td>{{ object.date_to }}</td>
+                    <td>{{ object.date_to|default:"—" }}</td>
                 </tr>
 
                 {% if object.image %}