From 161a2fa421f1e8e8f70cde10054a579544aaa496 Mon Sep 17 00:00:00 2001 From: sgfost Date: Tue, 7 May 2024 15:27:54 -0700 Subject: [PATCH] feat: add model for tracking codebase git mirror * fix release ordering to sort by semantic version number rather than by string --- .../library/codebases/releases/retrieve.jinja | 2 +- ...7_codebasegitmirror_codebase_git_mirror.py | 59 ++++++++++++++++ django/library/models.py | 70 +++++++++++++++++-- 3 files changed, 123 insertions(+), 8 deletions(-) create mode 100644 django/library/migrations/0027_codebasegitmirror_codebase_git_mirror.py diff --git a/django/library/jinja2/library/codebases/releases/retrieve.jinja b/django/library/jinja2/library/codebases/releases/retrieve.jinja index 90ce5a053..2389c6dc7 100644 --- a/django/library/jinja2/library/codebases/releases/retrieve.jinja +++ b/django/library/jinja2/library/codebases/releases/retrieve.jinja @@ -502,7 +502,7 @@ - {% for related_release in codebase.ordered_releases(has_change_perm) %} + {% for related_release in codebase.ordered_releases(has_change_perm, asc=False) %} {{ related_release.version_number }} diff --git a/django/library/migrations/0027_codebasegitmirror_codebase_git_mirror.py b/django/library/migrations/0027_codebasegitmirror_codebase_git_mirror.py new file mode 100644 index 000000000..42ff08c3f --- /dev/null +++ b/django/library/migrations/0027_codebasegitmirror_codebase_git_mirror.py @@ -0,0 +1,59 @@ +# Generated by Django 4.2.11 on 2024-05-07 22:25 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("library", "0026_alter_codebasereleaseplatformtag_tag_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="CodebaseGitMirror", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("repository_name", models.CharField(max_length=100, unique=True)), + ( + "remote_url", + models.URLField( + blank=True, help_text="URL of mirrored remote repository" + ), + ), + ("last_local_update", models.DateTimeField(blank=True, null=True)), + ("last_remote_update", models.DateTimeField(blank=True, null=True)), + ( + "local_releases", + models.ManyToManyField( + related_name="+", to="library.codebaserelease" + ), + ), + ( + "remote_releases", + models.ManyToManyField( + related_name="+", to="library.codebaserelease" + ), + ), + ], + ), + migrations.AddField( + model_name="codebase", + name="git_mirror", + field=models.OneToOneField( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="codebase", + to="library.codebasegitmirror", + ), + ), + ] diff --git a/django/library/models.py b/django/library/models.py index e09aad294..5ad25493f 100644 --- a/django/library/models.py +++ b/django/library/models.py @@ -5,6 +5,7 @@ import uuid from collections import OrderedDict from datetime import timedelta +from packaging.version import Version import semver from django.conf import settings @@ -535,6 +536,38 @@ def get_all_release_programming_languages(self, codebase): ).values_list("programming_languages__name", flat=True) +class CodebaseGitMirror(models.Model): + """ + model to keep track of the state of a git mirror for a codebase + """ + + repository_name = models.CharField(max_length=100, unique=True) + remote_url = models.URLField( + blank=True, + help_text=_("URL of mirrored remote repository"), + ) + # keep track of timestamp and releases that have been mirrored locally + last_local_update = models.DateTimeField(null=True, blank=True) + local_releases = models.ManyToManyField("CodebaseRelease", related_name="+") + # keep track of timestamp and releases that have been synced to the remote + last_remote_update = models.DateTimeField(null=True, blank=True) + remote_releases = models.ManyToManyField("CodebaseRelease", related_name="+") + + @property + def latest_local_release(self): + return max(self.local_releases.all(), key=lambda r: Version(r.version_number)) + + @property + def latest_remote_release(self): + return max(self.remote_releases.all(), key=lambda r: Version(r.version_number)) + + @property + def unmirrored_local_releases(self): + return self.codebase.ordered_releases().exclude( + id__in=self.local_releases.values_list("id", flat=True) + ) + + @add_to_comses_permission_whitelist class Codebase(index.Indexed, ModeratedContent, ClusterableModel): """ @@ -570,6 +603,13 @@ class Codebase(index.Indexed, ModeratedContent, ClusterableModel): on_delete=models.SET_NULL, ) + git_mirror = models.OneToOneField( + "CodebaseGitMirror", + null=True, + related_name="codebase", + on_delete=models.SET_NULL, + ) + repository_url = models.URLField( blank=True, help_text=_( @@ -718,6 +758,16 @@ def base_library_dir(self): def base_git_dir(self): return pathlib.Path(settings.REPOSITORY_ROOT, str(self.uuid)) + def create_git_mirror(self, repository_name, **kwargs): + if not self.git_mirror: + git_mirror = CodebaseGitMirror.objects.create( + repository_name=repository_name, + **kwargs, + ) + self.git_mirror = git_mirror + self.save() + return self.git_mirror + @property def codebase_contributors_redis_key(self): return f"codebase:contributors:{self.identifier}" @@ -807,13 +857,19 @@ def download_count(self): release__codebase__id=self.id ).count() - def ordered_releases(self, has_change_perm=False, **kwargs): - releases = self.releases.order_by("-version_number").filter(**kwargs) - return ( - releases - if has_change_perm - else releases.filter(status=CodebaseRelease.Status.PUBLISHED) - ) + def ordered_releases(self, has_change_perm=False, asc=True, **kwargs): + """ + list public releases (or all release if has_change_perm is True) in ascending (default) or descending order + """ + if has_change_perm: + releases = self.releases.filter(**kwargs) + else: + releases = self.releases.filter( + status=CodebaseRelease.Status.PUBLISHED, **kwargs + ) + releases_list = list(releases) + releases_list.sort(key=lambda r: Version(r.version_number), reverse=not asc) + return releases_list @classmethod def get_list_url(cls):