Add migration command for channel versioned content databases

contentcuration/contentcuration/management/commands/ensure_versioned_databse_exists.py

@@ -0,0 +1,48 @@
+"""
+Management command to ensure all published channels have versioned content databases.
+
+This command iterates through all published channels and ensures they have a versioned
+content database for their most recent version. For old channels that were published
+before versioned databases were introduced, this copies the unversioned database
+to create the versioned one.
+"""
+import logging
+
+from django.core.management.base import BaseCommand
+
+from contentcuration.models import Channel
+from contentcuration.utils.publish import ensure_versioned_database_exists
+
+logging.basicConfig()
+logger = logging.getLogger("command")
+
+
+class Command(BaseCommand):
+    help = "Ensure all published channels have versioned content databases."
+
+    def handle(self, *args, **options):
+        published_channels = Channel.objects.filter(main_tree_published=True)
+
+        processed_count = 0
+        error_count = 0
+
+        for channel in published_channels.iterator():
+            try:
+                if channel.version > 0:
+                    ensure_versioned_database_exists(channel.id, channel.version)
+                    processed_count += 1
+                    if processed_count % 100 == 0:
+                        logger.info(f"Processed {processed_count} channels...")
+                else:
+                    logger.warning(
+                        f"Channel {channel.id} has main_tree.published=Truebut version=0, skipping."
+                    )
+            except Exception as e:
+                error_count += 1
+                logger.error(
+                    f"Error ensuring versioned database for channel {channel.id}: {e}"
+                )
+
+        logger.info(
+            f"Completed: processed {processed_count} channels, {error_count} errors"
+        )

contentcuration/contentcuration/models.py

@@ -2856,9 +2856,6 @@ class CommunityLibrarySubmission(models.Model):
     internal_notes = models.TextField(blank=True, null=True)
 
     def save(self, *args, **kwargs):
-        # Not a top-level import to avoid circular import issues
-        from contentcuration.tasks import ensure_versioned_database_exists_task
-
         # Validate on save that the submission author is an editor of the channel
         # and that the version is not greater than the current channel version.
         # These cannot be expressed as constraints because traversing
@@ -2888,14 +2885,6 @@ def save(self, *args, **kwargs):
             )
 
         if self.pk is None:
-            # When creating a new submission, ensure the channel has a versioned database
-            # (it might not have if the channel was published before versioned databases
-            # were introduced).
-            ensure_versioned_database_exists_task.fetch_or_enqueue(
-                user=self.author,
-                channel_id=self.channel.id,
-                channel_version=self.channel.version,
-            )
             # Create a ChannelVersion and token for this submission
             channel_version, _ = ChannelVersion.objects.get_or_create(
                 channel=self.channel, version=self.channel_version

contentcuration/contentcuration/tasks.py

@@ -19,7 +19,6 @@
 from contentcuration.utils.csv_writer import write_user_csv
 from contentcuration.utils.nodes import calculate_resource_size
 from contentcuration.utils.nodes import generate_diff
-from contentcuration.utils.publish import ensure_versioned_database_exists
 from contentcuration.viewsets.user import AdminUserFilter
 
 
@@ -160,8 +159,3 @@ def sendcustomemails_task(subject, message, query):
             text,
             settings.DEFAULT_FROM_EMAIL,
         )
-
-
-@app.task(name="ensure_versioned_database_exists_task")
-def ensure_versioned_database_exists_task(channel_id, channel_version):
-    ensure_versioned_database_exists(channel_id, channel_version)

contentcuration/contentcuration/tests/test_models.py

@@ -555,18 +555,14 @@ def test_make_content_id_unique(self):
         self.assertNotEqual(copied_node_old_content_id, copied_node.content_id)
 
 
-@mock.patch(
-    "contentcuration.tasks.ensure_versioned_database_exists_task.fetch_or_enqueue",
-    return_value=None,
-)
 class CommunityLibrarySubmissionTestCase(
     EagerTasksTestMixin, PermissionQuerysetTestCase
 ):
     @property
     def base_queryset(self):
         return CommunityLibrarySubmission.objects.all()
 
-    def test_create_submission(self, mock_ensure_db_exists_task_fetch_or_enqueue):
+    def test_create_submission(self):
         # Smoke test
         channel = testdata.channel()
         author = testdata.user()
@@ -589,85 +585,51 @@ def test_create_submission(self, mock_ensure_db_exists_task_fetch_or_enqueue):
         submission.full_clean()
         submission.save()
 
-    def test_save__author_not_editor(self, mock_ensure_db_exists):
+    def test_save__author_not_editor(self):
         submission = testdata.community_library_submission()
         user = testdata.user("some@email.com")
         submission.author = user
         with self.assertRaises(ValidationError):
             submission.save()
 
-    def test_save__nonpositive_channel_version(
-        self, mock_ensure_db_exists_task_fetch_or_enqueue
-    ):
+    def test_save__nonpositive_channel_version(self):
         submission = testdata.community_library_submission()
         submission.channel_version = 0
         with self.assertRaises(ValidationError):
             submission.save()
 
-    def test_save__matching_channel_version(
-        self, mock_ensure_db_exists_task_fetch_or_enqueue
-    ):
+    def test_save__matching_channel_version(self):
         submission = testdata.community_library_submission()
         submission.channel.version = 5
         submission.channel.save()
         submission.channel_version = 5
         submission.save()
 
-    def test_save__impossibly_high_channel_version(
-        self, mock_ensure_db_exists_task_fetch_or_enqueue
-    ):
+    def test_save__impossibly_high_channel_version(self):
         submission = testdata.community_library_submission()
         submission.channel.version = 5
         submission.channel.save()
         submission.channel_version = 6
         with self.assertRaises(ValidationError):
             submission.save()
 
-    def test_save__ensure_versioned_database_exists_on_create(
-        self, mock_ensure_db_exists_task_fetch_or_enqueue
-    ):
-        submission = testdata.community_library_submission()
-
-        mock_ensure_db_exists_task_fetch_or_enqueue.assert_called_once_with(
-            user=submission.author,
-            channel_id=submission.channel.id,
-            channel_version=submission.channel.version,
-        )
-
-    def test_save__dont_ensure_versioned_database_exists_on_update(
-        self, mock_ensure_db_exists_task_fetch_or_enqueue
-    ):
-        submission = testdata.community_library_submission()
-        mock_ensure_db_exists_task_fetch_or_enqueue.reset_mock()
-
-        submission.description = "Updated description"
-        submission.save()
-
-        mock_ensure_db_exists_task_fetch_or_enqueue.assert_not_called()
-
-    def test_filter_view_queryset__anonymous(
-        self, mock_ensure_db_exists_task_fetch_or_enqueue
-    ):
+    def test_filter_view_queryset__anonymous(self):
         _ = testdata.community_library_submission()
 
         queryset = CommunityLibrarySubmission.filter_view_queryset(
             self.base_queryset, user=self.anonymous_user
         )
         self.assertFalse(queryset.exists())
 
-    def test_filter_view_queryset__forbidden_user(
-        self, mock_ensure_db_exists_task_fetch_or_enqueue
-    ):
+    def test_filter_view_queryset__forbidden_user(self):
         _ = testdata.community_library_submission()
 
         queryset = CommunityLibrarySubmission.filter_view_queryset(
             self.base_queryset, user=self.forbidden_user
         )
         self.assertFalse(queryset.exists())
 
-    def test_filter_view_queryset__channel_editor(
-        self, mock_ensure_db_exists_task_fetch_or_enqueue
-    ):
+    def test_filter_view_queryset__channel_editor(self):
         submission_a = testdata.community_library_submission()
         submission_b = testdata.community_library_submission()
 
@@ -681,39 +643,31 @@ def test_filter_view_queryset__channel_editor(
         self.assertQuerysetContains(queryset, pk=submission_a.id)
         self.assertQuerysetDoesNotContain(queryset, pk=submission_b.id)
 
-    def test_filter_view_queryset__admin(
-        self, mock_ensure_db_exists_task_fetch_or_enqueue
-    ):
+    def test_filter_view_queryset__admin(self):
         submission_a = testdata.community_library_submission()
 
         queryset = CommunityLibrarySubmission.filter_view_queryset(
             self.base_queryset, user=self.admin_user
         )
         self.assertQuerysetContains(queryset, pk=submission_a.id)
 
-    def test_filter_edit_queryset__anonymous(
-        self, mock_ensure_db_exists_task_fetch_or_enqueue
-    ):
+    def test_filter_edit_queryset__anonymous(self):
         _ = testdata.community_library_submission()
 
         queryset = CommunityLibrarySubmission.filter_edit_queryset(
             self.base_queryset, user=self.anonymous_user
         )
         self.assertFalse(queryset.exists())
 
-    def test_filter_edit_queryset__forbidden_user(
-        self, mock_ensure_db_exists_task_fetch_or_enqueue
-    ):
+    def test_filter_edit_queryset__forbidden_user(self):
         _ = testdata.community_library_submission()
 
         queryset = CommunityLibrarySubmission.filter_edit_queryset(
             self.base_queryset, user=self.forbidden_user
         )
         self.assertFalse(queryset.exists())
 
-    def test_filter_edit_queryset__channel_editor(
-        self, mock_ensure_db_exists_task_fetch_or_enqueue
-    ):
+    def test_filter_edit_queryset__channel_editor(self):
         submission = testdata.community_library_submission()
 
         user = testdata.user()
@@ -725,9 +679,7 @@ def test_filter_edit_queryset__channel_editor(
         )
         self.assertFalse(queryset.exists())
 
-    def test_filter_edit_queryset__author(
-        self, mock_ensure_db_exists_task_fetch_or_enqueue
-    ):
+    def test_filter_edit_queryset__author(self):
         submission_a = testdata.community_library_submission()
         submission_b = testdata.community_library_submission()
 
@@ -737,17 +689,15 @@ def test_filter_edit_queryset__author(
         self.assertQuerysetContains(queryset, pk=submission_a.id)
         self.assertQuerysetDoesNotContain(queryset, pk=submission_b.id)
 
-    def test_filter_edit_queryset__admin(
-        self, mock_ensure_db_exists_task_fetch_or_enqueue
-    ):
+    def test_filter_edit_queryset__admin(self):
         submission_a = testdata.community_library_submission()
 
         queryset = CommunityLibrarySubmission.filter_edit_queryset(
             self.base_queryset, user=self.admin_user
         )
         self.assertQuerysetContains(queryset, pk=submission_a.id)
 
-    def test_mark_live(self, mock_ensure_db_exists_task_fetch_or_enqueue):
+    def test_mark_live(self):
         submission_a = testdata.community_library_submission()
         submission_b = testdata.community_library_submission()
 
@@ -783,9 +733,7 @@ def test_mark_live(self, mock_ensure_db_exists_task_fetch_or_enqueue):
             community_library_submission.STATUS_LIVE,
         )
 
-    def test_cannot_create_multiple_submissions_same_channel_same_version(
-        self, mock_ensure_db_exists_task_fetch_or_enqueue
-    ):
+    def test_cannot_create_multiple_submissions_same_channel_same_version(self):
         from django.db import IntegrityError, transaction
 
         channel = testdata.channel()
@@ -825,9 +773,7 @@ def test_cannot_create_multiple_submissions_same_channel_same_version(
         self.assertEqual(submission1.channel, channel)
         self.assertEqual(submission1.channel_version, 1)
 
-    def test_can_create_submission_for_new_version_when_previous_pending(
-        self, mock_ensure_db_exists_task_fetch_or_enqueue
-    ):
+    def test_can_create_submission_for_new_version_when_previous_pending(self):
         channel = testdata.channel()
         author = testdata.user()
         channel.editors.add(author)