Add support for automatic publishing and distributing

CHANGES/365.feature

@@ -0,0 +1 @@
+Add support for automatic publishing and distributing.

docs/_scripts/autoupdate.sh

@@ -0,0 +1,4 @@
+# This configures the repository to produce new publications when a new version is created
+pulp python repository update --name foo --autopublish
+# This configures the distribution to be track the latest repository version for a given repository
+pulp python distribution update --name foo --repository foo

docs/_scripts/quickstart.sh

@@ -18,6 +18,8 @@ source sync.sh
 
 source publication.sh
 source distribution.sh
+# TODO: Test autoupdate when CLI PR is merged
+# source autoupdate.sh
 source pip.sh
 
 source upload.sh

docs/workflows/publish.rst

@@ -6,8 +6,8 @@ Publish and Host
 This section assumes that you have a repository with content in it. To do this, see the
 :doc:`sync` or :doc:`upload` documentation.
 
-Create a Publication
---------------------
+Create a Publication (manually)
+-------------------------------
 
 Kick off a publish task by creating a new publication. The publish task will generate all the
 metadata that ``pip`` needs to install packages (although it will need to be hosted through a
@@ -48,6 +48,20 @@ Response::
       "publication": "/pulp/api/v3/publications/python/pypi/a09111b1-6bce-43ac-aed7-2e8441c22704/"
     }
 
+Automate Publication and Distribution
+-------------------------------------
+
+With a little more initial setup, you can have publications and distributions for your repositories
+updated automatically when new repository versions are created.
+
+.. literalinclude:: ../_scripts/autoupdate.sh
+    :language: bash
+
+.. warning::
+    Support for automatic publication and distribution is provided as a tech preview in Pulp 3.
+    Functionality may not work or may be incomplete. Also, backwards compatibility when upgrading
+    is not guaranteed.
+
 .. _using-distributions:
 
 Use the newly created distribution

pulp_python/app/migrations/0006_pythonrepository_autopublish.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.20 on 2021-04-27 20:14
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('python', '0005_pythonpackagecontent_sha256'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='pythonrepository',
+            name='autopublish',
+            field=models.BooleanField(default=False),
+        ),
+    ]

pulp_python/app/models.py

@@ -182,5 +182,31 @@ class PythonRepository(Repository):
     CONTENT_TYPES = [PythonPackageContent]
     REMOTE_TYPES = [PythonRemote]
 
+    autopublish = models.BooleanField(default=False)
+
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"
+
+    def on_new_version(self, version):
+        """
+        Called when new repository versions are created.
+
+        Args:
+            version: The new repository version
+        """
+        super().on_new_version(version)
+
+        # avoid circular import issues
+        from pulp_python.app import tasks
+
+        if self.autopublish:
+            publication = tasks.publish(
+                repository_version_pk=version.pk,
+            )
+
+            distributions = self.distributions.all()
+
+            if publication and distributions:
+                for distribution in distributions:
+                    distribution.publication = publication
+                    distribution.save()

pulp_python/app/serializers.py

@@ -19,15 +19,23 @@ class PythonRepositorySerializer(core_serializers.RepositorySerializer):
     Serializer for Python Repositories.
     """
 
+    autopublish = serializers.BooleanField(
+        help_text=_(
+            "Whether to automatically create publications for new repository versions, "
+            "and update any distributions pointing to this repository."
+        ),
+        default=False,
+        required=False,
+    )
+
     class Meta:
-        fields = core_serializers.RepositorySerializer.Meta.fields
+        fields = core_serializers.RepositorySerializer.Meta.fields + ("autopublish",)
         model = python_models.PythonRepository
 
 
 class PythonDistributionSerializer(core_serializers.DistributionSerializer):
     """
     Serializer for Pulp distributions for the Python type.
-
     """
 
     publication = core_serializers.DetailRelatedField(

pulp_python/app/tasks/publish.py

@@ -64,7 +64,8 @@ def publish(repository_version_pk):
         with python_models.PythonPublication.create(repository_version, pass_through=True) as pub:
             write_simple_api(pub)
 
-    log.info(_('Publication: {pk} created').format(pk=pub.pk))
+        log.info(_('Publication: {pk} created').format(pk=pub.pk))
+        return pub
 
 
 def write_simple_api(publication):

pulp_python/tests/functional/api/test_auto_publish.py

@@ -0,0 +1,93 @@
+# coding=utf-8
+"""Tests automatic updating of publications and distributions."""
+import unittest
+
+from pulp_smash import config
+from pulp_smash.pulp3.bindings import monitor_task
+from pulp_smash.pulp3.utils import gen_repo, gen_distribution
+
+from pulp_python.tests.functional.utils import gen_python_client, gen_python_remote
+from pulp_python.tests.functional.utils import set_up_module as setUpModule  # noqa:F401
+
+from pulpcore.client.pulp_python import (
+    RepositoriesPythonApi,
+    RemotesPythonApi,
+    PublicationsPypiApi,
+    ContentPackagesApi,
+    DistributionsPypiApi,
+    RepositorySyncURL,
+)
+
+
+class AutoPublishDistributeTestCase(unittest.TestCase):
+    """Test auto-publish and auto-distribution"""
+
+    @classmethod
+    def setUpClass(cls):
+        """Create class-wide variables."""
+        cls.cfg = config.get_config()
+        cls.client = gen_python_client()
+
+        cls.content_api = ContentPackagesApi(cls.client)
+        cls.repo_api = RepositoriesPythonApi(cls.client)
+        cls.remote_api = RemotesPythonApi(cls.client)
+        cls.publications_api = PublicationsPypiApi(cls.client)
+        cls.distributions_api = DistributionsPypiApi(cls.client)
+
+    def setUp(self):
+        """Create remote, repo, publish settings, and distribution."""
+        self.remote = self.remote_api.create(gen_python_remote(policy="immediate"))
+        self.repo = self.repo_api.create(gen_repo(autopublish=True))
+        response = self.distributions_api.create(gen_distribution(repository=self.repo.pulp_href))
+        distribution_href = monitor_task(response.task).created_resources[0]
+        self.distribution = self.distributions_api.read(distribution_href)
+
+    def tearDown(self):
+        """Clean up."""
+        self.repo_api.delete(self.repo.pulp_href)
+        self.remote_api.delete(self.remote.pulp_href)
+        self.distributions_api.delete(self.distribution.pulp_href)
+
+    def test_01_sync(self):
+        """Assert that syncing the repository triggers auto-publish and auto-distribution."""
+        self.assertEqual(self.publications_api.list().count, 0)
+        self.assertTrue(self.distribution.publication is None)
+
+        # Sync the repository.
+        repository_sync_data = RepositorySyncURL(remote=self.remote.pulp_href)
+        sync_response = self.repo_api.sync(self.repo.pulp_href, repository_sync_data)
+        task = monitor_task(sync_response.task)
+        self.distribution = self.distributions_api.read(self.distribution.pulp_href)
+
+        # Check that all the appropriate resources were created
+        self.assertGreater(len(task.created_resources), 1)
+        self.assertEqual(self.publications_api.list().count, 1)
+        self.assertTrue(self.distribution.publication is not None)
+        self.assertTrue(self.distribution.publication in task.created_resources)
+
+        # Sync the repository again. Since there should be no new repository version, there
+        # should be no new publications or distributions either.
+        sync_response = self.repo_api.sync(self.repo.pulp_href, repository_sync_data)
+        task = monitor_task(sync_response.task)
+
+        self.assertEqual(len(task.created_resources), 0)
+        self.assertEqual(self.publications_api.list().count, 1)
+
+    def test_02_modify(self):
+        """Assert that modifying the repository triggers auto-publish and auto-distribution."""
+        self.assertEqual(self.publications_api.list().count, 0)
+        self.assertTrue(self.distribution.publication is None)
+
+        # Modify the repository by adding a coment unit
+        content = self.content_api.list().results[0].pulp_href
+        modify_response = self.repo_api.modify(
+            self.repo.pulp_href, {"add_content_units": [content]}
+        )
+        task = monitor_task(modify_response.task)
+        self.distribution = self.distributions_api.read(self.distribution.pulp_href)
+
+        # Check that all the appropriate resources were created
+        self.assertGreater(len(task.created_resources), 1)
+        self.assertEqual(self.publications_api.list().count, 1)
+        self.assertTrue(self.distribution.publication is not None)
+        self.assertTrue(self.distribution.publication in task.created_resources)