From 5ea1f7ce163b72a4eefc0b6caec6a767400e9fad Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Tue, 27 Apr 2021 17:30:46 -0400 Subject: [PATCH] Add support for automatic publishing and distributing fixes: #365 --- CHANGES/365.feature | 1 + docs/_scripts/autoupdate.sh | 4 + docs/_scripts/quickstart.sh | 2 + docs/workflows/publish.rst | 18 +++- .../0006_pythonrepository_autopublish.py | 18 ++++ pulp_python/app/models.py | 26 ++++++ pulp_python/app/serializers.py | 12 ++- pulp_python/app/tasks/publish.py | 3 +- .../tests/functional/api/test_auto_publish.py | 93 +++++++++++++++++++ 9 files changed, 172 insertions(+), 5 deletions(-) create mode 100644 CHANGES/365.feature create mode 100644 docs/_scripts/autoupdate.sh create mode 100644 pulp_python/app/migrations/0006_pythonrepository_autopublish.py create mode 100644 pulp_python/tests/functional/api/test_auto_publish.py diff --git a/CHANGES/365.feature b/CHANGES/365.feature new file mode 100644 index 000000000..d8490631c --- /dev/null +++ b/CHANGES/365.feature @@ -0,0 +1 @@ +Add support for automatic publishing and distributing. diff --git a/docs/_scripts/autoupdate.sh b/docs/_scripts/autoupdate.sh new file mode 100644 index 000000000..b242f0448 --- /dev/null +++ b/docs/_scripts/autoupdate.sh @@ -0,0 +1,4 @@ +# This configures the repository to produce new publications when a new version is created +pulp python repository update --name foo --autopublish +# This configures the distribution to be track the latest repository version for a given repository +pulp python distribution update --name foo --repository foo diff --git a/docs/_scripts/quickstart.sh b/docs/_scripts/quickstart.sh index a8e5ede56..1398fbff1 100644 --- a/docs/_scripts/quickstart.sh +++ b/docs/_scripts/quickstart.sh @@ -18,6 +18,8 @@ source sync.sh source publication.sh source distribution.sh +# TODO: Test autoupdate when CLI PR is merged +# source autoupdate.sh source pip.sh source upload.sh diff --git a/docs/workflows/publish.rst b/docs/workflows/publish.rst index b1191be2d..71dc7e931 100644 --- a/docs/workflows/publish.rst +++ b/docs/workflows/publish.rst @@ -6,8 +6,8 @@ Publish and Host This section assumes that you have a repository with content in it. To do this, see the :doc:`sync` or :doc:`upload` documentation. -Create a Publication --------------------- +Create a Publication (manually) +------------------------------- Kick off a publish task by creating a new publication. The publish task will generate all the metadata that ``pip`` needs to install packages (although it will need to be hosted through a @@ -48,6 +48,20 @@ Response:: "publication": "/pulp/api/v3/publications/python/pypi/a09111b1-6bce-43ac-aed7-2e8441c22704/" } +Automate Publication and Distribution +------------------------------------- + +With a little more initial setup, you can have publications and distributions for your repositories +updated automatically when new repository versions are created. + +.. literalinclude:: ../_scripts/autoupdate.sh + :language: bash + +.. warning:: + Support for automatic publication and distribution is provided as a tech preview in Pulp 3. + Functionality may not work or may be incomplete. Also, backwards compatibility when upgrading + is not guaranteed. + .. _using-distributions: Use the newly created distribution diff --git a/pulp_python/app/migrations/0006_pythonrepository_autopublish.py b/pulp_python/app/migrations/0006_pythonrepository_autopublish.py new file mode 100644 index 000000000..58315eff7 --- /dev/null +++ b/pulp_python/app/migrations/0006_pythonrepository_autopublish.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.20 on 2021-04-27 20:14 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('python', '0005_pythonpackagecontent_sha256'), + ] + + operations = [ + migrations.AddField( + model_name='pythonrepository', + name='autopublish', + field=models.BooleanField(default=False), + ), + ] diff --git a/pulp_python/app/models.py b/pulp_python/app/models.py index af65faedc..ac2a83b26 100644 --- a/pulp_python/app/models.py +++ b/pulp_python/app/models.py @@ -182,5 +182,31 @@ class PythonRepository(Repository): CONTENT_TYPES = [PythonPackageContent] REMOTE_TYPES = [PythonRemote] + autopublish = models.BooleanField(default=False) + class Meta: default_related_name = "%(app_label)s_%(model_name)s" + + def on_new_version(self, version): + """ + Called when new repository versions are created. + + Args: + version: The new repository version + """ + super().on_new_version(version) + + # avoid circular import issues + from pulp_python.app import tasks + + if self.autopublish: + publication = tasks.publish( + repository_version_pk=version.pk, + ) + + distributions = self.distributions.all() + + if publication and distributions: + for distribution in distributions: + distribution.publication = publication + distribution.save() diff --git a/pulp_python/app/serializers.py b/pulp_python/app/serializers.py index 187ac4079..ed6df81c5 100644 --- a/pulp_python/app/serializers.py +++ b/pulp_python/app/serializers.py @@ -19,15 +19,23 @@ class PythonRepositorySerializer(core_serializers.RepositorySerializer): Serializer for Python Repositories. """ + autopublish = serializers.BooleanField( + help_text=_( + "Whether to automatically create publications for new repository versions, " + "and update any distributions pointing to this repository." + ), + default=False, + required=False, + ) + class Meta: - fields = core_serializers.RepositorySerializer.Meta.fields + fields = core_serializers.RepositorySerializer.Meta.fields + ("autopublish",) model = python_models.PythonRepository class PythonDistributionSerializer(core_serializers.DistributionSerializer): """ Serializer for Pulp distributions for the Python type. - """ publication = core_serializers.DetailRelatedField( diff --git a/pulp_python/app/tasks/publish.py b/pulp_python/app/tasks/publish.py index 5e1581b6b..7dbc354cd 100644 --- a/pulp_python/app/tasks/publish.py +++ b/pulp_python/app/tasks/publish.py @@ -64,7 +64,8 @@ def publish(repository_version_pk): with python_models.PythonPublication.create(repository_version, pass_through=True) as pub: write_simple_api(pub) - log.info(_('Publication: {pk} created').format(pk=pub.pk)) + log.info(_('Publication: {pk} created').format(pk=pub.pk)) + return pub def write_simple_api(publication): diff --git a/pulp_python/tests/functional/api/test_auto_publish.py b/pulp_python/tests/functional/api/test_auto_publish.py new file mode 100644 index 000000000..1358125bf --- /dev/null +++ b/pulp_python/tests/functional/api/test_auto_publish.py @@ -0,0 +1,93 @@ +# coding=utf-8 +"""Tests automatic updating of publications and distributions.""" +import unittest + +from pulp_smash import config +from pulp_smash.pulp3.bindings import monitor_task +from pulp_smash.pulp3.utils import gen_repo, gen_distribution + +from pulp_python.tests.functional.utils import gen_python_client, gen_python_remote +from pulp_python.tests.functional.utils import set_up_module as setUpModule # noqa:F401 + +from pulpcore.client.pulp_python import ( + RepositoriesPythonApi, + RemotesPythonApi, + PublicationsPypiApi, + ContentPackagesApi, + DistributionsPypiApi, + RepositorySyncURL, +) + + +class AutoPublishDistributeTestCase(unittest.TestCase): + """Test auto-publish and auto-distribution""" + + @classmethod + def setUpClass(cls): + """Create class-wide variables.""" + cls.cfg = config.get_config() + cls.client = gen_python_client() + + cls.content_api = ContentPackagesApi(cls.client) + cls.repo_api = RepositoriesPythonApi(cls.client) + cls.remote_api = RemotesPythonApi(cls.client) + cls.publications_api = PublicationsPypiApi(cls.client) + cls.distributions_api = DistributionsPypiApi(cls.client) + + def setUp(self): + """Create remote, repo, publish settings, and distribution.""" + self.remote = self.remote_api.create(gen_python_remote(policy="immediate")) + self.repo = self.repo_api.create(gen_repo(autopublish=True)) + response = self.distributions_api.create(gen_distribution(repository=self.repo.pulp_href)) + distribution_href = monitor_task(response.task).created_resources[0] + self.distribution = self.distributions_api.read(distribution_href) + + def tearDown(self): + """Clean up.""" + self.repo_api.delete(self.repo.pulp_href) + self.remote_api.delete(self.remote.pulp_href) + self.distributions_api.delete(self.distribution.pulp_href) + + def test_01_sync(self): + """Assert that syncing the repository triggers auto-publish and auto-distribution.""" + self.assertEqual(self.publications_api.list().count, 0) + self.assertTrue(self.distribution.publication is None) + + # Sync the repository. + repository_sync_data = RepositorySyncURL(remote=self.remote.pulp_href) + sync_response = self.repo_api.sync(self.repo.pulp_href, repository_sync_data) + task = monitor_task(sync_response.task) + self.distribution = self.distributions_api.read(self.distribution.pulp_href) + + # Check that all the appropriate resources were created + self.assertGreater(len(task.created_resources), 1) + self.assertEqual(self.publications_api.list().count, 1) + self.assertTrue(self.distribution.publication is not None) + self.assertTrue(self.distribution.publication in task.created_resources) + + # Sync the repository again. Since there should be no new repository version, there + # should be no new publications or distributions either. + sync_response = self.repo_api.sync(self.repo.pulp_href, repository_sync_data) + task = monitor_task(sync_response.task) + + self.assertEqual(len(task.created_resources), 0) + self.assertEqual(self.publications_api.list().count, 1) + + def test_02_modify(self): + """Assert that modifying the repository triggers auto-publish and auto-distribution.""" + self.assertEqual(self.publications_api.list().count, 0) + self.assertTrue(self.distribution.publication is None) + + # Modify the repository by adding a coment unit + content = self.content_api.list().results[0].pulp_href + modify_response = self.repo_api.modify( + self.repo.pulp_href, {"add_content_units": [content]} + ) + task = monitor_task(modify_response.task) + self.distribution = self.distributions_api.read(self.distribution.pulp_href) + + # Check that all the appropriate resources were created + self.assertGreater(len(task.created_resources), 1) + self.assertEqual(self.publications_api.list().count, 1) + self.assertTrue(self.distribution.publication is not None) + self.assertTrue(self.distribution.publication in task.created_resources)