From e66c4c489d58daafd8218318f48778b825aa75dc Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Fri, 23 Apr 2021 11:40:16 -0400 Subject: [PATCH 1/3] Fixed over publishing and improved publication tests fixes: #347 fixes: #362 --- CHANGES/347.misc | 1 + CHANGES/362.bugfix | 1 + functest_requirements.txt | 1 + pulp_python/app/tasks/publish.py | 4 +- .../functional/api/test_crud_publications.py | 313 ++++++++++++++++-- pulp_python/tests/functional/constants.py | 21 ++ 6 files changed, 320 insertions(+), 21 deletions(-) create mode 100644 CHANGES/347.misc create mode 100644 CHANGES/362.bugfix diff --git a/CHANGES/347.misc b/CHANGES/347.misc new file mode 100644 index 000000000..0d43fa782 --- /dev/null +++ b/CHANGES/347.misc @@ -0,0 +1 @@ +Improved functional tests for publications diff --git a/CHANGES/362.bugfix b/CHANGES/362.bugfix new file mode 100644 index 000000000..27ab4cd99 --- /dev/null +++ b/CHANGES/362.bugfix @@ -0,0 +1 @@ +Fixed publications publishing more content than was in the repository diff --git a/functest_requirements.txt b/functest_requirements.txt index 11db5441a..843f289b6 100644 --- a/functest_requirements.txt +++ b/functest_requirements.txt @@ -1,2 +1,3 @@ git+https://github.com/pulp/pulp-smash.git#egg=pulp-smash pytest +lxml diff --git a/pulp_python/app/tasks/publish.py b/pulp_python/app/tasks/publish.py index b8bcb234e..94bb49069 100644 --- a/pulp_python/app/tasks/publish.py +++ b/pulp_python/app/tasks/publish.py @@ -117,7 +117,9 @@ def find_artifact(): project_dir = '{simple_dir}{name}/'.format(simple_dir=simple_dir, name=canonical_name) os.mkdir(project_dir) - packages = python_models.PythonPackageContent.objects.filter(name=name) + packages = python_models.PythonPackageContent.objects.filter( + name=name, pk__in=publication.repository_version.content + ) package_detail_data = [] for package in packages.iterator(): artifact_set = package.contentartifact_set.all() diff --git a/pulp_python/tests/functional/api/test_crud_publications.py b/pulp_python/tests/functional/api/test_crud_publications.py index c81ababfc..b34648576 100644 --- a/pulp_python/tests/functional/api/test_crud_publications.py +++ b/pulp_python/tests/functional/api/test_crud_publications.py @@ -1,13 +1,29 @@ # coding=utf-8 """Tests that publish python plugin repositories.""" +import random import unittest from random import choice from pulp_smash import config from pulp_smash.pulp3.bindings import monitor_task -from pulp_smash.pulp3.utils import gen_repo, get_content, get_versions, modify_repo +from pulp_smash.pulp3.utils import ( + gen_repo, + get_content, + get_versions, + modify_repo, + gen_distribution, +) +from pulp_smash.utils import http_get +from urllib.parse import urljoin -from pulp_python.tests.functional.constants import PYTHON_CONTENT_NAME +from pulp_python.tests.functional.constants import ( + PYTHON_CONTENT_NAME, + PULP_CONTENT_BASE_URL, + PYTHON_SM_PROJECT_SPECIFIER, + PYTHON_SM_FIXTURE_RELEASES, + PYTHON_EGG_FILENAME, + PYTHON_WHEEL_FILENAME, +) from pulp_python.tests.functional.utils import gen_python_client, gen_python_remote from pulp_python.tests.functional.utils import set_up_module as setUpModule # noqa:F401 @@ -17,8 +33,10 @@ RepositorySyncURL, RemotesPythonApi, PythonPythonPublication, + DistributionsPypiApi, ) from pulpcore.client.pulp_python.exceptions import ApiException +from lxml import html class PublishAnyRepoVersionTestCase(unittest.TestCase): @@ -116,27 +134,282 @@ def test_on_demand(self): remote_api = RemotesPythonApi(client) publications = PublicationsPypiApi(client) - body = gen_python_remote(policy="on_demand") - remote = remote_api.create(body) - self.addCleanup(remote_api.delete, remote.pulp_href) + repo, _, pub = create_workflow(repo_api, publications, remote_api=remote_api, + body={"policy": "on_demand"}, cleanup=self.addCleanup) - repo = repo_api.create(gen_repo()) - self.addCleanup(repo_api.delete, repo.pulp_href) + self.assertEqual(pub.repository_version, repo.latest_version_href) - repository_sync_data = RepositorySyncURL(remote=remote.pulp_href) - sync_response = repo_api.sync(repo.pulp_href, repository_sync_data) + +class ListPublications(unittest.TestCase): + """ + This tests that publications can be listed. + + This also tests that associated distributions show up when listing a publication. + + # TODO Make test RBAC specific when RBAC is implemented + """ + + @classmethod + def setUpClass(cls): + """Setup apis for all tests.""" + client = gen_python_client() + cls.repo_api = RepositoriesPythonApi(client) + cls.pub_api = PublicationsPypiApi(client) + cls.dis_api = DistributionsPypiApi(client) + + def test_multiple_list(self): + """Test listing multiple publications.""" + _, publication = create_workflow(self.repo_api, self.pub_api, cleanup=self.addCleanup) + _, publication2 = create_workflow(self.repo_api, self.pub_api, cleanup=self.addCleanup) + + publications = self.pub_api.list() + self.assertEqual(publications.results, [publication2, publication]) + + def test_none_list(self): + """Test listing publications when empty.""" + publications = self.pub_api.list() + self.assertEqual(publications.count, 0) + + def test_publication_distribution(self): + """Test that a publication properly lists it's connected distributions.""" + repo, pub, distro = create_workflow(self.repo_api, self.pub_api, + distro_api=self.dis_api, cleanup=self.addCleanup) + distro2 = create_distribution(self.dis_api, pub, cleanup=self.addCleanup) + + publication = self.pub_api.read(pub.pulp_href) + self.assertEqual(publication.distributions, [distro.pulp_href, distro2.pulp_href]) + + +class DeletePublications(unittest.TestCase): + """ + These tests are dedicated to making sure that deleting works correctly. + + Deleted publications should be removed from associated distributions + Deleted repositories should remove publications + """ + + @classmethod + def setUpClass(cls): + """Setup apis for all tests.""" + client = gen_python_client() + cls.repo_api = RepositoriesPythonApi(client) + cls.pub_api = PublicationsPypiApi(client) + cls.dis_api = DistributionsPypiApi(client) + + def test_basic_delete(self): + """Checks that deleting decreases publication count.""" + publications = self.pub_api.list() + current_length = publications.count + repo, publication = create_workflow(self.repo_api, self.pub_api) + self.addCleanup(self.repo_api.delete, repo.pulp_href) + + publications = self.pub_api.list() + self.assertEqual(publications.count, 1 + current_length) + + self.pub_api.delete(publication.pulp_href) + publications = self.pub_api.list() + self.assertEqual(publications.count, current_length) + + def test_distro_removed_on_delete(self): + """Test that distributions no longer point to a publication after deleted.""" + repo, publication, distro = create_workflow( + self.repo_api, self.pub_api, distro_api=self.dis_api + ) + self.addCleanup(self.repo_api.delete, repo.pulp_href) + self.addCleanup(self.dis_api.delete, distro.pulp_href) + + self.assertEqual(distro.publication, publication.pulp_href) + + self.pub_api.delete(publication.pulp_href) + distro = self.dis_api.read(distro.pulp_href) + + self.assertEqual(distro.publication, None) + + def test_repository_delete(self): + """Tests that deleting the repository of a publication deletes the publication.""" + repo, publication = create_workflow(self.repo_api, self.pub_api) + + monitor_task(self.repo_api.delete(repo.pulp_href).task) + with self.assertRaises(ApiException): + self.pub_api.read(publication.pulp_href) + + +class PublishedCorrectContent(unittest.TestCase): + """ + These tests ensure that publications only serve content currently in the repository version. + + This test targets the following issues: + + * `Pulp #362 `_ + """ + + @staticmethod + def ensure_simple(base_path, packages): + """ + Tests that the simple api at `base_path` matches the packages supplied. + + `packages`: dictionary of form {package_name: [release_filenames]} + + First check `/simple/index.html` has each package name, no more, no less + Second check `/simple/package_name/index.html` for each package exists + Third check each package's index has all their releases listed, no more, no less + + Returns tuple (`proper`: bool, `error_msg`: str) + + *Technically, if there was a bug, other packages' indexes could be posted, but not present + in the simple index and thus be accessible from the distribution, but if one can't see it + how would one know that it's there?* + """ + def explore_links(page_url, page_name, links_found): + legit_found_links = [] + page = html.fromstring(http_get(page_url)) + page_links = page.xpath("/html/body/a") + for link in page_links: + if link.text in links_found: + if links_found[link.text]: + msgs += f"\nDuplicate {page_name} name {link.text}" # noqa: F823 + links_found[link.text] = True + if link.get("href"): + legit_found_links.append(link.get("href")) + else: + msgs += f"\nFound {page_name} link without href {link.text}" # noqa: F823 + else: + msgs += f"\nFound extra {page_name} link {link.text}" # noqa: F823 + return legit_found_links + + packages_found = {name: False for name in packages.keys()} + releases_found = {name: False for releases in packages.values() for name in releases} + msgs = "" + simple_url = urljoin(PULP_CONTENT_BASE_URL, f"{base_path}/simple/") + found_release_links = explore_links(simple_url, "simple", packages_found) + dl_links = [] + for release_link in found_release_links: + dl_links += explore_links(urljoin(simple_url, release_link), "release", releases_found) + for dl_link in dl_links: + _, _, sha = dl_link.partition("#sha256=") + if len(sha) != 64: + msgs += f"\nRelease download link has bad sha256 {dl_link}" + msgs += "".join(map(lambda x: f"\nSimple link not found for {x}", + [name for name, val in packages_found.items() if not val])) + msgs += "".join(map(lambda x: f"\nReleases link not found for {x}", + [name for name, val in releases_found.items() if not val])) + return len(msgs) == 0, msgs + + @classmethod + def setUpClass(cls): + """Sets up the apis for the tests.""" + client = gen_python_client() + cls.repo_api = RepositoriesPythonApi(client) + cls.rem_api = RemotesPythonApi(client) + cls.pub_api = PublicationsPypiApi(client) + cls.dis_api = DistributionsPypiApi(client) + cls.releases = [PYTHON_EGG_FILENAME, PYTHON_WHEEL_FILENAME] + + def test_all_content_published(self): + """Publishes SM Project and ensures correctness of simple api.""" + body = {"includes": PYTHON_SM_PROJECT_SPECIFIER} + _, _, _, distro = create_workflow(self.repo_api, self.pub_api, self.rem_api, + body, self.dis_api, self.addCleanup) + + proper, msgs = self.ensure_simple(distro.base_path, PYTHON_SM_FIXTURE_RELEASES) + self.assertTrue(proper, msg=msgs) + + def test_removed_content_not_published(self): + """Ensure content removed from a repository doesn't get published again.""" + repo, _, _, distro = create_workflow(self.repo_api, self.pub_api, remote_api=self.rem_api, + distro_api=self.dis_api, cleanup=self.addCleanup) + + proper, msgs = self.ensure_simple(distro.base_path, {"shelf-reader": self.releases}) + self.assertTrue(proper, msg=msgs) + + cfg = config.get_config() + removed_content = random.choice(get_content(repo.to_dict())[PYTHON_CONTENT_NAME]) + modify_repo(cfg, repo.to_dict(), remove_units=[removed_content]) + + publication = create_publication(self.pub_api, repo, cleanup=self.addCleanup) + distro = create_distribution(self.dis_api, publication, cleanup=self.addCleanup) + + if removed_content["filename"] == PYTHON_WHEEL_FILENAME: + remaining_release = [PYTHON_EGG_FILENAME] + else: + remaining_release = [PYTHON_WHEEL_FILENAME] + proper, msgs = self.ensure_simple(distro.base_path, {"shelf-reader": remaining_release}) + + self.assertTrue(proper, msg=msgs) + + def test_new_content_is_published(self): + """Ensures added content is published with a new publication.""" + body = {"package_types": ["sdist"]} + repo, _, _, distro = create_workflow(self.repo_api, self.pub_api, self.rem_api, + body, self.dis_api, self.addCleanup) + + proper, m = self.ensure_simple(distro.base_path, {"shelf-reader": [PYTHON_EGG_FILENAME]}) + self.assertTrue(proper, msg=m) + + remote = create_remote(self.rem_api, cleanup=self.addCleanup) + repository_sync_data = RepositorySyncURL(remote.pulp_href) + sync_response = self.repo_api.sync(repo.pulp_href, repository_sync_data) monitor_task(sync_response.task) - repo = repo_api.read(repo.pulp_href) - publish_data = PythonPythonPublication(repository=repo.pulp_href) - publish_response = publications.create(publish_data) - created_resources = monitor_task(publish_response.task).created_resources - publication_href = created_resources[0] - self.addCleanup(publications.delete, publication_href) - publication = publications.read(publication_href) + publication = create_publication(self.pub_api, repo, cleanup=self.addCleanup) + distro = create_distribution(self.dis_api, publication, cleanup=self.addCleanup) + + proper, msgs = self.ensure_simple(distro.base_path, {"shelf-reader": self.releases}) + self.assertTrue(proper, msg=msgs) + + +def create_repository(repo_api, cleanup=None): + """Creates a new python repository.""" + repo = repo_api.create(gen_repo()) + if cleanup: + cleanup(repo_api.delete, repo.pulp_href) + return repo - self.assertEqual(publication.repository_version, repo.latest_version_href) - # TODO TEST LIST - # TODO TEST PARTIAL AND FULL UPDATE - # TODO Maybe test DELETE +def create_remote(remote_api, body={}, cleanup=None): + """Creates a new python remote.""" + remote = remote_api.create(gen_python_remote(**body)) + if cleanup: + cleanup(remote_api.delete, remote.pulp_href) + return remote + + +def create_publication(pub_api, repository, cleanup=None): + """Creates a new python publication.""" + publish_data = PythonPythonPublication(repository=repository.pulp_href) + publish_response = pub_api.create(publish_data) + publication = pub_api.read(monitor_task(publish_response.task).created_resources[0]) + if cleanup: + cleanup(pub_api.delete, publication.pulp_href) + return publication + + +def create_distribution(dis_api, publication, cleanup=None): + """Creates a new python distribution.""" + dis_body = gen_distribution(publication=publication.pulp_href) + distro_response = dis_api.create(dis_body) + distro = dis_api.read(monitor_task(distro_response.task).created_resources[0]) + if cleanup: + cleanup(dis_api.delete, distro.pulp_href) + return distro + + +def create_workflow(repo_api, pub_api, remote_api=None, body={}, distro_api=None, cleanup=None): + """Creates repository, publication, and potentially remote and distribution if specified.""" + created_objects = [] + repo = create_repository(repo_api, cleanup=cleanup) + created_objects.append(repo) + if remote_api: + remote = create_remote(remote_api, body=body, cleanup=cleanup) + repository_sync_data = RepositorySyncURL(remote=remote.pulp_href) + sync_response = repo_api.sync(repo.pulp_href, repository_sync_data) + monitor_task(sync_response.task) + repo = repo_api.read(repo.pulp_href) + created_objects[0] = repo + created_objects.append(remote) + publication = create_publication(pub_api, repo, cleanup=cleanup) + created_objects.append(publication) + if distro_api: + distro = create_distribution(distro_api, publication, cleanup=cleanup) + created_objects.append(distro) + return created_objects diff --git a/pulp_python/tests/functional/constants.py b/pulp_python/tests/functional/constants.py index 4753e5cab..697423939 100644 --- a/pulp_python/tests/functional/constants.py +++ b/pulp_python/tests/functional/constants.py @@ -28,6 +28,10 @@ PYTHON_REPO_PATH = urljoin(BASE_REPO_PATH, "python/python/") +PULP_CONTENT_HOST_BASE_URL = config.get_config().get_content_host_base_url() + +PULP_CONTENT_BASE_URL = urljoin(PULP_CONTENT_HOST_BASE_URL, "pulp/content/") + # Specifier for testing empty syncs, or no excludes PYTHON_EMPTY_PROJECT_SPECIFIER = [] @@ -86,6 +90,23 @@ ] PYTHON_SM_PACKAGE_COUNT = 13 PYTHON_SM_FIXTURE_SUMMARY = {PYTHON_CONTENT_NAME: PYTHON_SM_PACKAGE_COUNT} +PYTHON_SM_FIXTURE_RELEASES = { + "aiohttp": ["aiohttp-3.3.0.tar.gz", "aiohttp-3.2.1.tar.gz", "aiohttp-3.2.0.tar.gz"], + "Django": [ + "Django-1.10.4.tar.gz", + "Django-1.10.4-py2.py3-none-any.whl", + "Django-1.10.3.tar.gz", + "Django-1.10.3-py2.py3-none-any.whl", + "Django-1.10.2.tar.gz", + "Django-1.10.2-py2.py3-none-any.whl", + "Django-1.10.1.tar.gz", + "Django-1.10.1-py2.py3-none-any.whl", + ], + "celery": [ + "celery-4.2.0-py2.py3-none-any.whl", + "celery-4.1.1-py2.py3-none-any.whl", + ], +} PYTHON_MD_PROJECT_SPECIFIER = [ "shelf-reader", # matches 2 From 759591ebd3992c9ec523d1338944bc4a0809b11d Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Sat, 24 Apr 2021 09:53:55 -0400 Subject: [PATCH 2/3] Greatly increased publication speed [noissue] --- pulp_python/app/tasks/publish.py | 78 +++++++++++++++++++------------- 1 file changed, 47 insertions(+), 31 deletions(-) diff --git a/pulp_python/app/tasks/publish.py b/pulp_python/app/tasks/publish.py index 94bb49069..6710e068e 100644 --- a/pulp_python/app/tasks/publish.py +++ b/pulp_python/app/tasks/publish.py @@ -61,7 +61,7 @@ def publish(repository_version_pk): )) with tempfile.TemporaryDirectory("."): - with python_models.PythonPublication.create(repository_version) as publication: + with python_models.PythonPublication.create(repository_version, True) as publication: write_simple_api(publication) log.info(_('Publication: {pk} created').format(pk=publication.pk)) @@ -107,41 +107,41 @@ def write_simple_api(publication): ) index_metadata.save() - def find_artifact(): - _art = content_artifact.artifact - if not _art: - _art = models.RemoteArtifact.objects.filter(content_artifact=content_artifact).first() - return _art + if len(index_names) == 0: + return - for (name, canonical_name) in index_names: - project_dir = '{simple_dir}{name}/'.format(simple_dir=simple_dir, name=canonical_name) - os.mkdir(project_dir) - - packages = python_models.PythonPackageContent.objects.filter( - name=name, pk__in=publication.repository_version.content + releases = ( + python_models.PythonPackageContent.objects.filter( + pk__in=publication.repository_version.content ) - package_detail_data = [] - for package in packages.iterator(): - artifact_set = package.contentartifact_set.all() - for content_artifact in artifact_set: - artifact = find_artifact() - published_artifact = models.PublishedArtifact( - relative_path=content_artifact.relative_path, - publication=publication, - content_artifact=content_artifact - ) - published_artifact.save() - - checksum = artifact.sha256 - path = "../../{}".format(package.filename) - package_detail_data.append((package.filename, path, checksum)) - - metadata_relative_path = '{project_dir}index.html'.format(project_dir=project_dir) + .values("name", "filename", "contentartifact", "_artifacts__sha256") + .order_by("name") + ) + release_content_artifacts = ( + python_models.PythonPackageContent.objects.filter( + pk__in=publication.repository_version.content + ) + .values_list("contentartifact", flat=True) + ) + remote_artifacts = ( + models.RemoteArtifact.objects.filter( + content_artifact__in=release_content_artifacts + ) + .values_list("content_artifact", "sha256").iterator() + ) + # This can grow to 4 million elements if fully PyPI synced + checksums = {ca: sha for ca, sha in remote_artifacts} + + def write_project_page(): + name = index_names[ind][1] + project_dir = f'{simple_dir}{name}/' + os.mkdir(project_dir) + metadata_relative_path = f'{project_dir}index.html' with open(metadata_relative_path, 'w') as simple_metadata: context = Context({ 'project_name': name, - 'project_packages': package_detail_data + 'project_packages': package_releases }) template = Template(simple_detail_template) simple_metadata.write(template.render(context)) @@ -151,4 +151,20 @@ def find_artifact(): publication=publication, file=File(open(metadata_relative_path, 'rb')) ) - project_metadata.save() + project_metadata.save() # change to bulk create when multi-table supported + + ind = 0 + current_name = index_names[ind][0] + package_releases = [] + for release in releases.iterator(): + if release['name'] != current_name: + write_project_page() + package_releases = [] + ind += 1 + current_name = index_names[ind][0] + content_artifact = release['contentartifact'] + relative_path = release['filename'] + path = f"../../{release['filename']}" + checksum = release['_artifacts__sha256'] or checksums[content_artifact] + package_releases.append((relative_path, path, checksum)) + write_project_page() # Write the final project's page From 4cad081634c53a0cb2c87e78585624720def8f5b Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Mon, 26 Apr 2021 12:49:25 -0400 Subject: [PATCH 3/3] Added sha256 to PythonPackageContent [noissue] --- .../0005_pythonpackagecontent_sha256.py | 41 +++++++++ pulp_python/app/models.py | 1 + pulp_python/app/tasks/publish.py | 92 +++++++++---------- pulp_python/app/tasks/sync.py | 2 +- pulp_python/app/utils.py | 2 +- .../functional/api/test_crud_publications.py | 64 ++++++++++--- pulp_python/tests/functional/constants.py | 21 +++++ 7 files changed, 161 insertions(+), 62 deletions(-) create mode 100644 pulp_python/app/migrations/0005_pythonpackagecontent_sha256.py diff --git a/pulp_python/app/migrations/0005_pythonpackagecontent_sha256.py b/pulp_python/app/migrations/0005_pythonpackagecontent_sha256.py new file mode 100644 index 000000000..e8d7df7e5 --- /dev/null +++ b/pulp_python/app/migrations/0005_pythonpackagecontent_sha256.py @@ -0,0 +1,41 @@ +# Generated by Django 2.2.20 on 2021-04-26 16:28 + +from django.db import migrations, models, transaction + + +def add_sha256_to_current_models(apps, schema_editor): + """Adds the sha256 to current PythonPackageContent models.""" + PythonPackageContent = apps.get_model('python', 'PythonPackageContent') + RemoteArtifact = apps.get_model('core', 'RemoteArtifact') + package_bulk = [] + for python_package in PythonPackageContent.objects.only("pk", "sha256").iterator(): + content_artifact = python_package.contentartifact_set.first() + if content_artifact.artifact: + artifact = content_artifact.artifact + else: + artifact = RemoteArtifact.objects.filter(content_artifact=content_artifact).first() + python_package.sha256 = artifact.sha256 + package_bulk.append(python_package) + if len(package_bulk) == 100000: + with transaction.atomic(): + PythonPackageContent.objects.bulk_update(package_bulk, ["sha256",]) + package_bulk = [] + with transaction.atomic(): + PythonPackageContent.objects.bulk_update(package_bulk, ["sha256",]) + + +class Migration(migrations.Migration): + + dependencies = [ + ('python', '0004_DATA_swap_distribution_model'), + ] + + operations = [ + migrations.AddField( + model_name='pythonpackagecontent', + name='sha256', + field=models.CharField(max_length=64, default=''), + preserve_default=False, + ), + migrations.RunPython(add_sha256_to_current_models, migrations.RunPython.noop) + ] diff --git a/pulp_python/app/models.py b/pulp_python/app/models.py index 7f19450f6..af65faedc 100644 --- a/pulp_python/app/models.py +++ b/pulp_python/app/models.py @@ -93,6 +93,7 @@ class PythonPackageContent(Content): packagetype = models.TextField(choices=PACKAGE_TYPES) name = models.TextField() version = models.TextField() + sha256 = models.CharField(max_length=64) # Optional metadata python_version = models.TextField() metadata_version = models.TextField() diff --git a/pulp_python/app/tasks/publish.py b/pulp_python/app/tasks/publish.py index 6710e068e..5e1581b6b 100644 --- a/pulp_python/app/tasks/publish.py +++ b/pulp_python/app/tasks/publish.py @@ -61,10 +61,10 @@ def publish(repository_version_pk): )) with tempfile.TemporaryDirectory("."): - with python_models.PythonPublication.create(repository_version, True) as publication: - write_simple_api(publication) + with python_models.PythonPublication.create(repository_version, pass_through=True) as pub: + write_simple_api(pub) - log.info(_('Publication: {pk} created').format(pk=publication.pk)) + log.info(_('Publication: {pk} created').format(pk=pub.pk)) def write_simple_api(publication): @@ -110,61 +110,55 @@ def write_simple_api(publication): if len(index_names) == 0: return - releases = ( - python_models.PythonPackageContent.objects.filter( - pk__in=publication.repository_version.content - ) - .values("name", "filename", "contentartifact", "_artifacts__sha256") - .order_by("name") - ) - release_content_artifacts = ( - python_models.PythonPackageContent.objects.filter( - pk__in=publication.repository_version.content - ) - .values_list("contentartifact", flat=True) - ) - remote_artifacts = ( - models.RemoteArtifact.objects.filter( - content_artifact__in=release_content_artifacts - ) - .values_list("content_artifact", "sha256").iterator() + packages = python_models.PythonPackageContent.objects.filter( + pk__in=publication.repository_version.content ) - # This can grow to 4 million elements if fully PyPI synced - checksums = {ca: sha for ca, sha in remote_artifacts} - - def write_project_page(): - name = index_names[ind][1] - project_dir = f'{simple_dir}{name}/' - os.mkdir(project_dir) - metadata_relative_path = f'{project_dir}index.html' - - with open(metadata_relative_path, 'w') as simple_metadata: - context = Context({ - 'project_name': name, - 'project_packages': package_releases - }) - template = Template(simple_detail_template) - simple_metadata.write(template.render(context)) - - project_metadata = models.PublishedMetadata.create_from_file( - relative_path=metadata_relative_path, - publication=publication, - file=File(open(metadata_relative_path, 'rb')) - ) - project_metadata.save() # change to bulk create when multi-table supported + releases = packages.order_by("name").values("name", "filename", "sha256") ind = 0 current_name = index_names[ind][0] package_releases = [] for release in releases.iterator(): if release['name'] != current_name: - write_project_page() + write_project_page( + name=index_names[ind][1], + simple_dir=simple_dir, + package_releases=package_releases, + publication=publication + ) package_releases = [] ind += 1 current_name = index_names[ind][0] - content_artifact = release['contentartifact'] relative_path = release['filename'] - path = f"../../{release['filename']}" - checksum = release['_artifacts__sha256'] or checksums[content_artifact] + path = f"../../{relative_path}" + checksum = release['sha256'] package_releases.append((relative_path, path, checksum)) - write_project_page() # Write the final project's page + # Write the final project's page + write_project_page( + name=index_names[ind][1], + simple_dir=simple_dir, + package_releases=package_releases, + publication=publication + ) + + +def write_project_page(name, simple_dir, package_releases, publication): + """Writes a project's simple page.""" + project_dir = f'{simple_dir}{name}/' + os.mkdir(project_dir) + metadata_relative_path = f'{project_dir}index.html' + + with open(metadata_relative_path, 'w') as simple_metadata: + context = Context({ + 'project_name': name, + 'project_packages': package_releases + }) + template = Template(simple_detail_template) + simple_metadata.write(template.render(context)) + + project_metadata = models.PublishedMetadata.create_from_file( + relative_path=metadata_relative_path, + publication=publication, + file=File(open(metadata_relative_path, 'rb')) + ) + project_metadata.save() # change to bulk create when multi-table supported diff --git a/pulp_python/app/tasks/sync.py b/pulp_python/app/tasks/sync.py index d23f36acf..3114a3a56 100644 --- a/pulp_python/app/tasks/sync.py +++ b/pulp_python/app/tasks/sync.py @@ -215,7 +215,7 @@ async def create_content(self, pkg): entry = parse_metadata(pkg.info, version, package) url = entry.pop("url") - artifact = Artifact(sha256=entry.pop("sha256_digest")) + artifact = Artifact(sha256=entry["sha256"]) package = PythonPackageContent(**entry) da = DeclarativeArtifact( diff --git a/pulp_python/app/utils.py b/pulp_python/app/utils.py index f031b48f2..8718d0fa6 100644 --- a/pulp_python/app/utils.py +++ b/pulp_python/app/utils.py @@ -67,7 +67,7 @@ def parse_metadata(project, version, distribution): package['packagetype'] = distribution.get('packagetype') or "" package['version'] = version package['url'] = distribution.get('url') or "" - package['sha256_digest'] = distribution.get('digests', {}).get('sha256') or "" + package['sha256'] = distribution.get('digests', {}).get('sha256') or "" package['python_version'] = distribution.get('python_version') or "" package.update(parse_project_metadata(project)) diff --git a/pulp_python/tests/functional/api/test_crud_publications.py b/pulp_python/tests/functional/api/test_crud_publications.py index b34648576..31e3a853d 100644 --- a/pulp_python/tests/functional/api/test_crud_publications.py +++ b/pulp_python/tests/functional/api/test_crud_publications.py @@ -21,6 +21,7 @@ PULP_CONTENT_BASE_URL, PYTHON_SM_PROJECT_SPECIFIER, PYTHON_SM_FIXTURE_RELEASES, + PYTHON_SM_FIXTURE_CHECKSUMS, PYTHON_EGG_FILENAME, PYTHON_WHEEL_FILENAME, ) @@ -113,14 +114,22 @@ def test_all(self): publications.create(body) -class PublishOnDemandContent(unittest.TestCase): - """Test whether a 'on_demand' synced repository can be published. +class PublishDifferentPolicyContent(unittest.TestCase): + """Test whether a 'on_demand', 'immediate', or 'streamed' synced repository can be published. This test targets the following issues: * `Pulp #7128 `_ """ + @classmethod + def setUpClass(cls): + """Sets up APIs used by tests.""" + client = gen_python_client() + cls.repo_api = RepositoriesPythonApi(client) + cls.remote_api = RemotesPythonApi(client) + cls.pub_api = PublicationsPypiApi(client) + def test_on_demand(self): """Test whether a particular repository version can be published. @@ -129,16 +138,44 @@ def test_on_demand(self): 3. Sync 4. Publish repository """ - client = gen_python_client() - repo_api = RepositoriesPythonApi(client) - remote_api = RemotesPythonApi(client) - publications = PublicationsPypiApi(client) - - repo, _, pub = create_workflow(repo_api, publications, remote_api=remote_api, + repo, _, pub = create_workflow(self.repo_api, self.pub_api, remote_api=self.remote_api, body={"policy": "on_demand"}, cleanup=self.addCleanup) self.assertEqual(pub.repository_version, repo.latest_version_href) + def test_immediate(self): + """Test if immediate synced content can be published.""" + repo, _, pub = create_workflow(self.repo_api, self.pub_api, remote_api=self.remote_api, + body={"policy": "immediate"}, cleanup=self.addCleanup) + + self.assertEqual(pub.repository_version, repo.latest_version_href) + + def test_streamed(self): + """Test if streamed synced content can be published.""" + repo, _, pub = create_workflow(self.repo_api, self.pub_api, remote_api=self.remote_api, + body={"policy": "streamed"}, cleanup=self.addCleanup) + + self.assertEqual(pub.repository_version, repo.latest_version_href) + + def test_mixed(self): + """Test if repository with mixed synced content can be published.""" + # Sync on demand content + body = {"includes": PYTHON_SM_PROJECT_SPECIFIER} + repo, _, pub = create_workflow(self.repo_api, self.pub_api, remote_api=self.remote_api, + body=body, cleanup=self.addCleanup) + + self.assertEqual(pub.repository_version, repo.latest_version_href) + # Add immediate content + body = {"policy": "immediate"} + remote = create_remote(self.remote_api, body=body, cleanup=self.addCleanup) + repository_sync_data = RepositorySyncURL(remote=remote.pulp_href) + sync_response = self.repo_api.sync(repo.pulp_href, repository_sync_data) + monitor_task(sync_response.task) + repo = self.repo_api.read(repo.pulp_href) + pub = create_publication(self.pub_api, repo, cleanup=self.addCleanup) + + self.assertEqual(pub.repository_version, repo.latest_version_href) + class ListPublications(unittest.TestCase): """ @@ -244,7 +281,7 @@ class PublishedCorrectContent(unittest.TestCase): """ @staticmethod - def ensure_simple(base_path, packages): + def ensure_simple(base_path, packages, sha_digests=None): """ Tests that the simple api at `base_path` matches the packages supplied. @@ -286,9 +323,13 @@ def explore_links(page_url, page_name, links_found): for release_link in found_release_links: dl_links += explore_links(urljoin(simple_url, release_link), "release", releases_found) for dl_link in dl_links: - _, _, sha = dl_link.partition("#sha256=") + package_link, _, sha = dl_link.partition("#sha256=") if len(sha) != 64: msgs += f"\nRelease download link has bad sha256 {dl_link}" + if sha_digests: + package = package_link.split("/")[-1] + if sha_digests[package] != sha: + msgs += f"\nRelease has bad sha256 attached to it {package}" msgs += "".join(map(lambda x: f"\nSimple link not found for {x}", [name for name, val in packages_found.items() if not val])) msgs += "".join(map(lambda x: f"\nReleases link not found for {x}", @@ -311,7 +352,8 @@ def test_all_content_published(self): _, _, _, distro = create_workflow(self.repo_api, self.pub_api, self.rem_api, body, self.dis_api, self.addCleanup) - proper, msgs = self.ensure_simple(distro.base_path, PYTHON_SM_FIXTURE_RELEASES) + proper, msgs = self.ensure_simple(distro.base_path, PYTHON_SM_FIXTURE_RELEASES, + sha_digests=PYTHON_SM_FIXTURE_CHECKSUMS) self.assertTrue(proper, msg=msgs) def test_removed_content_not_published(self): diff --git a/pulp_python/tests/functional/constants.py b/pulp_python/tests/functional/constants.py index 697423939..5c71e2995 100644 --- a/pulp_python/tests/functional/constants.py +++ b/pulp_python/tests/functional/constants.py @@ -107,6 +107,27 @@ "celery-4.1.1-py2.py3-none-any.whl", ], } +PYTHON_SM_FIXTURE_CHECKSUMS = { + "aiohttp-3.3.0.tar.gz": "3128d3ef7b575dbb272cdacd4d4c9a7cf67b18899e96260d55ae3a5782d886e7", + "aiohttp-3.2.1.tar.gz": "1b95d53f8dac13898f0a3e4af76f6f36d540fbfaefc4f4c9f43e436fa0e53d22", + "aiohttp-3.2.0.tar.gz": "1be3903fe6a36d20492e74efb326522dd4702bf32b45ffc7acbc0fb34ab240a6", + "Django-1.10.4.tar.gz": "fff7f062e510d812badde7cfc57745b7779edb4d209b2bc5ea8d954c22305c2b", + "Django-1.10.4-py2.py3-none-any.whl": + "a8e1a552205cda15023c39ecf17f7e525e96c5b0142e7879e8bd0c445351f2cc", + "Django-1.10.3.tar.gz": "6f92f08dee8a1bd7680e098a91bf5acd08b5cdfe74137f695b60fd79f4478c30", + "Django-1.10.3-py2.py3-none-any.whl": + "94426cc28d8721fbf13c333053f08d32427671a4ca7986f7030fc82bdf9c88c1", + "Django-1.10.2.tar.gz": "e127f12a0bfb34843b6e8c82f91e26fff6445a7ca91d222c0794174cf97cbce1", + "Django-1.10.2-py2.py3-none-any.whl": + "4d48ab8e84a7c8b2bc4b2f4f199bc3a8bfcc9cbdbc29e355ac5c44a501d73a1a", + "Django-1.10.1.tar.gz": "d6e6c5b25cb67f46afd7c82f536529b11981183423dad8932e15bce93d1a24f3", + "Django-1.10.1-py2.py3-none-any.whl": + "3d689905cd0635bbb33b87f9a5df7ca70a3db206faae4ec58cda5e7f5f47050d", + "celery-4.2.0-py2.py3-none-any.whl": + "2082cbd82effa8ac8a8a58977d70bb203a9f362817e3b66f4578117b9f93d8a9", + "celery-4.1.1-py2.py3-none-any.whl": + "6fc4678d1692af97e137b2a9f1c04efd8e7e2fb7134c5c5ad60738cdd927762f", +} PYTHON_MD_PROJECT_SPECIFIER = [ "shelf-reader", # matches 2