From 9cec8f01231f11046f16f1b88264022502d8eb12 Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Tue, 18 May 2021 18:12:49 -0400 Subject: [PATCH] Add PyPI endpoints to Pulp API fixes: #376 --- CHANGES/376.feature | 1 + MANIFEST.in | 3 +- docs/_scripts/pip.sh | 4 +- docs/workflows/publish.rst | 12 +- pulp_python/app/pypi/__init__.py | 0 pulp_python/app/pypi/serializers.py | 24 +++ pulp_python/app/pypi/views.py | 159 ++++++++++++++++++ pulp_python/app/serializers.py | 5 + pulp_python/app/urls.py | 24 +++ .../app/webserver_snippets/__init__.py | 0 .../app/webserver_snippets/apache.conf | 2 + pulp_python/app/webserver_snippets/nginx.conf | 9 + .../functional/api/test_download_content.py | 134 ++------------- .../tests/functional/api/test_pypi_apis.py | 159 ++++++++++++++++++ pulp_python/tests/functional/constants.py | 2 + 15 files changed, 406 insertions(+), 132 deletions(-) create mode 100644 CHANGES/376.feature create mode 100644 pulp_python/app/pypi/__init__.py create mode 100644 pulp_python/app/pypi/serializers.py create mode 100644 pulp_python/app/pypi/views.py create mode 100644 pulp_python/app/urls.py create mode 100644 pulp_python/app/webserver_snippets/__init__.py create mode 100644 pulp_python/app/webserver_snippets/apache.conf create mode 100644 pulp_python/app/webserver_snippets/nginx.conf create mode 100644 pulp_python/tests/functional/api/test_pypi_apis.py diff --git a/CHANGES/376.feature b/CHANGES/376.feature new file mode 100644 index 000000000..0823c4c7d --- /dev/null +++ b/CHANGES/376.feature @@ -0,0 +1 @@ +PyPI endpoints are now available at ``/pypi/{base_path}/`` diff --git a/MANIFEST.in b/MANIFEST.in index f0c827c5c..aca0d5946 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -6,4 +6,5 @@ include COMMITMENT include COPYRIGHT include functest_requirements.txt include test_requirements.txt -include unittest_requirements.txt \ No newline at end of file +include unittest_requirements.txt +include pulp_python/app/webserver_snippets/* diff --git a/docs/_scripts/pip.sh b/docs/_scripts/pip.sh index 0d81a67d1..69e68600d 100644 --- a/docs/_scripts/pip.sh +++ b/docs/_scripts/pip.sh @@ -1,5 +1,5 @@ -echo 'pip install --trusted-host pulp -i $CONTENT_ADDR/pulp/content/foo/simple/ shelf-reader' -pip install --trusted-host pulp -i $CONTENT_ADDR/pulp/content/foo/simple/ shelf-reader +echo 'pip install --trusted-host pulp -i $BASE_ADDR/pypi/foo/simple/ shelf-reader' +pip install --trusted-host pulp -i $BASE_ADDR/pypi/foo/simple/ shelf-reader echo "is shelf reader installed?" pip list | grep shelf-reader diff --git a/docs/workflows/publish.rst b/docs/workflows/publish.rst index 71dc7e931..c2464f6fc 100644 --- a/docs/workflows/publish.rst +++ b/docs/workflows/publish.rst @@ -30,7 +30,7 @@ Host a Publication (Create a Distribution) -------------------------------------------- To host a publication, (which makes it consumable by ``pip``), users create a distribution which -will serve the associated publication at ``/pulp/content/`` +will serve the associated publication at ``/pypi//`` .. literalinclude:: ../_scripts/distribution.sh :language: bash @@ -41,7 +41,7 @@ Response:: "pulp_href": "/pulp/api/v3/distributions/python/pypi/4839c056-6f2b-46b9-ac5f-88eb8a7739a5/", "pulp_created": "2021-03-09T04:36:48.289737Z", "base_path": "foo", - "base_url": "https://pulp3-source-fedora33.localhost.example.com/pulp/content/foo/", + "base_url": "/pypi/foo/", "content_guard": null, "pulp_labels": {}, "name": "foo", @@ -69,12 +69,12 @@ Use the newly created distribution The metadata and packages can now be retrieved from the distribution:: -$ http $CONTENT_ADDR/pulp/content/foo/simple/ -$ http $CONTENT_ADDR/pulp/content/foo/simple/shelf-reader/ +$ http $BASE_ADDR/pypi/foo/simple/ +$ http $BASE_ADDR/pypi/foo/simple/shelf-reader/ The content is also pip installable:: -$ pip install --trusted-host localhost -i $CONTENT_ADDR/pulp/content/foo/simple/ shelf-reader +$ pip install --trusted-host localhost -i $BASE_ADDR/pypi/foo/simple/ shelf-reader If you don't want to specify the distribution path every time, you can modify your ``pip.conf`` file. See the `pip docs `_ for more @@ -85,7 +85,7 @@ $ cat pip.conf .. code:: [global] - index-url = http://localhost:24816/pulp/content/foo/simple/ + index-url = http://localhost:24817/pypi/foo/simple/ The above configuration informs ``pip`` to install from ``pulp``:: diff --git a/pulp_python/app/pypi/__init__.py b/pulp_python/app/pypi/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pulp_python/app/pypi/serializers.py b/pulp_python/app/pypi/serializers.py new file mode 100644 index 000000000..856f5145c --- /dev/null +++ b/pulp_python/app/pypi/serializers.py @@ -0,0 +1,24 @@ +from gettext import gettext as _ + +from rest_framework import serializers + + +class SummarySerializer(serializers.Serializer): + """ + A Serializer for summary information of an index. + """ + + projects = serializers.IntegerField(help_text=_("Number of Python projects in index")) + releases = serializers.IntegerField(help_text=_("Number of Python distributions in index")) + files = serializers.IntegerField(help_text=_("Number of files for all distributions in index")) + + +class PackageMetadataSerializer(serializers.Serializer): + """ + A Serializer for a package's metadata. + """ + + last_serial = serializers.IntegerField(help_text=_("Cache value from last PyPI sync")) + info = serializers.JSONField(help_text=_("Core metadata of the package")) + releases = serializers.JSONField(help_text=_("List of all the releases of the package")) + urls = serializers.JSONField() diff --git a/pulp_python/app/pypi/views.py b/pulp_python/app/pypi/views.py new file mode 100644 index 000000000..c1c28dcac --- /dev/null +++ b/pulp_python/app/pypi/views.py @@ -0,0 +1,159 @@ +import logging + +from rest_framework.viewsets import ViewSet +from rest_framework.response import Response +from django.core.exceptions import ObjectDoesNotExist +from django.shortcuts import redirect +from django.http.response import Http404, StreamingHttpResponse +from drf_spectacular.utils import extend_schema +from dynaconf import settings +from urllib.parse import urljoin +from pathlib import PurePath + +from pulp_python.app.models import ( + PythonDistribution, + PythonPackageContent, + PythonPublication, +) +from pulp_python.app.pypi.serializers import ( + SummarySerializer, + PackageMetadataSerializer, +) +from pulp_python.app.utils import ( + write_simple_index, + write_simple_detail, + python_content_to_json, + PYPI_LAST_SERIAL, + PYPI_SERIAL_CONSTANT, +) + +log = logging.getLogger(__name__) + +BASE_CONTENT_URL = urljoin(settings.CONTENT_ORIGIN, settings.CONTENT_PATH_PREFIX) + + +class PyPIMixin: + """Mixin to get index specific info.""" + + @staticmethod + def get_repository_version(distribution): + """Finds the repository version this distribution is serving.""" + pub = distribution.publication + rep = distribution.repository + if pub: + return pub.repository_version or pub.repository.latest_version() + elif rep: + return rep.latest_version() + else: + raise Http404("No repository associated with this index") + + @staticmethod + def get_distribution(path): + """Finds the distribution associated with this base_path.""" + distro_qs = PythonDistribution.objects.select_related( + "repository", "publication", "publication__repository_version" + ) + try: + return distro_qs.get(base_path=path) + except ObjectDoesNotExist: + raise Http404(f"No PythonDistribution found for base_path {path}") + + @staticmethod + def get_content(repository_version): + """Returns queryset of the content in this repository version.""" + return PythonPackageContent.objects.filter(pk__in=repository_version.content) + + def should_redirect(self, distro, repo_version=None): + """Checks if there is a publication the content app can serve.""" + if distro.publication: + return True + rv = repo_version or self.get_repository_version(distro) + return PythonPublication.objects.filter(repository_version=rv).exists() + + def get_drvc(self, path): + """Takes the base_path and returns the distribution, repository_version and content.""" + distro = self.get_distribution(path) + repo_ver = self.get_repository_version(distro) + content = self.get_content(repo_ver) + return distro, repo_ver, content + + +class SimpleView(ViewSet, PyPIMixin): + """View for the PyPI simple API.""" + + authentication_classes = [] + permission_classes = [] + + @extend_schema(summary="Get index simple page") + def list(self, request, path): + """Gets the simple api html page for the index.""" + distro, repo_version, content = self.get_drvc(path) + if self.should_redirect(distro, repo_version=repo_version): + return redirect(urljoin(BASE_CONTENT_URL, f'{path}/simple/')) + names = content.order_by('name').values_list('name', flat=True).distinct().iterator() + return StreamingHttpResponse(write_simple_index(names, streamed=True)) + + @extend_schema(summary="Get package simple page") + def retrieve(self, request, path, package): + """Retrieves the simple api html page for a package.""" + distro, repo_ver, content = self.get_drvc(path) + if self.should_redirect(distro, repo_version=repo_ver): + # Maybe this name needs to be normalized? + return redirect(urljoin(BASE_CONTENT_URL, f'{path}/simple/{package}/')) + packages = content.filter(name__iexact=package).values_list('filename', 'sha256').iterator() + detail_packages = ((f, urljoin(BASE_CONTENT_URL, f'{path}/{f}'), d) for f, d in packages) + return StreamingHttpResponse( + write_simple_detail(package, detail_packages, streamed=True) + ) + + +class MetadataView(ViewSet, PyPIMixin): + """View for the PyPI JSON metadata endpoint.""" + + authentication_classes = [] + permission_classes = [] + + @extend_schema(tags=["Pypi: Metadata"], + responses={200: PackageMetadataSerializer}, + summary="Get package metadata") + def retrieve(self, request, path, meta): + """ + Retrieves the package's core-metadata specified by + https://packaging.python.org/specifications/core-metadata/. + `meta` must be a path in form of `{package}/json/` or `{package}/{version}/json/` + """ + distro, repo_ver, content = self.get_drvc(path) + meta_path = PurePath(meta) + name = None + version = None + if meta_path.match("*/*/json"): + version = meta_path.parts[1] + name = meta_path.parts[0] + elif meta_path.match("*/json"): + name = meta_path.parts[0] + if name: + package_content = content.filter(name__iexact=name) + # TODO Change this value to the Repo's serial value when implemented + headers = {PYPI_LAST_SERIAL: str(PYPI_SERIAL_CONSTANT)} + json_body = python_content_to_json(path, package_content, version=version) + if json_body: + return Response(data=json_body, headers=headers) + return Response(status="404") + + +class PyPIView(ViewSet, PyPIMixin): + """View for base_url of distribution.""" + + authentication_classes = [] + permission_classes = [] + + @extend_schema(responses={200: SummarySerializer}, + summary="Get index summary") + def retrieve(self, request, path): + """Gets package summary stats of index.""" + distro, repo_ver, content = self.get_drvc(path) + files = content.count() + releases = content.distinct("name", "version").count() + projects = content.distinct("name").count() + data = {"projects": projects, "releases": releases, "files": files} + return Response(data=data) diff --git a/pulp_python/app/serializers.py b/pulp_python/app/serializers.py index 505ad3858..3336ee8dc 100644 --- a/pulp_python/app/serializers.py +++ b/pulp_python/app/serializers.py @@ -45,6 +45,11 @@ class PythonDistributionSerializer(core_serializers.DistributionSerializer): queryset=core_models.Publication.objects.exclude(complete=False), allow_null=True, ) + base_url = serializers.SerializerMethodField(read_only=True) + + def get_base_url(self, obj): + """Gets the base url.""" + return f"/pypi/{obj.base_path}/" def validate(self, data): """ diff --git a/pulp_python/app/urls.py b/pulp_python/app/urls.py new file mode 100644 index 000000000..8a5082560 --- /dev/null +++ b/pulp_python/app/urls.py @@ -0,0 +1,24 @@ +from django.urls import path + +from pulp_python.app.pypi.views import SimpleView, MetadataView, PyPIView + +PYPI_API_URL = 'pypi//' + +# TODO: Implement remaining PyPI endpoints +# path("project/", PackageProject.as_view()), # Endpoints to nicely see contents of index +# path("search/", PackageSearch.as_view()), + +urlpatterns = [ + path( + PYPI_API_URL + "pypi//", + MetadataView.as_view({"get": "retrieve"}), + name="pypi-metadata" + ), + path( + PYPI_API_URL + "simple//", + SimpleView.as_view({"get": "retrieve"}), + name="simple-package-detail" + ), + path(PYPI_API_URL + 'simple/', SimpleView.as_view({"get": "list"}), name="simple-detail"), + path(PYPI_API_URL, PyPIView.as_view({"get": "retrieve"}), name="pypi-detail"), +] diff --git a/pulp_python/app/webserver_snippets/__init__.py b/pulp_python/app/webserver_snippets/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pulp_python/app/webserver_snippets/apache.conf b/pulp_python/app/webserver_snippets/apache.conf new file mode 100644 index 000000000..ce3078e3e --- /dev/null +++ b/pulp_python/app/webserver_snippets/apache.conf @@ -0,0 +1,2 @@ +ProxyPass /pulp_python/pypi ${pulp-api}/pypi +ProxyPassReverse /pulp_python/pypi ${pulp-api}/pypi diff --git a/pulp_python/app/webserver_snippets/nginx.conf b/pulp_python/app/webserver_snippets/nginx.conf new file mode 100644 index 000000000..943005cc7 --- /dev/null +++ b/pulp_python/app/webserver_snippets/nginx.conf @@ -0,0 +1,9 @@ +location /pypi/ { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + # we don't want nginx trying to do something clever with + # redirects, we set the Host: header above already. + proxy_redirect off; + proxy_pass http://pulp-api; +} diff --git a/pulp_python/tests/functional/api/test_download_content.py b/pulp_python/tests/functional/api/test_download_content.py index e61841cfd..ecce748b0 100644 --- a/pulp_python/tests/functional/api/test_download_content.py +++ b/pulp_python/tests/functional/api/test_download_content.py @@ -1,37 +1,28 @@ # coding=utf-8 """Tests that verify download of content served by Pulp.""" import hashlib -import json import unittest -import requests from random import choice from urllib.parse import urljoin from pulp_smash import utils -from pulp_smash.pulp3.bindings import monitor_task from pulp_smash.pulp3.utils import ( download_content_unit, - gen_distribution, - gen_repo, get_content_summary, ) from pulp_python.tests.functional.constants import ( PYTHON_FIXTURE_URL, - SHELF_PYTHON_JSON, PYTHON_LG_FIXTURE_SUMMARY, PYTHON_LG_PROJECT_SPECIFIER, ) from pulp_python.tests.functional.utils import ( cfg, get_python_content_paths, - gen_python_remote, - publish, TestCaseUsingBindings, TestHelpersMixin, ) from pulp_python.tests.functional.utils import set_up_module as setUpModule # noqa:F401 -from pulpcore.client.pulp_python import RepositorySyncURL PYPI_LAST_SERIAL = "X-PYPI-LAST-SERIAL" PYPI_SERIAL_CONSTANT = 1000000000 @@ -92,7 +83,7 @@ def test_all(self): self.assertEqual(fixtures_hashes, pulp_hashes) -class PublishPyPiJSON(TestCaseUsingBindings): +class PublishPyPiJSON(TestCaseUsingBindings, TestHelpersMixin): """Test whether a distributed Python repository has a PyPi json endpoint a.k.a Can be consumed by another Pulp instance @@ -101,129 +92,26 @@ class PublishPyPiJSON(TestCaseUsingBindings): * `Pulp #2886 `_ """ - @classmethod - def setUp(cls): - """Sets up the repo before every test""" - cls.repo = cls.repo_api.create(gen_repo()) - - def test_pypi_json(self): - """Checks the basics 'pypi/{package_name}/json' endpoint - Steps: - 1. Create Repo and Remote to only sync shelf-reader - 2. Sync with immediate policy - 3. Publish and Distribute new Repo - 4. Access JSON endpoint and verify received JSON matches source - """ - self.addCleanup(self.repo_api.delete, self.repo.pulp_href) - body = gen_python_remote(includes=["shelf-reader"], policy="immediate") - self.sync_to_remote(body, create=True) - self.addCleanup(self.remote_api.delete, self.remote.pulp_href) - distro = self.gen_pub_dist() - rel_url = "pypi/shelf-reader/json" - package_json = download_content_unit(cfg, distro.to_dict(), rel_url) - package = json.loads(package_json) - self.assertEqual(SHELF_PYTHON_JSON["last_serial"], package["last_serial"]) - self.assertTrue(SHELF_PYTHON_JSON["info"].items() <= package["info"].items()) - self.assertEqual(len(SHELF_PYTHON_JSON["urls"]), len(package["urls"])) - self.assert_download_info(SHELF_PYTHON_JSON["urls"], package["urls"], - "Failed to match URLS") - self.assertTrue(SHELF_PYTHON_JSON["releases"].keys() <= package["releases"].keys()) - for version in SHELF_PYTHON_JSON["releases"].keys(): - self.assert_download_info(SHELF_PYTHON_JSON["releases"][version], - package["releases"][version], "Failed to match version") - - def assert_download_info(self, expected, received, message="Failed to match"): - """ - Each version has a list of dists of that version, but the lists might - not be in the same order, so check each dist of the second list - """ - for dist in expected: - dist = dict(dist) - matched = False - dist_items = dist.items() - for dist2 in received: - dist2 = dict(dist2) - dist2["digests"].pop("md5", "") - if dist_items <= dist2.items(): - matched = True - break - self.assertTrue(matched, message) - - def test_pypi_last_serial(self): - """ - Checks that the endpoint has the header PYPI_LAST_SERIAL and is set - TODO when serial field is added to Repo's, check this header against that - """ - self.addCleanup(self.repo_api.delete, self.repo.pulp_href) - body = gen_python_remote(includes=["shelf-reader"]) - self.sync_to_remote(body, create=True) - distro = self.gen_pub_dist().to_dict() - rel_url = "pypi/shelf-reader/json" - url_fragments = [ - cfg.get_content_host_base_url(), - "pulp/content", - distro["base_path"], - rel_url, - ] - unit_url = "/".join(url_fragments) - response = requests.get(unit_url) - self.assertIn(PYPI_LAST_SERIAL, response.headers) - self.assertEqual(response.headers[PYPI_LAST_SERIAL], str(PYPI_SERIAL_CONSTANT)) - @unittest.skip("Content can not be synced without https") def test_basic_pulp_to_pulp_sync(self): """ This test checks that the JSON endpoint is setup correctly to allow one Pulp instance to perform a basic sync from another Pulp instance """ - self.addCleanup(self.repo_api.delete, self.repo.pulp_href) - body = gen_python_remote(includes=PYTHON_LG_PROJECT_SPECIFIER, policy="on_demand", - prereleases=True) - self.sync_to_remote(body, create=True) - self.addCleanup(self.remote_api.delete, self.remote.pulp_href) - self.assertEqual(get_content_summary(self.repo.to_dict()), PYTHON_LG_FIXTURE_SUMMARY) - distro = self.gen_pub_dist().to_dict() + body = {"includes": PYTHON_LG_PROJECT_SPECIFIER, "prereleases": True} + remote = self._create_remote(**body) + repo = self._create_repo_and_sync_with_remote(remote) + pub = self._create_publication(repo) + distro = self._create_distribution_from_publication(pub) url_fragments = [ cfg.get_content_host_base_url(), "pulp/content", - distro["base_path"], + distro.base_path, "" ] unit_url = "/".join(url_fragments) - repo2 = self.repo_api.create(gen_repo()) - self.addCleanup(self.repo_api.delete, repo2.pulp_href) - body2 = gen_python_remote(url=unit_url, includes=PYTHON_LG_PROJECT_SPECIFIER, - policy="on_demand", prereleases=True) - self.repo = repo2 - self.sync_to_remote(body2, create=True) - self.assertEqual(get_content_summary(self.repo.to_dict()), PYTHON_LG_FIXTURE_SUMMARY) - self.addCleanup(self.remote_api.delete, self.remote.pulp_href) - - def sync_to_remote(self, body, create=False, mirror=False): - """Takes a body and creates/updates a remote object, then it performs a sync""" - if create: - self.remote = self.remote_api.create(body) - else: - remote_task = self.remote_api.partial_update(self.remote.pulp_href, body) - monitor_task(remote_task.task) - self.remote = self.remote_api.read(self.remote.pulp_href) - - repository_sync_data = RepositorySyncURL( - remote=self.remote.pulp_href, mirror=mirror - ) - sync_response = self.repo_api.sync(self.repo.pulp_href, repository_sync_data) - monitor_task(sync_response.task) - self.repo = self.repo_api.read(self.repo.pulp_href) - - def gen_pub_dist(self): - """Takes a repo and generates a publication and then distributes it""" - publication = publish(self.repo.to_dict()) - self.addCleanup(self.publications_api.delete, publication["pulp_href"]) - - body = gen_distribution() - body["publication"] = publication["pulp_href"] - response = self.distributions_api.create(body) - distro = self.distributions_api.read(monitor_task(response.task).created_resources[0]) - self.addCleanup(self.distributions_api.delete, distro.pulp_href) - return distro + body["url"] = unit_url + remote = self._create_remote(**body) + repo2 = self._create_repo_and_sync_with_remote(remote) + self.assertEqual(get_content_summary(repo2.to_dict()), PYTHON_LG_FIXTURE_SUMMARY) diff --git a/pulp_python/tests/functional/api/test_pypi_apis.py b/pulp_python/tests/functional/api/test_pypi_apis.py new file mode 100644 index 000000000..b215502cd --- /dev/null +++ b/pulp_python/tests/functional/api/test_pypi_apis.py @@ -0,0 +1,159 @@ +"""Tests all the PyPI apis available at `pulp_python/pypi/`.""" +import requests + +from urllib.parse import urljoin + +from pulp_python.tests.functional.constants import ( + PYTHON_SM_PROJECT_SPECIFIER, + PYTHON_SM_FIXTURE_RELEASES, + PYTHON_SM_FIXTURE_CHECKSUMS, + PYTHON_MD_PROJECT_SPECIFIER, + PYTHON_MD_PYPI_SUMMARY, + PULP_CONTENT_BASE_URL, + PULP_PYPI_BASE_URL, + SHELF_PYTHON_JSON +) + +from pulp_python.tests.functional.utils import ( + py_client as client, + ensure_simple, + TestCaseUsingBindings, + TestHelpersMixin, +) +from pulpcore.client.pulp_python import PypiApi + +PYPI_LAST_SERIAL = "X-PYPI-LAST-SERIAL" +PYPI_SERIAL_CONSTANT = 1000000000 +HOST = client.configuration.host +PYPI_HOST = urljoin(HOST, PULP_PYPI_BASE_URL) + + +class PyPISummaryTestCase(TestCaseUsingBindings, TestHelpersMixin): + """Tests the summary response of the base url of an index.""" + + @classmethod + def setUpClass(cls): + """Set up class variables.""" + super().setUpClass() + cls.pypi_api = PypiApi(client) + + def test_empty_index(self): + """Checks that summary stats are 0 when index is empty.""" + _, distro = self._create_empty_repo_and_distribution() + + summary = self.pypi_api.read(path=distro.base_path) + self.assertTrue(not any(summary.to_dict().values())) + + def test_live_index(self): + """Checks summary stats are correct for indexes pointing to repositories.""" + remote = self._create_remote(includes=PYTHON_MD_PROJECT_SPECIFIER) + repo = self._create_repo_and_sync_with_remote(remote) + distro = self._create_distribution_from_repo(repo) + + summary = self.pypi_api.read(path=distro.base_path) + self.assertDictEqual(summary.to_dict(), PYTHON_MD_PYPI_SUMMARY) + + def test_published_index(self): + """Checks summary stats are correct for indexes pointing to publications.""" + remote = self._create_remote(includes=PYTHON_MD_PROJECT_SPECIFIER) + repo = self._create_repo_and_sync_with_remote(remote) + pub = self._create_publication(repo) + distro = self._create_distribution_from_publication(pub) + + summary = self.pypi_api.read(path=distro.base_path) + self.assertDictEqual(summary.to_dict(), PYTHON_MD_PYPI_SUMMARY) + + +class PyPISimpleApi(TestCaseUsingBindings, TestHelpersMixin): + """Tests that the simple api is correct.""" + + def test_simple_redirect_with_publications(self): + """Checks that requests to `/simple/` get redirected when serving a publication.""" + remote = self._create_remote() + repo = self._create_repo_and_sync_with_remote(remote) + pub = self._create_publication(repo) + distro = self._create_distribution_from_publication(pub) + response = requests.get(urljoin(PYPI_HOST, f'{distro.base_path}/simple/')) + self.assertEqual( + response.url, str(urljoin(PULP_CONTENT_BASE_URL, f"{distro.base_path}/simple/")) + ) + + def test_simple_correctness_live(self): + """Checks that the simple api on live distributions are correct.""" + remote = self._create_remote(includes=PYTHON_SM_PROJECT_SPECIFIER) + repo = self._create_repo_and_sync_with_remote(remote) + distro = self._create_distribution_from_repo(repo) + proper, msgs = ensure_simple( + urljoin(PYPI_HOST, f'{distro.base_path}/simple/'), + PYTHON_SM_FIXTURE_RELEASES, + sha_digests=PYTHON_SM_FIXTURE_CHECKSUMS, + ) + self.assertTrue(proper, msg=msgs) + + +class PyPIPackageMetadata(TestCaseUsingBindings, TestHelpersMixin): + """Test whether a distributed Python repository has a PyPI json endpoint.""" + + def test_pypi_json(self): + """Checks the data of `pypi/{package_name}/json` endpoint.""" + remote = self._create_remote(policy="immediate") + repo = self._create_repo_and_sync_with_remote(remote) + distro = self._create_distribution_from_repo(repo) + rel_url = f"{distro.base_path}/pypi/shelf-reader/json" + response = requests.get(urljoin(PYPI_HOST, rel_url)) + self.assert_pypi_json(response.json()) + + def test_pypi_json_content_app(self): + """Checks that the pypi json endpoint of the content app still works. Needs Publication.""" + remote = self._create_remote(policy="immediate") + repo = self._create_repo_and_sync_with_remote(remote) + pub = self._create_publication(repo) + distro = self._create_distribution_from_publication(pub) + rel_url = f"{distro.base_path}/pypi/shelf-reader/json/" + response = requests.get(urljoin(PULP_CONTENT_BASE_URL, rel_url)) + self.assert_pypi_json(response.json()) + + def test_pypi_last_serial(self): + """ + Checks that the endpoint has the header PYPI_LAST_SERIAL and is set + TODO when serial field is added to Repo's, check this header against that + """ + remote = self._create_remote() + repo = self._create_repo_and_sync_with_remote(remote) + pub = self._create_publication(repo) + distro = self._create_distribution_from_publication(pub) + content_url = urljoin(PULP_CONTENT_BASE_URL, f"{distro.base_path}/pypi/shelf-reader/json") + pypi_url = urljoin(PYPI_HOST, f"{distro.base_path}/pypi/shelf-reader/json/") + for url in [content_url, pypi_url]: + response = requests.get(url) + self.assertIn(PYPI_LAST_SERIAL, response.headers, msg=url) + self.assertEqual(response.headers[PYPI_LAST_SERIAL], str(PYPI_SERIAL_CONSTANT), msg=url) + + def assert_pypi_json(self, package): + """Asserts that shelf-reader package json is correct.""" + self.assertEqual(SHELF_PYTHON_JSON["last_serial"], package["last_serial"]) + self.assertTrue(SHELF_PYTHON_JSON["info"].items() <= package["info"].items()) + self.assertEqual(len(SHELF_PYTHON_JSON["urls"]), len(package["urls"])) + self.assert_download_info(SHELF_PYTHON_JSON["urls"], package["urls"], + "Failed to match URLS") + self.assertTrue(SHELF_PYTHON_JSON["releases"].keys() <= package["releases"].keys()) + for version in SHELF_PYTHON_JSON["releases"].keys(): + self.assert_download_info(SHELF_PYTHON_JSON["releases"][version], + package["releases"][version], "Failed to match version") + + def assert_download_info(self, expected, received, message="Failed to match"): + """ + Each version has a list of dists of that version, but the lists might + not be in the same order, so check each dist of the second list + """ + for dist in expected: + dist = dict(dist) + matched = False + dist_items = dist.items() + for dist2 in received: + dist2 = dict(dist2) + dist2["digests"].pop("md5", "") + if dist_items <= dist2.items(): + matched = True + break + self.assertTrue(matched, message) diff --git a/pulp_python/tests/functional/constants.py b/pulp_python/tests/functional/constants.py index 5c71e2995..92804ecf3 100644 --- a/pulp_python/tests/functional/constants.py +++ b/pulp_python/tests/functional/constants.py @@ -32,6 +32,7 @@ PULP_CONTENT_BASE_URL = urljoin(PULP_CONTENT_HOST_BASE_URL, "pulp/content/") +PULP_PYPI_BASE_URL = "/pypi/" # Specifier for testing empty syncs, or no excludes PYTHON_EMPTY_PROJECT_SPECIFIER = [] @@ -137,6 +138,7 @@ ] PYTHON_MD_PACKAGE_COUNT = 26 PYTHON_MD_FIXTURE_SUMMARY = {PYTHON_CONTENT_NAME: PYTHON_MD_PACKAGE_COUNT} +PYTHON_MD_PYPI_SUMMARY = {"projects": 4, "releases": 17, "files": 26} PYTHON_LG_PROJECT_SPECIFIER = [ "aiohttp", # matches 7