Add Python endpoints to Pulp API

CHANGES/376.feature

@@ -0,0 +1 @@
+PyPI endpoints are now available at ``/pypi/{base_path}/``

MANIFEST.in

@@ -6,4 +6,5 @@ include COMMITMENT
 include COPYRIGHT
 include functest_requirements.txt
 include test_requirements.txt
-include unittest_requirements.txt
+include unittest_requirements.txt
+include pulp_python/app/webserver_snippets/*

docs/_scripts/pip.sh

@@ -1,5 +1,5 @@
-echo 'pip install --trusted-host pulp -i $CONTENT_ADDR/pulp/content/foo/simple/ shelf-reader'
-pip install --trusted-host pulp -i $CONTENT_ADDR/pulp/content/foo/simple/ shelf-reader
+echo 'pip install --trusted-host pulp -i $BASE_ADDR/pypi/foo/simple/ shelf-reader'
+pip install --trusted-host pulp -i $BASE_ADDR/pypi/foo/simple/ shelf-reader
 
 echo "is shelf reader installed?"
 pip list | grep shelf-reader

docs/workflows/publish.rst

@@ -30,7 +30,7 @@ Host a Publication (Create a Distribution)
 --------------------------------------------
 
 To host a publication, (which makes it consumable by ``pip``), users create a distribution which
-will serve the associated publication at ``/pulp/content/<distribution.base_path>``
+will serve the associated publication at ``/pypi/<distribution.base_path>/``
 
 .. literalinclude:: ../_scripts/distribution.sh
     :language: bash
@@ -41,7 +41,7 @@ Response::
       "pulp_href": "/pulp/api/v3/distributions/python/pypi/4839c056-6f2b-46b9-ac5f-88eb8a7739a5/",
       "pulp_created": "2021-03-09T04:36:48.289737Z",
       "base_path": "foo",
-      "base_url": "https://pulp3-source-fedora33.localhost.example.com/pulp/content/foo/",
+      "base_url": "/pypi/foo/",
       "content_guard": null,
       "pulp_labels": {},
       "name": "foo",
@@ -69,12 +69,12 @@ Use the newly created distribution
 
 The metadata and packages can now be retrieved from the distribution::
 
-$ http $CONTENT_ADDR/pulp/content/foo/simple/
-$ http $CONTENT_ADDR/pulp/content/foo/simple/shelf-reader/
+$ http $BASE_ADDR/pypi/foo/simple/
+$ http $BASE_ADDR/pypi/foo/simple/shelf-reader/
 
 The content is also pip installable::
 
-$ pip install --trusted-host localhost -i $CONTENT_ADDR/pulp/content/foo/simple/ shelf-reader
+$ pip install --trusted-host localhost -i $BASE_ADDR/pypi/foo/simple/ shelf-reader
 
 If you don't want to specify the distribution path every time, you can modify your ``pip.conf``
 file. See the `pip docs <https://pip.pypa.io/en/stable/user_guide/#configuration>`_ for more
@@ -85,7 +85,7 @@ $ cat pip.conf
 .. code::
 
   [global]
-  index-url = http://localhost:24816/pulp/content/foo/simple/
+  index-url = http://localhost:24817/pypi/foo/simple/
 
 The above configuration informs ``pip`` to install from ``pulp``::
 

pulp_python/app/pypi/serializers.py

@@ -0,0 +1,24 @@
+from gettext import gettext as _
+
+from rest_framework import serializers
+
+
+class SummarySerializer(serializers.Serializer):
+    """
+    A Serializer for summary information of an index.
+    """
+
+    projects = serializers.IntegerField(help_text=_("Number of Python projects in index"))
+    releases = serializers.IntegerField(help_text=_("Number of Python distributions in index"))
+    files = serializers.IntegerField(help_text=_("Number of files for all distributions in index"))
+
+
+class PackageMetadataSerializer(serializers.Serializer):
+    """
+    A Serializer for a package's metadata.
+    """
+
+    last_serial = serializers.IntegerField(help_text=_("Cache value from last PyPI sync"))
+    info = serializers.JSONField(help_text=_("Core metadata of the package"))
+    releases = serializers.JSONField(help_text=_("List of all the releases of the package"))
+    urls = serializers.JSONField()

pulp_python/app/pypi/views.py

@@ -0,0 +1,159 @@
+import logging
+
+from rest_framework.viewsets import ViewSet
+from rest_framework.response import Response
+from django.core.exceptions import ObjectDoesNotExist
+from django.shortcuts import redirect
+from django.http.response import Http404, StreamingHttpResponse
+from drf_spectacular.utils import extend_schema
+from dynaconf import settings
+from urllib.parse import urljoin
+from pathlib import PurePath
+
+from pulp_python.app.models import (
+    PythonDistribution,
+    PythonPackageContent,
+    PythonPublication,
+)
+from pulp_python.app.pypi.serializers import (
+    SummarySerializer,
+    PackageMetadataSerializer,
+)
+from pulp_python.app.utils import (
+    write_simple_index,
+    write_simple_detail,
+    python_content_to_json,
+    PYPI_LAST_SERIAL,
+    PYPI_SERIAL_CONSTANT,
+)
+
+log = logging.getLogger(__name__)
+
+BASE_CONTENT_URL = urljoin(settings.CONTENT_ORIGIN, settings.CONTENT_PATH_PREFIX)
+
+
+class PyPIMixin:
+    """Mixin to get index specific info."""
+
+    @staticmethod
+    def get_repository_version(distribution):
+        """Finds the repository version this distribution is serving."""
+        pub = distribution.publication
+        rep = distribution.repository
+        if pub:
+            return pub.repository_version or pub.repository.latest_version()
+        elif rep:
+            return rep.latest_version()
+        else:
+            raise Http404("No repository associated with this index")
+
+    @staticmethod
+    def get_distribution(path):
+        """Finds the distribution associated with this base_path."""
+        distro_qs = PythonDistribution.objects.select_related(
+            "repository", "publication", "publication__repository_version"
+        )
+        try:
+            return distro_qs.get(base_path=path)
+        except ObjectDoesNotExist:
+            raise Http404(f"No PythonDistribution found for base_path {path}")
+
+    @staticmethod
+    def get_content(repository_version):
+        """Returns queryset of the content in this repository version."""
+        return PythonPackageContent.objects.filter(pk__in=repository_version.content)
+
+    def should_redirect(self, distro, repo_version=None):
+        """Checks if there is a publication the content app can serve."""
+        if distro.publication:
+            return True
+        rv = repo_version or self.get_repository_version(distro)
+        return PythonPublication.objects.filter(repository_version=rv).exists()
+
+    def get_drvc(self, path):
+        """Takes the base_path and returns the distribution, repository_version and content."""
+        distro = self.get_distribution(path)
+        repo_ver = self.get_repository_version(distro)
+        content = self.get_content(repo_ver)
+        return distro, repo_ver, content
+
+
+class SimpleView(ViewSet, PyPIMixin):
+    """View for the PyPI simple API."""
+
+    authentication_classes = []
+    permission_classes = []
+
+    @extend_schema(summary="Get index simple page")
+    def list(self, request, path):
+        """Gets the simple api html page for the index."""
+        distro, repo_version, content = self.get_drvc(path)
+        if self.should_redirect(distro, repo_version=repo_version):
+            return redirect(urljoin(BASE_CONTENT_URL, f'{path}/simple/'))
+        names = content.order_by('name').values_list('name', flat=True).distinct().iterator()
+        return StreamingHttpResponse(write_simple_index(names, streamed=True))
+
+    @extend_schema(summary="Get package simple page")
+    def retrieve(self, request, path, package):
+        """Retrieves the simple api html page for a package."""
+        distro, repo_ver, content = self.get_drvc(path)
+        if self.should_redirect(distro, repo_version=repo_ver):
+            # Maybe this name needs to be normalized?
+            return redirect(urljoin(BASE_CONTENT_URL, f'{path}/simple/{package}/'))
+        packages = content.filter(name__iexact=package).values_list('filename', 'sha256').iterator()
+        detail_packages = ((f, urljoin(BASE_CONTENT_URL, f'{path}/{f}'), d) for f, d in packages)
+        return StreamingHttpResponse(
+            write_simple_detail(package, detail_packages, streamed=True)
+        )
+
+
+class MetadataView(ViewSet, PyPIMixin):
+    """View for the PyPI JSON metadata endpoint."""
+
+    authentication_classes = []
+    permission_classes = []
+
+    @extend_schema(tags=["Pypi: Metadata"],
+                   responses={200: PackageMetadataSerializer},
+                   summary="Get package metadata")
+    def retrieve(self, request, path, meta):
+        """
+        Retrieves the package's core-metadata specified by
+        https://packaging.python.org/specifications/core-metadata/.
+        `meta` must be a path in form of `{package}/json/` or `{package}/{version}/json/`
+        """
+        distro, repo_ver, content = self.get_drvc(path)
+        meta_path = PurePath(meta)
+        name = None
+        version = None
+        if meta_path.match("*/*/json"):
+            version = meta_path.parts[1]
+            name = meta_path.parts[0]
+        elif meta_path.match("*/json"):
+            name = meta_path.parts[0]
+        if name:
+            package_content = content.filter(name__iexact=name)
+            # TODO Change this value to the Repo's serial value when implemented
+            headers = {PYPI_LAST_SERIAL: str(PYPI_SERIAL_CONSTANT)}
+            json_body = python_content_to_json(path, package_content, version=version)
+            if json_body:
+                return Response(data=json_body, headers=headers)
+        return Response(status="404")
+
+
+class PyPIView(ViewSet, PyPIMixin):
+    """View for base_url of distribution."""
+
+    authentication_classes = []
+    permission_classes = []
+
+    @extend_schema(responses={200: SummarySerializer},
+                   summary="Get index summary")
+    def retrieve(self, request, path):
+        """Gets package summary stats of index."""
+        distro, repo_ver, content = self.get_drvc(path)
+        files = content.count()
+        releases = content.distinct("name", "version").count()
+        projects = content.distinct("name").count()
+        data = {"projects": projects, "releases": releases, "files": files}
+        return Response(data=data)

pulp_python/app/serializers.py

@@ -45,6 +45,11 @@ class PythonDistributionSerializer(core_serializers.DistributionSerializer):
         queryset=core_models.Publication.objects.exclude(complete=False),
         allow_null=True,
     )
+    base_url = serializers.SerializerMethodField(read_only=True)
+
+    def get_base_url(self, obj):
+        """Gets the base url."""
+        return f"/pypi/{obj.base_path}/"
 
     def validate(self, data):
         """

pulp_python/app/urls.py

@@ -0,0 +1,24 @@
+from django.urls import path
+
+from pulp_python.app.pypi.views import SimpleView, MetadataView, PyPIView
+
+PYPI_API_URL = 'pypi/<path:path>/'
+
+# TODO: Implement remaining PyPI endpoints
+# path("project/", PackageProject.as_view()), # Endpoints to nicely see contents of index
+# path("search/", PackageSearch.as_view()),
+
+urlpatterns = [
+    path(
+        PYPI_API_URL + "pypi/<path:meta>/",
+        MetadataView.as_view({"get": "retrieve"}),
+        name="pypi-metadata"
+    ),
+    path(
+        PYPI_API_URL + "simple/<str:package>/",
+        SimpleView.as_view({"get": "retrieve"}),
+        name="simple-package-detail"
+    ),
+    path(PYPI_API_URL + 'simple/', SimpleView.as_view({"get": "list"}), name="simple-detail"),
+    path(PYPI_API_URL, PyPIView.as_view({"get": "retrieve"}), name="pypi-detail"),
+]

pulp_python/app/webserver_snippets/apache.conf

@@ -0,0 +1,2 @@
+ProxyPass /pulp_python/pypi ${pulp-api}/pypi
+ProxyPassReverse /pulp_python/pypi ${pulp-api}/pypi

pulp_python/app/webserver_snippets/nginx.conf

@@ -0,0 +1,9 @@
+location /pypi/ {
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_set_header Host $http_host;
+    # we don't want nginx trying to do something clever with
+    # redirects, we set the Host: header above already.
+    proxy_redirect off;
+    proxy_pass http://pulp-api;
+}