From 921a04cc26ec20b37745b7afb717269cb05ddf6f Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Wed, 28 Apr 2021 14:37:50 -0400 Subject: [PATCH] Add twine upload support fixes: #342 --- CHANGES/342.feature | 1 + doc_requirements.txt | 1 + docs/_scripts/index.sh | 1 + docs/_scripts/quickstart.sh | 4 + docs/_scripts/twine.sh | 4 + docs/tech-preview.rst | 3 +- docs/workflows/index.rst | 1 + docs/workflows/pypi.rst | 70 ++++++ functest_requirements.txt | 1 + .../0009_pythondistribution_allow_uploads.py | 18 ++ pulp_python/app/models.py | 2 + pulp_python/app/pypi/serializers.py | 70 +++++- pulp_python/app/pypi/views.py | 123 +++++++++- pulp_python/app/serializers.py | 11 +- pulp_python/app/tasks/__init__.py | 1 + pulp_python/app/tasks/upload.py | 108 ++++++--- pulp_python/app/urls.py | 12 +- .../functional/api/test_download_content.py | 4 - .../tests/functional/api/test_pypi_apis.py | 223 +++++++++++++++++- pulp_python/tests/functional/constants.py | 2 + 20 files changed, 594 insertions(+), 66 deletions(-) create mode 100644 CHANGES/342.feature create mode 100644 docs/_scripts/index.sh create mode 100644 docs/_scripts/twine.sh create mode 100644 docs/workflows/pypi.rst create mode 100644 pulp_python/app/migrations/0009_pythondistribution_allow_uploads.py diff --git a/CHANGES/342.feature b/CHANGES/342.feature new file mode 100644 index 000000000..13cef62f6 --- /dev/null +++ b/CHANGES/342.feature @@ -0,0 +1 @@ +Added ``twine`` (and other similar Python tools) package upload support diff --git a/doc_requirements.txt b/doc_requirements.txt index 95bd62b4f..59f6e13c4 100644 --- a/doc_requirements.txt +++ b/doc_requirements.txt @@ -1,3 +1,4 @@ +build coreapi django djangorestframework diff --git a/docs/_scripts/index.sh b/docs/_scripts/index.sh new file mode 100644 index 000000000..636a54d29 --- /dev/null +++ b/docs/_scripts/index.sh @@ -0,0 +1 @@ +pulp python distribution create --name my-pypi --base-path my-pypi --repository foo diff --git a/docs/_scripts/quickstart.sh b/docs/_scripts/quickstart.sh index d03728c36..091a8e158 100644 --- a/docs/_scripts/quickstart.sh +++ b/docs/_scripts/quickstart.sh @@ -7,6 +7,7 @@ # From the _scripts directory, run with `source quickstart.sh` (source to preserve the environment # variables) +export PLUGIN_SOURCE="../../" set -e source base.sh @@ -23,3 +24,6 @@ source pip.sh source upload.sh source add_content_repo.sh + +source index.sh +source twine.sh diff --git a/docs/_scripts/twine.sh b/docs/_scripts/twine.sh new file mode 100644 index 000000000..32d02ab46 --- /dev/null +++ b/docs/_scripts/twine.sh @@ -0,0 +1,4 @@ +# Build custom package +python -m build $PLUGIN_SOURCE +# Upload built package distributions to my-pypi +twine upload --repository-url $BASE_ADDR/pypi/my-pypi/simple/ -u admin -p password "$PLUGIN_SOURCE"dist/* diff --git a/docs/tech-preview.rst b/docs/tech-preview.rst index 80ea44d90..a2699c1f1 100644 --- a/docs/tech-preview.rst +++ b/docs/tech-preview.rst @@ -6,4 +6,5 @@ The following features are currently being released as part of a tech preview * New endpoint “pulp/api/v3/remotes/python/python/from_bandersnatch/” that allows for Python remote creation from a Bandersnatch config file. * PyPI’s json API at content endpoint ‘/pypi/{package-name}/json’. Allows for basic Pulp-to-Pulp syncing. -* Fully mirror Python repositories like PyPI +* Fully mirror Python repositories like PyPI. +* ``Twine`` upload packages to indexes at endpoints '/simple` or '/legacy'. diff --git a/docs/workflows/index.rst b/docs/workflows/index.rst index 5e07db1a5..d411c2229 100644 --- a/docs/workflows/index.rst +++ b/docs/workflows/index.rst @@ -24,6 +24,7 @@ Pulp CLI documentation on how to do that. .. toctree:: :maxdepth: 2 + pypi sync upload publish diff --git a/docs/workflows/pypi.rst b/docs/workflows/pypi.rst new file mode 100644 index 000000000..6a8a0f002 --- /dev/null +++ b/docs/workflows/pypi.rst @@ -0,0 +1,70 @@ +.. _pypi-workflow: + +Setup your own PyPI: +==================== + +This section guides you through the quickest way to to setup ``pulp_python`` to act as your very own +private ``PyPI``. + +Create a Repository: +-------------------- + +Repositories are the base objects ``Pulp`` uses to store and organize its content. They are automatically +versioned when content is added or deleted and allow for easy rollbacks to previous versions. + +.. literalinclude:: ../_scripts/repo.sh + :language: bash + +Repository Create Response:: + + { + "pulp_href": "/pulp/api/v3/repositories/python/python/3fe0d204-217f-4250-8177-c83b30751fca/", + "pulp_created": "2021-06-02T14:54:53.387054Z", + "versions_href": "/pulp/api/v3/repositories/python/python/3fe0d204-217f-4250-8177-c83b30751fca/versions/", + "pulp_labels": {}, + "latest_version_href": "/pulp/api/v3/repositories/python/python/3fe0d204-217f-4250-8177-c83b30751fca/versions/1/", + "name": "foo", + "description": null, + "retained_versions": null, + "remote": null, + "autopublish": false + } + +Create a Distribution: +---------------------- + +Distributions serve the content stored in repositories so that it can be used by tools like ``pip``. + +.. literalinclude:: ../_scripts/index.sh + :language: bash + +Distribution Create Response:: + + { + "pulp_href": "/pulp/api/v3/distributions/python/pypi/e8438593-fd40-4654-8577-65398833c331/", + "pulp_created": "2021-06-03T20:04:18.233230Z", + "base_path": "my-pypi", + "base_url": "https://pulp3-source-fedora33.localhost.example.com/pypi/foo/", + "content_guard": null, + "pulp_labels": {}, + "name": "my-pypi", + "repository": "/pulp/api/v3/repositories/python/python/3fe0d204-217f-4250-8177-c83b30751fca/", + "publication": null, + "allow_uploads": true + } + +Upload and Install Packages: +---------------------------- + +Packages can now be uploaded to the index using your favorite Python tool. The index url will be available +at ``/pypi//simple/``. + +.. literalinclude:: ../_scripts/twine.sh + :language: bash + +Packages can then be installed using your favorite Python tool:: + +$ pip install --trusted-host localhost -i $BASE_ADDR/pypi/my-pypi/simple/ shelf-reader + +Now you have a fully operational Python package index. Check out the other workflows to see more features of +``pulp_python``. diff --git a/functest_requirements.txt b/functest_requirements.txt index 843f289b6..e62167415 100644 --- a/functest_requirements.txt +++ b/functest_requirements.txt @@ -1,3 +1,4 @@ git+https://github.com/pulp/pulp-smash.git#egg=pulp-smash pytest lxml +twine diff --git a/pulp_python/app/migrations/0009_pythondistribution_allow_uploads.py b/pulp_python/app/migrations/0009_pythondistribution_allow_uploads.py new file mode 100644 index 000000000..de123d2c8 --- /dev/null +++ b/pulp_python/app/migrations/0009_pythondistribution_allow_uploads.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.24 on 2021-06-10 14:50 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('python', '0008_pythonpackagecontent_unique_sha256'), + ] + + operations = [ + migrations.AddField( + model_name='pythondistribution', + name='allow_uploads', + field=models.BooleanField(default=True), + ), + ] diff --git a/pulp_python/app/models.py b/pulp_python/app/models.py index f80974214..d9df96acd 100644 --- a/pulp_python/app/models.py +++ b/pulp_python/app/models.py @@ -43,6 +43,8 @@ class PythonDistribution(Distribution): TYPE = 'python' + allow_uploads = models.BooleanField(default=True) + def content_handler(self, path): """ Handler to serve extra, non-Artifact content for this Distribution diff --git a/pulp_python/app/pypi/serializers.py b/pulp_python/app/pypi/serializers.py index 856f5145c..d574d0b98 100644 --- a/pulp_python/app/pypi/serializers.py +++ b/pulp_python/app/pypi/serializers.py @@ -1,6 +1,12 @@ +import logging from gettext import gettext as _ from rest_framework import serializers +from pulp_python.app.tasks.upload import DIST_EXTENSIONS +from pulpcore.plugin.models import Artifact +from django.db.utils import IntegrityError + +log = logging.getLogger(__name__) class SummarySerializer(serializers.Serializer): @@ -9,7 +15,9 @@ class SummarySerializer(serializers.Serializer): """ projects = serializers.IntegerField(help_text=_("Number of Python projects in index")) - releases = serializers.IntegerField(help_text=_("Number of Python distributions in index")) + releases = serializers.IntegerField( + help_text=_("Number of Python distribution releases in index") + ) files = serializers.IntegerField(help_text=_("Number of files for all distributions in index")) @@ -22,3 +30,63 @@ class PackageMetadataSerializer(serializers.Serializer): info = serializers.JSONField(help_text=_("Core metadata of the package")) releases = serializers.JSONField(help_text=_("List of all the releases of the package")) urls = serializers.JSONField() + + +class PackageUploadSerializer(serializers.Serializer): + """ + A Serializer for Python packages being uploaded to the index. + """ + + content = serializers.FileField( + help_text=_("A Python package release file to upload to the index."), + required=True, + write_only=True, + ) + action = serializers.CharField( + help_text=_("Defaults to `file_upload`, don't change it or request will fail!"), + default="file_upload", + source=":action" + ) + sha256_digest = serializers.CharField( + help_text=_("SHA256 of package to validate upload integrity."), + required=True, + min_length=64, + max_length=64, + ) + + def validate(self, data): + """Validates the request.""" + action = data.get(":action") + if action != "file_upload": + raise serializers.ValidationError( + _("We do not support the :action {}").format(action) + ) + file = data.get("content") + for ext, packagetype in DIST_EXTENSIONS.items(): + if file.name.endswith(ext): + break + else: + raise serializers.ValidationError(_( + "Extension on {} is not a valid python extension " + "(.whl, .exe, .egg, .tar.gz, .tar.bz2, .zip)").format(file.name) + ) + sha256 = data.get("sha256_digest") + digests = {"sha256": sha256} if sha256 else None + artifact = Artifact.init_and_validate(file, expected_digests=digests) + try: + artifact.save() + except IntegrityError: + artifact = Artifact.objects.get(sha256=artifact.sha256) + log.info(f"Artifact for {file.name} already existed in database") + data["content"] = (artifact, file.name) + return data + + +class PackageUploadTaskSerializer(serializers.Serializer): + """ + A Serializer for responding to a package upload task. + """ + + session = serializers.CharField() + task = serializers.CharField() + task_start_time = serializers.DateTimeField() diff --git a/pulp_python/app/pypi/views.py b/pulp_python/app/pypi/views.py index c1c28dcac..663ac04c5 100644 --- a/pulp_python/app/pypi/views.py +++ b/pulp_python/app/pypi/views.py @@ -2,14 +2,27 @@ from rest_framework.viewsets import ViewSet from rest_framework.response import Response +from rest_framework.permissions import IsAuthenticatedOrReadOnly from django.core.exceptions import ObjectDoesNotExist from django.shortcuts import redirect -from django.http.response import Http404, StreamingHttpResponse +from datetime import datetime, timezone, timedelta + +from rest_framework.reverse import reverse +from django.contrib.sessions.models import Session +from django.db import transaction +from django.db.utils import DatabaseError +from django.http.response import ( + Http404, + HttpResponseForbidden, + HttpResponseBadRequest, + StreamingHttpResponse +) from drf_spectacular.utils import extend_schema from dynaconf import settings from urllib.parse import urljoin from pathlib import PurePath +from pulpcore.plugin.tasking import dispatch from pulp_python.app.models import ( PythonDistribution, PythonPackageContent, @@ -18,6 +31,8 @@ from pulp_python.app.pypi.serializers import ( SummarySerializer, PackageMetadataSerializer, + PackageUploadSerializer, + PackageUploadTaskSerializer ) from pulp_python.app.utils import ( write_simple_index, @@ -27,6 +42,8 @@ PYPI_SERIAL_CONSTANT, ) +from pulp_python.app import tasks + log = logging.getLogger(__name__) BASE_CONTENT_URL = urljoin(settings.CONTENT_ORIGIN, settings.CONTENT_PATH_PREFIX) @@ -78,11 +95,82 @@ def get_drvc(self, path): return distro, repo_ver, content -class SimpleView(ViewSet, PyPIMixin): +class PackageUploadMixin(PyPIMixin): + """A Mixin to provide package upload support.""" + + def upload(self, request, path): + """ + Upload a package to the index. + + 0. Check if the index allows uploaded packages (live-api-enabled) + 1. Check request is in correct format + 2. Check if the package is in the repository already + 3. If present then reject request + 4. Spawn task to add content if no/old session present + 5. Add uploads to current session to group into one task + """ + distro = self.get_distribution(path) + if not distro.allow_uploads: + return HttpResponseForbidden(reason="Index is not allowing uploads") + + repo = distro.repository + if not repo: + return HttpResponseBadRequest(reason="Index is not pointing to a repository") + + serializer = PackageUploadSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + artifact, filename = serializer.validated_data["content"] + repo_content = self.get_content(self.get_repository_version(distro)) + if repo_content.filter(filename=filename).exists(): + return HttpResponseBadRequest(reason=f"Package {filename} already exists in index") + + return self.upload_package(repo, artifact, filename, request.session) + + def upload_package(self, repo, artifact, filename, session): + """Steps 4 & 5, spawns tasks to add packages to index.""" + start_time = datetime.now(tz=timezone.utc) + timedelta(seconds=5) + task = "updated" + if not session.get("start"): + task = self.create_upload_task(session, repo, artifact, filename, start_time) + else: + sq = Session.objects.select_for_update(nowait=True).filter(pk=session.session_key) + with transaction.atomic(): + try: + sq.first() + try: + current_start = datetime.fromisoformat(session['start']) + except AttributeError: + from dateutil.parser import parse + current_start = parse(session['start']) + if current_start >= datetime.now(tz=timezone.utc): + session['artifacts'].append((str(artifact.sha256), filename)) + session['start'] = str(start_time) + session.modified = False + session.save() + else: + raise DatabaseError + except DatabaseError: + session.cycle_key() + task = self.create_upload_task(session, repo, artifact, filename, start_time) + data = {"session": session.session_key, "task": task, "task_start_time": start_time} + return Response(data=data) + + def create_upload_task(self, cur_session, repository, artifact, filename, start_time): + """Creates the actual task that adds the packages to the index.""" + cur_session['start'] = str(start_time) + cur_session['artifacts'] = [(str(artifact.sha256), filename)] + cur_session.modified = False + cur_session.save() + result = dispatch(tasks.upload, [artifact, repository], + kwargs={"session_pk": str(cur_session.session_key), + "repository_pk": str(repository.pk)}) + return reverse('tasks-detail', args=[result.pk], request=None) + + +class SimpleView(ViewSet, PackageUploadMixin): """View for the PyPI simple API.""" - authentication_classes = [] - permission_classes = [] + permission_classes = [IsAuthenticatedOrReadOnly] @extend_schema(summary="Get index simple page") def list(self, request, path): @@ -106,6 +194,18 @@ def retrieve(self, request, path, package): write_simple_detail(package, detail_packages, streamed=True) ) + @extend_schema(request=PackageUploadSerializer, + responses={200: PackageUploadTaskSerializer}, + summary="Upload a package") + def create(self, request, path): + """ + Upload package to the index. + This endpoint has the same functionality as the upload endpoint at the `/legacy` url of the + index. This is provided for convenience for users who want a single index url for all their + Python tools. (pip, twine, poetry, pipenv, ...) + """ + return self.upload(request, path) + class MetadataView(ViewSet, PyPIMixin): """View for the PyPI JSON metadata endpoint.""" @@ -157,3 +257,18 @@ def retrieve(self, request, path): projects = content.distinct("name").count() data = {"projects": projects, "releases": releases, "files": files} return Response(data=data) + + +class UploadView(ViewSet, PackageUploadMixin): + """View for the `/legacy` upload endpoint.""" + + @extend_schema(request=PackageUploadSerializer, + responses={200: PackageUploadTaskSerializer}, + summary="Upload a package") + def create(self, request, path): + """ + Upload package to the index. + + This is the endpoint that tools like Twine and Poetry use for their upload commands. + """ + return self.upload(request, path) diff --git a/pulp_python/app/serializers.py b/pulp_python/app/serializers.py index 473c77181..8276ee774 100644 --- a/pulp_python/app/serializers.py +++ b/pulp_python/app/serializers.py @@ -12,6 +12,7 @@ from pulp_python.app import models as python_models from pulp_python.app.tasks.upload import DIST_EXTENSIONS, DIST_TYPES from pulp_python.app.utils import parse_project_metadata +from pulp_python.app.urls import PYPI_API_HOSTNAME class PythonRepositorySerializer(core_serializers.RepositorySerializer): @@ -46,10 +47,14 @@ class PythonDistributionSerializer(core_serializers.DistributionSerializer): allow_null=True, ) base_url = serializers.SerializerMethodField(read_only=True) + allow_uploads = serializers.BooleanField( + default=True, + help_text=_("Allow packages to be uploaded to this index.") + ) def get_base_url(self, obj): """Gets the base url.""" - return f"/pypi/{obj.base_path}/" + return f"{PYPI_API_HOSTNAME}/pypi/{obj.base_path}/" def validate(self, data): """ @@ -74,7 +79,9 @@ def validate(self, data): return data class Meta: - fields = core_serializers.DistributionSerializer.Meta.fields + ('publication', ) + fields = core_serializers.DistributionSerializer.Meta.fields + ( + 'publication', "allow_uploads" + ) model = python_models.PythonDistribution diff --git a/pulp_python/app/tasks/__init__.py b/pulp_python/app/tasks/__init__.py index 631b332f3..2bd926206 100644 --- a/pulp_python/app/tasks/__init__.py +++ b/pulp_python/app/tasks/__init__.py @@ -4,3 +4,4 @@ from .publish import publish # noqa:F401 from .sync import sync # noqa:F401 +from .upload import upload # noqa:F401 diff --git a/pulp_python/app/tasks/upload.py b/pulp_python/app/tasks/upload.py index 8c2274141..d994edbad 100644 --- a/pulp_python/app/tasks/upload.py +++ b/pulp_python/app/tasks/upload.py @@ -1,11 +1,13 @@ import os -from gettext import gettext as _ import pkginfo import shutil import tempfile +import time -from pulpcore.plugin.models import Artifact, CreatedResource -from rest_framework import serializers +from datetime import datetime, timezone +from django.db import transaction +from django.contrib.sessions.models import Session +from pulpcore.plugin.models import Artifact, CreatedResource, ContentArtifact from pulp_python.app.models import PythonPackageContent, PythonRepository from pulp_python.app.utils import parse_project_metadata @@ -28,53 +30,81 @@ } -def one_shot_upload(artifact_pk, filename, repository_pk=None): +def upload(session_pk, repository_pk=None): """ - One shot upload for pulp_python + Uploads a Python Package to Pulp Args: - artifact_pk: validated artifact - filename: file name + session_pk: the session that has the artifacts to upload repository_pk: optional repository to add Content to """ + s_query = Session.objects.select_for_update().filter(pk=session_pk) + while True: + with transaction.atomic(): + session_data = s_query.first().get_decoded() + now = datetime.now(tz=timezone.utc) + try: + start_time = datetime.fromisoformat(session_data['start']) + except AttributeError: + # TODO: Remove this once Python 3.7+ project + from dateutil.parser import parse + start_time = parse(session_data['start']) + if now >= start_time: + content_to_add = PythonPackageContent.objects.none() + for artifact_sha256, filename in session_data['artifacts']: + pre_check = PythonPackageContent.objects.filter(sha256=artifact_sha256) + content_to_add |= pre_check or create_content(artifact_sha256, filename) + + if repository_pk: + repository = PythonRepository.objects.get(pk=repository_pk) + with repository.new_version() as new_version: + new_version.add_content(content_to_add) + return + else: + sleep_time = start_time - now + time.sleep(sleep_time.seconds) + + +def create_content(artifact_sha256, filename): + """ + Creates PythonPackageContent from artifact. + + Args: + artifact_sha256: validated artifact + filename: file name + Returns: + queryset of the new created content + """ # iterate through extensions since splitext does not support things like .tar.gz - for ext, packagetype in DIST_EXTENSIONS.items(): - if filename.endswith(ext): - # Copy file to a temp directory under the user provided filename, we do this - # because pkginfo validates that the filename has a valid extension before - # reading it - with tempfile.TemporaryDirectory() as td: - temp_path = os.path.join(td, filename) - artifact = Artifact.objects.get(pk=artifact_pk) - shutil.copy2(artifact.file.path, temp_path) - metadata = DIST_TYPES[packagetype](temp_path) - metadata.packagetype = packagetype - break - else: - raise serializers.ValidationError(_( - "Extension on {} is not a valid python extension " - "(.whl, .exe, .egg, .tar.gz, .tar.bz2, .zip)").format(filename) - ) + extensions = list(DIST_EXTENSIONS.keys()) + pkg_type_index = [filename.endswith(ext) for ext in extensions].index(True) + packagetype = DIST_EXTENSIONS[extensions[pkg_type_index]] + # Copy file to a temp directory under the user provided filename, we do this + # because pkginfo validates that the filename has a valid extension before + # reading it + with tempfile.TemporaryDirectory() as td: + temp_path = os.path.join(td, filename) + artifact = Artifact.objects.get(sha256=artifact_sha256) + shutil.copy2(artifact.file.path, temp_path) + metadata = DIST_TYPES[packagetype](temp_path) + metadata.packagetype = packagetype + data = parse_project_metadata(vars(metadata)) - data['classifiers'] = [{'name': classifier} for classifier in metadata.classifiers] data['packagetype'] = metadata.packagetype data['version'] = metadata.version data['filename'] = filename - data['_relative_path'] = filename - - new_content = PythonPackageContent.objects.create( - filename=filename, - packagetype=metadata.packagetype, - name=data['classifiers'], - version=data['version'] - ) + data['sha256'] = artifact.sha256 - queryset = PythonPackageContent.objects.filter(pk=new_content.pk) - - if repository_pk: - repository = PythonRepository.objects.get(pk=repository_pk) - with repository.new_version() as new_version: - new_version.add_content(queryset) + @transaction.atomic() + def create(): + content = PythonPackageContent.objects.create(**data) + ContentArtifact.objects.create( + artifact=artifact, content=content, relative_path=filename + ) + return content + new_content = create() resource = CreatedResource(content_object=new_content) resource.save() + + return PythonPackageContent.objects.filter(pk=new_content.pk) diff --git a/pulp_python/app/urls.py b/pulp_python/app/urls.py index 8a5082560..e26cb020d 100644 --- a/pulp_python/app/urls.py +++ b/pulp_python/app/urls.py @@ -1,14 +1,16 @@ +import socket from django.urls import path -from pulp_python.app.pypi.views import SimpleView, MetadataView, PyPIView +from pulp_python.app.pypi.views import SimpleView, MetadataView, PyPIView, UploadView PYPI_API_URL = 'pypi//' - +PYPI_API_HOSTNAME = 'https://' + socket.getfqdn() # TODO: Implement remaining PyPI endpoints # path("project/", PackageProject.as_view()), # Endpoints to nicely see contents of index # path("search/", PackageSearch.as_view()), urlpatterns = [ + path(PYPI_API_URL + "legacy/", UploadView.as_view({"post": "create"}), name="upload"), path( PYPI_API_URL + "pypi//", MetadataView.as_view({"get": "retrieve"}), @@ -19,6 +21,10 @@ SimpleView.as_view({"get": "retrieve"}), name="simple-package-detail" ), - path(PYPI_API_URL + 'simple/', SimpleView.as_view({"get": "list"}), name="simple-detail"), + path( + PYPI_API_URL + 'simple/', + SimpleView.as_view({"get": "list", "post": "create"}), + name="simple-detail" + ), path(PYPI_API_URL, PyPIView.as_view({"get": "retrieve"}), name="pypi-detail"), ] diff --git a/pulp_python/tests/functional/api/test_download_content.py b/pulp_python/tests/functional/api/test_download_content.py index ecce748b0..597149b2a 100644 --- a/pulp_python/tests/functional/api/test_download_content.py +++ b/pulp_python/tests/functional/api/test_download_content.py @@ -24,10 +24,6 @@ from pulp_python.tests.functional.utils import set_up_module as setUpModule # noqa:F401 -PYPI_LAST_SERIAL = "X-PYPI-LAST-SERIAL" -PYPI_SERIAL_CONSTANT = 1000000000 - - class DownloadContentTestCase(TestCaseUsingBindings, TestHelpersMixin): """Verify whether content served by pulp can be downloaded.""" diff --git a/pulp_python/tests/functional/api/test_pypi_apis.py b/pulp_python/tests/functional/api/test_pypi_apis.py index b215502cd..421204c84 100644 --- a/pulp_python/tests/functional/api/test_pypi_apis.py +++ b/pulp_python/tests/functional/api/test_pypi_apis.py @@ -1,9 +1,15 @@ -"""Tests all the PyPI apis available at `pulp_python/pypi/`.""" +"""Tests all the PyPI apis available at `pypi/`.""" +import os import requests +import subprocess +import tempfile from urllib.parse import urljoin +from pulp_smash.pulp3.bindings import monitor_task, tasks as task_api +from pulp_smash.pulp3.utils import get_added_content_summary, get_content_summary from pulp_python.tests.functional.constants import ( + PYTHON_CONTENT_NAME, PYTHON_SM_PROJECT_SPECIFIER, PYTHON_SM_FIXTURE_RELEASES, PYTHON_SM_FIXTURE_CHECKSUMS, @@ -11,7 +17,13 @@ PYTHON_MD_PYPI_SUMMARY, PULP_CONTENT_BASE_URL, PULP_PYPI_BASE_URL, - SHELF_PYTHON_JSON + PYTHON_EGG_FILENAME, + PYTHON_EGG_URL, + PYTHON_EGG_SHA256, + PYTHON_WHEEL_FILENAME, + PYTHON_WHEEL_URL, + PYTHON_WHEEL_SHA256, + SHELF_PYTHON_JSON, ) from pulp_python.tests.functional.utils import ( @@ -64,6 +76,182 @@ def test_published_index(self): self.assertDictEqual(summary.to_dict(), PYTHON_MD_PYPI_SUMMARY) +class PyPIPackageUpload(TestCaseUsingBindings, TestHelpersMixin): + """Tests the package upload endpoints of an index.""" + + @classmethod + def setUpClass(cls): + """Set up class variables.""" + super().setUpClass() + cls.pypi_api = PypiApi(client) + cls.dists_dir = tempfile.TemporaryDirectory() + cls.egg = os.path.join(cls.dists_dir.name, PYTHON_EGG_FILENAME) + cls.wheel = os.path.join(cls.dists_dir.name, PYTHON_WHEEL_FILENAME) + with open(cls.egg, "wb") as fp: + fp.write(requests.get(PYTHON_EGG_URL).content) + with open(cls.wheel, "wb") as fp: + fp.write(requests.get(PYTHON_WHEEL_URL).content) + + @classmethod + def tearDownClass(cls): + """Tear down class variables.""" + cls.dists_dir.cleanup() + + def test_package_upload(self): + """Tests that packages can be uploaded.""" + repo, distro = self._create_empty_repo_and_distribution() + url = urljoin(PYPI_HOST, distro.base_path + "/legacy/") + response = requests.post( + url, + data={"sha256_digest": PYTHON_EGG_SHA256}, + files={"content": open(self.egg, "rb")}, + ) + self.assertEqual(response.status_code, 200) + task = response.json()["task"] + monitor_task(task) + content = get_added_content_summary(repo, f"{repo.versions_href}1/") + self.assertDictEqual({PYTHON_CONTENT_NAME: 1}, content) + # Test re-uploading same package gives 400 Bad Request + response = requests.post( + url, + data={"sha256_digest": PYTHON_EGG_SHA256}, + files={"content": open(self.egg, "rb")}, + ) + self.assertEqual(response.status_code, 400) + self.assertEqual( + response.reason, f"Package {PYTHON_EGG_FILENAME} already exists in index" + ) + + def test_package_upload_session(self): + """Tests that multiple packages can be uploaded in one session.""" + repo, distro = self._create_empty_repo_and_distribution() + url = urljoin(PYPI_HOST, distro.base_path + "/legacy/") + session = requests.Session() + response = session.post( + url, + data={"sha256_digest": PYTHON_EGG_SHA256}, + files={"content": open(self.egg, "rb")}, + ) + response2 = session.post( + url, + data={"sha256_digest": PYTHON_WHEEL_SHA256}, + files={"content": open(self.wheel, "rb")}, + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response2.status_code, 200) + response, response2 = response.json(), response2.json() + self.assertEqual(response["session"], response2["session"]) + self.assertEqual(response2["task"], "updated") + monitor_task(response["task"]) + content = get_added_content_summary(repo, f"{repo.versions_href}1/") + self.assertDictEqual({PYTHON_CONTENT_NAME: 2}, content) + + def test_package_upload_session_long(self): + """Tests that long uploads will be broken up into multiple tasks.""" + repo, distro = self._create_empty_repo_and_distribution() + url = urljoin(PYPI_HOST, distro.base_path + "/legacy/") + session = requests.Session() + response = session.post( + url, + data={"sha256_digest": PYTHON_EGG_SHA256}, + files={"content": open(self.egg, "rb")}, + ) + self.assertEqual(response.status_code, 200) + response = response.json() + monitor_task(response["task"]) + response2 = session.post( + url, + data={"sha256_digest": PYTHON_WHEEL_SHA256}, + files={"content": open(self.wheel, "rb")}, + ) + self.assertEqual(response2.status_code, 200) + response2 = response2.json() + self.assertNotEqual("updated", response2["task"]) + self.assertNotEqual(response["session"], response2["session"]) + self.assertNotEqual(response["task"], response2["task"]) + monitor_task(response2["task"]) + content = get_content_summary(repo, f"{repo.versions_href}2/") + self.assertDictEqual({PYTHON_CONTENT_NAME: 2}, content) + + def test_package_upload_simple(self): + """Tests that the package upload endpoint exposed at `/simple/` works.""" + repo, distro = self._create_empty_repo_and_distribution() + url = urljoin(PYPI_HOST, distro.base_path + "/simple/") + response = requests.post( + url, + data={"sha256_digest": PYTHON_EGG_SHA256}, + files={"content": open(self.egg, "rb")}, + ) + self.assertEqual(response.status_code, 200) + task = response.json()["task"] + monitor_task(task) + content = get_added_content_summary(repo, f"{repo.versions_href}1/") + self.assertDictEqual({PYTHON_CONTENT_NAME: 1}, content) + + def test_twine_upload(self): + """Tests that packages can be properly uploaded through Twine.""" + repo, distro = self._create_empty_repo_and_distribution() + url = urljoin(PYPI_HOST, distro.base_path + "/legacy/") + username, password = "admin", "password" + subprocess.run( + ( + "twine", + "upload", + "--repository-url", + url, + self.dists_dir.name + "/*", + "-u", + username, + "-p", + password, + ), + capture_output=True, + check=True, + ) + task = task_api.list(reserved_resources_record=repo.pulp_href).results[0] + monitor_task(task.pulp_href) + content = get_added_content_summary(repo, f"{repo.versions_href}1/") + self.assertDictEqual({PYTHON_CONTENT_NAME: 2}, content) + + # Test re-uploading same packages gives error + with self.assertRaises(subprocess.CalledProcessError): + subprocess.run( + ( + "twine", + "upload", + "--repository-url", + url, + self.dists_dir.name + "/*", + "-u", + username, + "-p", + password, + ), + capture_output=True, + check=True, + ) + + # Test re-uploading same packages with --skip-existing works + output = subprocess.run( + ( + "twine", + "upload", + "--repository-url", + url, + self.dists_dir.name + "/*", + "-u", + username, + "-p", + password, + "--skip-existing", + ), + capture_output=True, + check=True, + text=True + ) + self.assertEqual(output.stdout.count("Skipping"), 2) + + class PyPISimpleApi(TestCaseUsingBindings, TestHelpersMixin): """Tests that the simple api is correct.""" @@ -73,9 +261,10 @@ def test_simple_redirect_with_publications(self): repo = self._create_repo_and_sync_with_remote(remote) pub = self._create_publication(repo) distro = self._create_distribution_from_publication(pub) - response = requests.get(urljoin(PYPI_HOST, f'{distro.base_path}/simple/')) + response = requests.get(urljoin(PYPI_HOST, f"{distro.base_path}/simple/")) self.assertEqual( - response.url, str(urljoin(PULP_CONTENT_BASE_URL, f"{distro.base_path}/simple/")) + response.url, + str(urljoin(PULP_CONTENT_BASE_URL, f"{distro.base_path}/simple/")), ) def test_simple_correctness_live(self): @@ -84,7 +273,7 @@ def test_simple_correctness_live(self): repo = self._create_repo_and_sync_with_remote(remote) distro = self._create_distribution_from_repo(repo) proper, msgs = ensure_simple( - urljoin(PYPI_HOST, f'{distro.base_path}/simple/'), + urljoin(PYPI_HOST, f"{distro.base_path}/simple/"), PYTHON_SM_FIXTURE_RELEASES, sha_digests=PYTHON_SM_FIXTURE_CHECKSUMS, ) @@ -122,24 +311,34 @@ def test_pypi_last_serial(self): repo = self._create_repo_and_sync_with_remote(remote) pub = self._create_publication(repo) distro = self._create_distribution_from_publication(pub) - content_url = urljoin(PULP_CONTENT_BASE_URL, f"{distro.base_path}/pypi/shelf-reader/json") + content_url = urljoin( + PULP_CONTENT_BASE_URL, f"{distro.base_path}/pypi/shelf-reader/json" + ) pypi_url = urljoin(PYPI_HOST, f"{distro.base_path}/pypi/shelf-reader/json/") for url in [content_url, pypi_url]: response = requests.get(url) self.assertIn(PYPI_LAST_SERIAL, response.headers, msg=url) - self.assertEqual(response.headers[PYPI_LAST_SERIAL], str(PYPI_SERIAL_CONSTANT), msg=url) + self.assertEqual( + response.headers[PYPI_LAST_SERIAL], str(PYPI_SERIAL_CONSTANT), msg=url + ) def assert_pypi_json(self, package): """Asserts that shelf-reader package json is correct.""" self.assertEqual(SHELF_PYTHON_JSON["last_serial"], package["last_serial"]) self.assertTrue(SHELF_PYTHON_JSON["info"].items() <= package["info"].items()) self.assertEqual(len(SHELF_PYTHON_JSON["urls"]), len(package["urls"])) - self.assert_download_info(SHELF_PYTHON_JSON["urls"], package["urls"], - "Failed to match URLS") - self.assertTrue(SHELF_PYTHON_JSON["releases"].keys() <= package["releases"].keys()) + self.assert_download_info( + SHELF_PYTHON_JSON["urls"], package["urls"], "Failed to match URLS" + ) + self.assertTrue( + SHELF_PYTHON_JSON["releases"].keys() <= package["releases"].keys() + ) for version in SHELF_PYTHON_JSON["releases"].keys(): - self.assert_download_info(SHELF_PYTHON_JSON["releases"][version], - package["releases"][version], "Failed to match version") + self.assert_download_info( + SHELF_PYTHON_JSON["releases"][version], + package["releases"][version], + "Failed to match version", + ) def assert_download_info(self, expected, received, message="Failed to match"): """ diff --git a/pulp_python/tests/functional/constants.py b/pulp_python/tests/functional/constants.py index ce69f2564..3ff90f746 100644 --- a/pulp_python/tests/functional/constants.py +++ b/pulp_python/tests/functional/constants.py @@ -168,12 +168,14 @@ # Intended to be used with the XS specifier PYTHON_EGG_FILENAME = "shelf-reader-0.1.tar.gz" PYTHON_EGG_URL = urljoin(urljoin(PYTHON_FIXTURES_URL, "packages/"), PYTHON_EGG_FILENAME) +PYTHON_EGG_SHA256 = "04cfd8bb4f843e35d51bfdef2035109bdea831b55a57c3e6a154d14be116398c" # Intended to be used with the XS specifier PYTHON_WHEEL_FILENAME = "shelf_reader-0.1-py2-none-any.whl" PYTHON_WHEEL_URL = urljoin( urljoin(PYTHON_FIXTURES_URL, "packages/"), PYTHON_WHEEL_FILENAME ) +PYTHON_WHEEL_SHA256 = "2eceb1643c10c5e4a65970baf63bde43b79cbdac7de81dae853ce47ab05197e9" PYTHON_FIXTURES_PACKAGES = [ "shelf-reader",