Added twine upload support

CHANGES/342.feature

@@ -0,0 +1 @@
+Added ``twine`` (and other similar Python tools) package upload support

doc_requirements.txt

@@ -1,3 +1,4 @@
+build
 coreapi
 django
 djangorestframework

docs/_scripts/index.sh

@@ -0,0 +1 @@
+pulp python distribution create --name my-pypi --base-path my-pypi --repository foo

docs/_scripts/quickstart.sh

@@ -7,6 +7,7 @@
 
 # From the _scripts directory, run with `source quickstart.sh` (source to preserve the environment
 # variables)
+export PLUGIN_SOURCE="../../"
 set -e
 
 source base.sh
@@ -23,3 +24,6 @@ source pip.sh
 
 source upload.sh
 source add_content_repo.sh
+
+source index.sh
+source twine.sh

docs/_scripts/twine.sh

@@ -0,0 +1,4 @@
+# Build custom package
+python -m build $PLUGIN_SOURCE
+# Upload built package distributions to my-pypi
+twine upload --repository-url $BASE_ADDR/pypi/my-pypi/simple/ -u admin -p password "$PLUGIN_SOURCE"dist/*

docs/tech-preview.rst

@@ -6,4 +6,5 @@ The following features are currently being released as part of a tech preview
 * New endpoint “pulp/api/v3/remotes/python/python/from_bandersnatch/” that allows for Python remote creation from a
   Bandersnatch config file.
 * PyPI’s json API at content endpoint ‘/pypi/{package-name}/json’. Allows for basic Pulp-to-Pulp syncing.
-* Fully mirror Python repositories like PyPI
+* Fully mirror Python repositories like PyPI.
+* ``Twine`` upload packages to indexes at endpoints '/simple` or '/legacy'.

docs/workflows/index.rst

@@ -24,6 +24,7 @@ Pulp CLI documentation on how to do that.
 .. toctree::
    :maxdepth: 2
 
+   pypi
    sync
    upload
    publish

docs/workflows/pypi.rst

@@ -0,0 +1,70 @@
+.. _pypi-workflow:
+
+Setup your own PyPI:
+====================
+
+This section guides you through the quickest way to to setup ``pulp_python`` to act as your very own
+private ``PyPI``.
+
+Create a Repository:
+--------------------
+
+Repositories are the base objects ``Pulp`` uses to store and organize its content. They are automatically
+versioned when content is added or deleted and allow for easy rollbacks to previous versions.
+
+.. literalinclude:: ../_scripts/repo.sh
+   :language: bash
+
+Repository Create Response::
+
+   {
+        "pulp_href": "/pulp/api/v3/repositories/python/python/3fe0d204-217f-4250-8177-c83b30751fca/",
+        "pulp_created": "2021-06-02T14:54:53.387054Z",
+        "versions_href": "/pulp/api/v3/repositories/python/python/3fe0d204-217f-4250-8177-c83b30751fca/versions/",
+        "pulp_labels": {},
+        "latest_version_href": "/pulp/api/v3/repositories/python/python/3fe0d204-217f-4250-8177-c83b30751fca/versions/1/",
+        "name": "foo",
+        "description": null,
+        "retained_versions": null,
+        "remote": null,
+        "autopublish": false
+    }
+
+Create a Distribution:
+----------------------
+
+Distributions serve the content stored in repositories so that it can be used by tools like ``pip``.
+
+.. literalinclude:: ../_scripts/index.sh
+    :language: bash
+
+Distribution Create Response::
+
+    {
+      "pulp_href": "/pulp/api/v3/distributions/python/pypi/e8438593-fd40-4654-8577-65398833c331/",
+      "pulp_created": "2021-06-03T20:04:18.233230Z",
+      "base_path": "my-pypi",
+      "base_url": "https://pulp3-source-fedora33.localhost.example.com/pypi/foo/",
+      "content_guard": null,
+      "pulp_labels": {},
+      "name": "my-pypi",
+      "repository": "/pulp/api/v3/repositories/python/python/3fe0d204-217f-4250-8177-c83b30751fca/",
+      "publication": null,
+      "allow_uploads": true
+    }
+
+Upload and Install Packages:
+----------------------------
+
+Packages can now be uploaded to the index using your favorite Python tool. The index url will be available
+at ``/pypi/<distribution.base_path>/simple/``.
+
+.. literalinclude:: ../_scripts/twine.sh
+    :language: bash
+
+Packages can then be installed using your favorite Python tool::
+
+$ pip install --trusted-host localhost -i $BASE_ADDR/pypi/my-pypi/simple/ shelf-reader
+
+Now you have a fully operational Python package index. Check out the other workflows to see more features of
+``pulp_python``.

functest_requirements.txt

@@ -1,3 +1,4 @@
 git+https://github.com/pulp/pulp-smash.git#egg=pulp-smash
 pytest
 lxml
+twine

pulp_python/app/migrations/0009_pythondistribution_allow_uploads.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.24 on 2021-06-10 14:50
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('python', '0008_pythonpackagecontent_unique_sha256'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='pythondistribution',
+            name='allow_uploads',
+            field=models.BooleanField(default=True),
+        ),
+    ]

pulp_python/app/models.py

@@ -43,6 +43,8 @@ class PythonDistribution(Distribution):
 
     TYPE = 'python'
 
+    allow_uploads = models.BooleanField(default=True)
+
     def content_handler(self, path):
         """
         Handler to serve extra, non-Artifact content for this Distribution

pulp_python/app/pypi/serializers.py

@@ -1,6 +1,12 @@
+import logging
 from gettext import gettext as _
 
 from rest_framework import serializers
+from pulp_python.app.tasks.upload import DIST_EXTENSIONS
+from pulpcore.plugin.models import Artifact
+from django.db.utils import IntegrityError
+
+log = logging.getLogger(__name__)
 
 
 class SummarySerializer(serializers.Serializer):
@@ -9,7 +15,9 @@ class SummarySerializer(serializers.Serializer):
     """
 
     projects = serializers.IntegerField(help_text=_("Number of Python projects in index"))
-    releases = serializers.IntegerField(help_text=_("Number of Python distributions in index"))
+    releases = serializers.IntegerField(
+        help_text=_("Number of Python distribution releases in index")
+    )
     files = serializers.IntegerField(help_text=_("Number of files for all distributions in index"))
 
 
@@ -22,3 +30,63 @@ class PackageMetadataSerializer(serializers.Serializer):
     info = serializers.JSONField(help_text=_("Core metadata of the package"))
     releases = serializers.JSONField(help_text=_("List of all the releases of the package"))
     urls = serializers.JSONField()
+
+
+class PackageUploadSerializer(serializers.Serializer):
+    """
+    A Serializer for Python packages being uploaded to the index.
+    """
+
+    content = serializers.FileField(
+        help_text=_("A Python package release file to upload to the index."),
+        required=True,
+        write_only=True,
+    )
+    action = serializers.CharField(
+        help_text=_("Defaults to `file_upload`, don't change it or request will fail!"),
+        default="file_upload",
+        source=":action"
+    )
+    sha256_digest = serializers.CharField(
+        help_text=_("SHA256 of package to validate upload integrity."),
+        required=True,
+        min_length=64,
+        max_length=64,
+    )
+
+    def validate(self, data):
+        """Validates the request."""
+        action = data.get(":action")
+        if action != "file_upload":
+            raise serializers.ValidationError(
+                _("We do not support the :action {}").format(action)
+            )
+        file = data.get("content")
+        for ext, packagetype in DIST_EXTENSIONS.items():
+            if file.name.endswith(ext):
+                break
+        else:
+            raise serializers.ValidationError(_(
+                "Extension on {} is not a valid python extension "
+                "(.whl, .exe, .egg, .tar.gz, .tar.bz2, .zip)").format(file.name)
+            )
+        sha256 = data.get("sha256_digest")
+        digests = {"sha256": sha256} if sha256 else None
+        artifact = Artifact.init_and_validate(file, expected_digests=digests)
+        try:
+            artifact.save()
+        except IntegrityError:
+            artifact = Artifact.objects.get(sha256=artifact.sha256)
+            log.info(f"Artifact for {file.name} already existed in database")
+        data["content"] = (artifact, file.name)
+        return data
+
+
+class PackageUploadTaskSerializer(serializers.Serializer):
+    """
+    A Serializer for responding to a package upload task.
+    """
+
+    session = serializers.CharField()
+    task = serializers.CharField()
+    task_start_time = serializers.DateTimeField()