Add missing fields to PyPI JSON API

CHANGES/352.bugfix

@@ -0,0 +1 @@
+Added missing fields to PyPI live JSON API to be compliant with core metadata version 2.1

pulp_python/app/migrations/0007_pythonpackagecontent_mv-2-1.py

@@ -0,0 +1,25 @@
+# Generated by Django 2.2.23 on 2021-06-04 15:50
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('python', '0006_pythonrepository_autopublish'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='pythonpackagecontent',
+            name='description_content_type',
+            field=models.TextField(default=''),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='pythonpackagecontent',
+            name='project_urls',
+            field=django.contrib.postgres.fields.jsonb.JSONField(default=dict),
+        ),
+    ]

pulp_python/app/models.py

@@ -116,6 +116,8 @@ class PythonPackageContent(Content):
     obsoletes_dist = JSONField(default=list)
     requires_external = JSONField(default=list)
     classifiers = JSONField(default=list)
+    project_urls = JSONField(default=dict)
+    description_content_type = models.TextField()
 
     def __str__(self):
         """

pulp_python/app/serializers.py

@@ -118,6 +118,11 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploa
         required=False, allow_blank=True,
         help_text=_('A longer description of the package that can run to several paragraphs.')
     )
+    description_content_type = serializers.CharField(
+        required=False, allow_blank=True,
+        help_text=_('A string stating the markup syntax (if any) used in the distribution’s'
+                    ' description, so that tools can intelligently render the description.')
+    )
     keywords = serializers.CharField(
         required=False, allow_blank=True,
         help_text=_('Additional keywords to be used to assist searching for the '
@@ -162,6 +167,10 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploa
         required=False, allow_blank=True,
         help_text=_('A browsable URL for the project and a label for it, separated by a comma.')
     )
+    project_urls = serializers.JSONField(
+        required=False, default=dict,
+        help_text=_('A dictionary of labels and URLs for the project.')
+    )
     platform = serializers.CharField(
         required=False, allow_blank=True,
         help_text=_('A comma-separated list of platform specifications, '
@@ -264,10 +273,10 @@ def deferred_validate(self, data):
     class Meta:
         fields = core_serializers.SingleArtifactContentUploadSerializer.Meta.fields + (
             'filename', 'packagetype', 'name', 'version', 'sha256', 'metadata_version', 'summary',
-            'description', 'keywords', 'home_page', 'download_url', 'author', 'author_email',
-            'maintainer', 'maintainer_email', 'license', 'requires_python', 'project_url',
-            'platform', 'supported_platform', 'requires_dist', 'provides_dist',
-            'obsoletes_dist', 'requires_external', 'classifiers'
+            'description', 'description_content_type', 'keywords', 'home_page', 'download_url',
+            'author', 'author_email', 'maintainer', 'maintainer_email', 'license',
+            'requires_python', 'project_url', 'project_urls', 'platform', 'supported_platform',
+            'requires_dist', 'provides_dist', 'obsoletes_dist', 'requires_external', 'classifiers'
         )
         model = python_models.PythonPackageContent
 

pulp_python/app/utils.py

@@ -63,7 +63,6 @@ def parse_project_metadata(project):
     package['maintainer'] = project.get('maintainer') or ""
     package['maintainer_email'] = project.get('maintainer_email') or ""
     package['license'] = project.get('license') or ""
-    package['requires_python'] = project.get('requires_python') or ""
     package['project_url'] = project.get('project_url') or ""
     package['platform'] = project.get('platform') or ""
     package['supported_platform'] = project.get('supported_platform') or ""
@@ -72,6 +71,8 @@ def parse_project_metadata(project):
     package['obsoletes_dist'] = json.dumps(project.get('obsoletes_dist', []))
     package['requires_external'] = json.dumps(project.get('requires_external', []))
     package['classifiers'] = json.dumps(project.get('classifiers', []))
+    package['project_urls'] = json.dumps(project.get('project_urls', {}))
+    package['description_content_type'] = project.get('description_content_type') or ""
 
     return package
 
@@ -100,6 +101,7 @@ def parse_metadata(project, version, distribution):
     package['url'] = distribution.get('url') or ""
     package['sha256'] = distribution.get('digests', {}).get('sha256') or ""
     package['python_version'] = distribution.get('python_version') or ""
+    package['requires_python'] = distribution.get('requires_python') or ""
 
     package.update(parse_project_metadata(project))
 
@@ -148,32 +150,38 @@ def latest_content_version(content_query, version):
     return latest_content
 
 
-def python_content_to_info(latest_content):
+def python_content_to_info(content):
     """
     Takes in a PythonPackageContent instance and returns a dictionary of the Info fields
     """
     return {
-        "name": latest_content.name,
-        "version": latest_content.version,
-        "summary": latest_content.summary or None,
-        "description": latest_content.description or None,
-        "keywords": latest_content.keywords or None,
-        "home_page": latest_content.home_page or None,
+        "name": content.name,
+        "version": content.version,
+        "summary": content.summary or "",
+        "keywords": content.keywords or "",
+        "description": content.description or "",
+        "description_content_type": content.description_content_type or "",
+        "bugtrack_url": None,  # These two are basically never used
+        "docs_url": None,
         "downloads": {"last_day": -1, "last_month": -1, "last_week": -1},
-        "download_url": latest_content.download_url or None,
-        "author": latest_content.author or None,
-        "author_email": latest_content.author_email or None,
-        "maintainer": latest_content.maintainer or None,
-        "maintainer_email": latest_content.maintainer_email or None,
-        "license": latest_content.license or None,
-        "requires_python": latest_content.requires_python or None,
-        "project_url": latest_content.project_url or None,
-        "platform": latest_content.platform or None,
-        "requires_dist": json.loads(latest_content.requires_dist) or None,
-        "classifiers": json.loads(latest_content.classifiers) or None,
+        "download_url": content.download_url or "",
+        "home_page": content.home_page or "",
+        "author": content.author or "",
+        "author_email": content.author_email or "",
+        "maintainer": content.maintainer or "",
+        "maintainer_email": content.maintainer_email or "",
+        "license": content.license or "",
+        "requires_python": content.requires_python or None,
+        "package_url": content.project_url or "",  # These two are usually identical
+        "project_url": content.project_url or "",  # They also usually point to PyPI
+        "release_url": f"{content.project_url}{content.version}/" if content.project_url else "",
+        "project_urls": json.loads(content.project_urls) or None,
+        "platform": content.platform or "",
+        "requires_dist": json.loads(content.requires_dist) or None,
+        "classifiers": json.loads(content.classifiers) or None,
+        "yanked": False,  # These are no longer used on PyPI, but are still present
+        "yanked_reason": None,
     }
-    # fields missing: bugtrack_url, description_content_type, docs_url, package_url,
-    # project_urls {Download, Homepage}, release_url, yanked, yanked_reason
 
 
 def python_content_to_releases(content_query, base_path):

pulp_python/tests/functional/constants.py

@@ -223,8 +223,8 @@
     "download_url": "UNKNOWN",
     "author": "Austin Macdonald",
     "author_email": "asmacdo@gmail.com",
-    "maintainer": None,
-    "maintainer_email": None,
+    "maintainer": "",
+    "maintainer_email": "",
     # "license": "GNU GENERAL PUBLIC LICENSE Version 2, June 1991",
     "requires_python": None,
     "project_url": "https://pypi.org/project/shelf-reader/",