From dd894c32986ceb3f4da273638f8bcc94a8e887d6 Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Fri, 4 Jun 2021 13:19:47 -0400 Subject: [PATCH] Add missing fields to PyPI JSON API fixes: #352 --- CHANGES/352.bugfix | 1 + .../0007_pythonpackagecontent_mv-2-1.py | 25 ++++++++++ pulp_python/app/models.py | 2 + pulp_python/app/serializers.py | 17 +++++-- pulp_python/app/utils.py | 50 +++++++++++-------- pulp_python/tests/functional/constants.py | 4 +- 6 files changed, 72 insertions(+), 27 deletions(-) create mode 100644 CHANGES/352.bugfix create mode 100644 pulp_python/app/migrations/0007_pythonpackagecontent_mv-2-1.py diff --git a/CHANGES/352.bugfix b/CHANGES/352.bugfix new file mode 100644 index 000000000..26fc6538e --- /dev/null +++ b/CHANGES/352.bugfix @@ -0,0 +1 @@ +Added missing fields to PyPI live JSON API to be compliant with core metadata version 2.1 diff --git a/pulp_python/app/migrations/0007_pythonpackagecontent_mv-2-1.py b/pulp_python/app/migrations/0007_pythonpackagecontent_mv-2-1.py new file mode 100644 index 000000000..8a817cf6e --- /dev/null +++ b/pulp_python/app/migrations/0007_pythonpackagecontent_mv-2-1.py @@ -0,0 +1,25 @@ +# Generated by Django 2.2.23 on 2021-06-04 15:50 + +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('python', '0006_pythonrepository_autopublish'), + ] + + operations = [ + migrations.AddField( + model_name='pythonpackagecontent', + name='description_content_type', + field=models.TextField(default=''), + preserve_default=False, + ), + migrations.AddField( + model_name='pythonpackagecontent', + name='project_urls', + field=django.contrib.postgres.fields.jsonb.JSONField(default=dict), + ), + ] diff --git a/pulp_python/app/models.py b/pulp_python/app/models.py index 1975a4447..87d529ae1 100644 --- a/pulp_python/app/models.py +++ b/pulp_python/app/models.py @@ -116,6 +116,8 @@ class PythonPackageContent(Content): obsoletes_dist = JSONField(default=list) requires_external = JSONField(default=list) classifiers = JSONField(default=list) + project_urls = JSONField(default=dict) + description_content_type = models.TextField() def __str__(self): """ diff --git a/pulp_python/app/serializers.py b/pulp_python/app/serializers.py index 3336ee8dc..09af34baf 100644 --- a/pulp_python/app/serializers.py +++ b/pulp_python/app/serializers.py @@ -118,6 +118,11 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploa required=False, allow_blank=True, help_text=_('A longer description of the package that can run to several paragraphs.') ) + description_content_type = serializers.CharField( + required=False, allow_blank=True, + help_text=_('A string stating the markup syntax (if any) used in the distribution’s' + ' description, so that tools can intelligently render the description.') + ) keywords = serializers.CharField( required=False, allow_blank=True, help_text=_('Additional keywords to be used to assist searching for the ' @@ -162,6 +167,10 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploa required=False, allow_blank=True, help_text=_('A browsable URL for the project and a label for it, separated by a comma.') ) + project_urls = serializers.JSONField( + required=False, default=dict, + help_text=_('A dictionary of labels and URLs for the project.') + ) platform = serializers.CharField( required=False, allow_blank=True, help_text=_('A comma-separated list of platform specifications, ' @@ -264,10 +273,10 @@ def deferred_validate(self, data): class Meta: fields = core_serializers.SingleArtifactContentUploadSerializer.Meta.fields + ( 'filename', 'packagetype', 'name', 'version', 'sha256', 'metadata_version', 'summary', - 'description', 'keywords', 'home_page', 'download_url', 'author', 'author_email', - 'maintainer', 'maintainer_email', 'license', 'requires_python', 'project_url', - 'platform', 'supported_platform', 'requires_dist', 'provides_dist', - 'obsoletes_dist', 'requires_external', 'classifiers' + 'description', 'description_content_type', 'keywords', 'home_page', 'download_url', + 'author', 'author_email', 'maintainer', 'maintainer_email', 'license', + 'requires_python', 'project_url', 'project_urls', 'platform', 'supported_platform', + 'requires_dist', 'provides_dist', 'obsoletes_dist', 'requires_external', 'classifiers' ) model = python_models.PythonPackageContent diff --git a/pulp_python/app/utils.py b/pulp_python/app/utils.py index 0110eaad7..8e83619dc 100644 --- a/pulp_python/app/utils.py +++ b/pulp_python/app/utils.py @@ -63,7 +63,6 @@ def parse_project_metadata(project): package['maintainer'] = project.get('maintainer') or "" package['maintainer_email'] = project.get('maintainer_email') or "" package['license'] = project.get('license') or "" - package['requires_python'] = project.get('requires_python') or "" package['project_url'] = project.get('project_url') or "" package['platform'] = project.get('platform') or "" package['supported_platform'] = project.get('supported_platform') or "" @@ -72,6 +71,8 @@ def parse_project_metadata(project): package['obsoletes_dist'] = json.dumps(project.get('obsoletes_dist', [])) package['requires_external'] = json.dumps(project.get('requires_external', [])) package['classifiers'] = json.dumps(project.get('classifiers', [])) + package['project_urls'] = json.dumps(project.get('project_urls', {})) + package['description_content_type'] = project.get('description_content_type') or "" return package @@ -100,6 +101,7 @@ def parse_metadata(project, version, distribution): package['url'] = distribution.get('url') or "" package['sha256'] = distribution.get('digests', {}).get('sha256') or "" package['python_version'] = distribution.get('python_version') or "" + package['requires_python'] = distribution.get('requires_python') or "" package.update(parse_project_metadata(project)) @@ -148,32 +150,38 @@ def latest_content_version(content_query, version): return latest_content -def python_content_to_info(latest_content): +def python_content_to_info(content): """ Takes in a PythonPackageContent instance and returns a dictionary of the Info fields """ return { - "name": latest_content.name, - "version": latest_content.version, - "summary": latest_content.summary or None, - "description": latest_content.description or None, - "keywords": latest_content.keywords or None, - "home_page": latest_content.home_page or None, + "name": content.name, + "version": content.version, + "summary": content.summary or "", + "keywords": content.keywords or "", + "description": content.description or "", + "description_content_type": content.description_content_type or "", + "bugtrack_url": None, # These two are basically never used + "docs_url": None, "downloads": {"last_day": -1, "last_month": -1, "last_week": -1}, - "download_url": latest_content.download_url or None, - "author": latest_content.author or None, - "author_email": latest_content.author_email or None, - "maintainer": latest_content.maintainer or None, - "maintainer_email": latest_content.maintainer_email or None, - "license": latest_content.license or None, - "requires_python": latest_content.requires_python or None, - "project_url": latest_content.project_url or None, - "platform": latest_content.platform or None, - "requires_dist": json.loads(latest_content.requires_dist) or None, - "classifiers": json.loads(latest_content.classifiers) or None, + "download_url": content.download_url or "", + "home_page": content.home_page or "", + "author": content.author or "", + "author_email": content.author_email or "", + "maintainer": content.maintainer or "", + "maintainer_email": content.maintainer_email or "", + "license": content.license or "", + "requires_python": content.requires_python or None, + "package_url": content.project_url or "", # These two are usually identical + "project_url": content.project_url or "", # They also usually point to PyPI + "release_url": f"{content.project_url}{content.version}/" if content.project_url else "", + "project_urls": json.loads(content.project_urls) or None, + "platform": content.platform or "", + "requires_dist": json.loads(content.requires_dist) or None, + "classifiers": json.loads(content.classifiers) or None, + "yanked": False, # These are no longer used on PyPI, but are still present + "yanked_reason": None, } - # fields missing: bugtrack_url, description_content_type, docs_url, package_url, - # project_urls {Download, Homepage}, release_url, yanked, yanked_reason def python_content_to_releases(content_query, base_path): diff --git a/pulp_python/tests/functional/constants.py b/pulp_python/tests/functional/constants.py index 92804ecf3..ce69f2564 100644 --- a/pulp_python/tests/functional/constants.py +++ b/pulp_python/tests/functional/constants.py @@ -223,8 +223,8 @@ "download_url": "UNKNOWN", "author": "Austin Macdonald", "author_email": "asmacdo@gmail.com", - "maintainer": None, - "maintainer_email": None, + "maintainer": "", + "maintainer_email": "", # "license": "GNU GENERAL PUBLIC LICENSE Version 2, June 1991", "requires_python": None, "project_url": "https://pypi.org/project/shelf-reader/",