Fixed over publishing, improved publication tests, and increased publication speed

CHANGES/347.misc

@@ -0,0 +1 @@
+Improved functional tests for publications

CHANGES/362.bugfix

@@ -0,0 +1 @@
+Fixed publications publishing more content than was in the repository

functest_requirements.txt

@@ -1,2 +1,3 @@
 git+https://github.com/pulp/pulp-smash.git#egg=pulp-smash
 pytest
+lxml

pulp_python/app/migrations/0005_pythonpackagecontent_sha256.py

@@ -0,0 +1,41 @@
+# Generated by Django 2.2.20 on 2021-04-26 16:28
+
+from django.db import migrations, models, transaction
+
+
+def add_sha256_to_current_models(apps, schema_editor):
+    """Adds the sha256 to current PythonPackageContent models."""
+    PythonPackageContent = apps.get_model('python', 'PythonPackageContent')
+    RemoteArtifact = apps.get_model('core', 'RemoteArtifact')
+    package_bulk = []
+    for python_package in PythonPackageContent.objects.only("pk", "sha256").iterator():
+        content_artifact = python_package.contentartifact_set.first()
+        if content_artifact.artifact:
+            artifact = content_artifact.artifact
+        else:
+            artifact = RemoteArtifact.objects.filter(content_artifact=content_artifact).first()
+        python_package.sha256 = artifact.sha256
+        package_bulk.append(python_package)
+        if len(package_bulk) == 100000:
+            with transaction.atomic():
+                PythonPackageContent.objects.bulk_update(package_bulk, ["sha256",])
+                package_bulk = []
+    with transaction.atomic():
+        PythonPackageContent.objects.bulk_update(package_bulk, ["sha256",])
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('python', '0004_DATA_swap_distribution_model'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='pythonpackagecontent',
+            name='sha256',
+            field=models.CharField(max_length=64, default=''),
+            preserve_default=False,
+        ),
+        migrations.RunPython(add_sha256_to_current_models, migrations.RunPython.noop)
+    ]

pulp_python/app/models.py

@@ -93,6 +93,7 @@ class PythonPackageContent(Content):
     packagetype = models.TextField(choices=PACKAGE_TYPES)
     name = models.TextField()
     version = models.TextField()
+    sha256 = models.CharField(max_length=64)
     # Optional metadata
     python_version = models.TextField()
     metadata_version = models.TextField()

pulp_python/app/tasks/publish.py

@@ -61,10 +61,10 @@ def publish(repository_version_pk):
     ))
 
     with tempfile.TemporaryDirectory("."):
-        with python_models.PythonPublication.create(repository_version) as publication:
-            write_simple_api(publication)
+        with python_models.PythonPublication.create(repository_version, pass_through=True) as pub:
+            write_simple_api(pub)
 
-    log.info(_('Publication: {pk} created').format(pk=publication.pk))
+    log.info(_('Publication: {pk} created').format(pk=pub.pk))
 
 
 def write_simple_api(publication):
@@ -107,46 +107,58 @@ def write_simple_api(publication):
     )
     index_metadata.save()
 
-    def find_artifact():
-        _art = content_artifact.artifact
-        if not _art:
-            _art = models.RemoteArtifact.objects.filter(content_artifact=content_artifact).first()
-        return _art
-
-    for (name, canonical_name) in index_names:
-        project_dir = '{simple_dir}{name}/'.format(simple_dir=simple_dir, name=canonical_name)
-        os.mkdir(project_dir)
-
-        packages = python_models.PythonPackageContent.objects.filter(name=name)
-        package_detail_data = []
-        for package in packages.iterator():
-            artifact_set = package.contentartifact_set.all()
-            for content_artifact in artifact_set:
-                artifact = find_artifact()
-                published_artifact = models.PublishedArtifact(
-                    relative_path=content_artifact.relative_path,
-                    publication=publication,
-                    content_artifact=content_artifact
-                )
-                published_artifact.save()
-
-                checksum = artifact.sha256
-                path = "../../{}".format(package.filename)
-                package_detail_data.append((package.filename, path, checksum))
-
-        metadata_relative_path = '{project_dir}index.html'.format(project_dir=project_dir)
-
-        with open(metadata_relative_path, 'w') as simple_metadata:
-            context = Context({
-                'project_name': name,
-                'project_packages': package_detail_data
-            })
-            template = Template(simple_detail_template)
-            simple_metadata.write(template.render(context))
-
-        project_metadata = models.PublishedMetadata.create_from_file(
-            relative_path=metadata_relative_path,
-            publication=publication,
-            file=File(open(metadata_relative_path, 'rb'))
-        )
-        project_metadata.save()
+    if len(index_names) == 0:
+        return
+
+    packages = python_models.PythonPackageContent.objects.filter(
+        pk__in=publication.repository_version.content
+    )
+    releases = packages.order_by("name").values("name", "filename", "sha256")
+
+    ind = 0
+    current_name = index_names[ind][0]
+    package_releases = []
+    for release in releases.iterator():
+        if release['name'] != current_name:
+            write_project_page(
+                name=index_names[ind][1],
+                simple_dir=simple_dir,
+                package_releases=package_releases,
+                publication=publication
+            )
+            package_releases = []
+            ind += 1
+            current_name = index_names[ind][0]
+        relative_path = release['filename']
+        path = f"../../{relative_path}"
+        checksum = release['sha256']
+        package_releases.append((relative_path, path, checksum))
+    # Write the final project's page
+    write_project_page(
+        name=index_names[ind][1],
+        simple_dir=simple_dir,
+        package_releases=package_releases,
+        publication=publication
+    )
+
+
+def write_project_page(name, simple_dir, package_releases, publication):
+    """Writes a project's simple page."""
+    project_dir = f'{simple_dir}{name}/'
+    os.mkdir(project_dir)
+    metadata_relative_path = f'{project_dir}index.html'
+
+    with open(metadata_relative_path, 'w') as simple_metadata:
+        context = Context({
+            'project_name': name,
+            'project_packages': package_releases
+        })
+        template = Template(simple_detail_template)
+        simple_metadata.write(template.render(context))
+
+    project_metadata = models.PublishedMetadata.create_from_file(
+        relative_path=metadata_relative_path,
+        publication=publication,
+        file=File(open(metadata_relative_path, 'rb'))
+    )
+    project_metadata.save()  # change to bulk create when multi-table supported

pulp_python/app/tasks/sync.py

@@ -215,7 +215,7 @@ async def create_content(self, pkg):
                 entry = parse_metadata(pkg.info, version, package)
                 url = entry.pop("url")
 
-                artifact = Artifact(sha256=entry.pop("sha256_digest"))
+                artifact = Artifact(sha256=entry["sha256"])
                 package = PythonPackageContent(**entry)
 
                 da = DeclarativeArtifact(

pulp_python/app/utils.py

@@ -67,7 +67,7 @@ def parse_metadata(project, version, distribution):
     package['packagetype'] = distribution.get('packagetype') or ""
     package['version'] = version
     package['url'] = distribution.get('url') or ""
-    package['sha256_digest'] = distribution.get('digests', {}).get('sha256') or ""
+    package['sha256'] = distribution.get('digests', {}).get('sha256') or ""
     package['python_version'] = distribution.get('python_version') or ""
 
     package.update(parse_project_metadata(project))