Add simple page parsing for full syncs

.github/workflows/scripts/install.sh

@@ -71,7 +71,7 @@ services:
 VARSYAML
 
 cat >> vars/main.yaml << VARSYAML
-pulp_settings: {"orphan_protection_time": 0}
+pulp_settings: {"orphan_protection_time": 0, "pypi_api_hostname": "https://pulp:443"}
 pulp_scheme: https
 
 pulp_container_tag: https

CHANGES/462.feature

@@ -0,0 +1 @@
+Added ability to fully sync repositories that don't support the PyPI XMLRPC endpoints. Full Pulp-to-Pulp syncing is now available.

docs/tech-preview.rst

@@ -6,6 +6,6 @@ The following features are currently being released as part of a tech preview
 * New endpoint “pulp/api/v3/remotes/python/python/from_bandersnatch/” that allows for Python remote creation from a
   Bandersnatch config file.
 * PyPI’s json API at content endpoint ‘/pypi/{package-name}/json’. Allows for basic Pulp-to-Pulp syncing.
-* Fully mirror Python repositories like PyPI.
+* Fully mirror Python repositories provided PyPI and Pulp itself.
 * ``Twine`` upload packages to indexes at endpoints '/simple` or '/legacy'.
 * Create pull-through caches of remote sources.

pulp_python/app/tasks/sync.py

@@ -1,5 +1,7 @@
 import logging
 
+from aiohttp import ClientResponseError, ClientError
+from lxml.etree import LxmlError
 from gettext import gettext as _
 from os import environ
 
@@ -17,12 +19,14 @@
     PythonPackageContent,
     PythonRemote,
 )
-from pulp_python.app.utils import parse_metadata
+from pulp_python.app.utils import parse_metadata, PYPI_LAST_SERIAL
+from pypi_simple import parse_repo_index_page
 
 from bandersnatch.mirror import Mirror
 from bandersnatch.master import Master
 from bandersnatch.configuration import BandersnatchConfig
 from packaging.requirements import Requirement
+from urllib.parse import urljoin
 
 logger = logging.getLogger(__name__)
 
@@ -111,8 +115,10 @@ async def run(self):
         if self.remote.proxy_url:
             environ['http_proxy'] = self.remote.proxy_url
             environ['https_proxy'] = self.remote.proxy_url
+        # Bandersnatch includes leading slash when forming API urls
+        url = self.remote.url.rstrip("/")
         # local & global timeouts defaults to 10secs and 5 hours
-        async with Master(self.remote.url) as master:
+        async with Master(url) as master:
             deferred_download = self.remote.policy != Remote.IMMEDIATE
             workers = self.remote.download_concurrency or self.remote.DEFAULT_DOWNLOAD_CONCURRENCY
             async with ProgressReport(
@@ -179,20 +185,25 @@ async def determine_packages_to_sync(self):
                     self.target_serial = max(
                         [self.synced_serial] + [int(v) for v in self.packages_to_sync.values()]
                     )
-                self._filter_packages()
-                logger.info(f"Trying to reach serial: {self.target_serial}")
-                pkg_count = len(self.packages_to_sync)
-                logger.info(f"{pkg_count} packages to sync.")
-                return
-            except Exception as e:
-                """Handle different exceptions if it is XMLRPC error or Mirror error"""
-                logger.info("Encountered an error in Master {}".format(e))
-                pass
-        """
-        If we reach here, then the Mirror most likely doesn't support XMLRPC.
-        Could raise an exception or try to manually find all the packages from the index page,
-        Or just keep packages_to_sync empty and have the sync do no work
-        """
+                break
+            except (ClientError, ClientResponseError, LxmlError):
+                # Retry if XMLRPC endpoint failed, server might not support it.
+                continue
+        else:
+            logger.info("Failed to get package list using XMLRPC, trying parse simple page.")
+            url = urljoin(self.python_stage.remote.url, "simple/")
+            downloader = self.python_stage.remote.get_downloader(url=url)
+            result = await downloader.run()
+            with open(result.path) as f:
+                index = parse_repo_index_page(f.read())
+                self.packages_to_sync.update({p: 0 for p in index.projects})
+                self.target_serial = result.headers.get(PYPI_LAST_SERIAL, 0)
+
+        self._filter_packages()
+        if self.target_serial:
+            logger.info(f"Trying to reach serial: {self.target_serial}")
+        pkg_count = len(self.packages_to_sync)
+        logger.info(f"{pkg_count} packages to sync.")
 
     async def process_package(self, package):
         """Filters the package and creates content from it"""

pulp_python/tests/functional/api/test_download_content.py

@@ -1,7 +1,6 @@
 # coding=utf-8
 """Tests that verify download of content served by Pulp."""
 import hashlib
-import unittest
 from random import choice
 from urllib.parse import urljoin
 
@@ -12,6 +11,8 @@
 )
 from pulp_python.tests.functional.constants import (
     PYTHON_FIXTURE_URL,
+    PYTHON_MD_PROJECT_SPECIFIER,
+    PYTHON_MD_FIXTURE_SUMMARY,
     PYTHON_LG_FIXTURE_SUMMARY,
     PYTHON_LG_PROJECT_SPECIFIER,
 )
@@ -79,16 +80,15 @@ def test_all(self):
         self.assertEqual(fixtures_hashes, pulp_hashes)
 
 
-class PublishPyPiJSON(TestCaseUsingBindings, TestHelpersMixin):
-    """Test whether a distributed Python repository has a PyPi json endpoint
+class PublishPyPIJSON(TestCaseUsingBindings, TestHelpersMixin):
+    """Test whether a distributed Python repository has a PyPI json endpoint
     a.k.a Can be consumed by another Pulp instance
 
     Test targets the following issue:
 
     * `Pulp #2886 <https://pulp.plan.io/issues/2886>`_
     """
 
-    @unittest.skip("Content can not be synced without https")
     def test_basic_pulp_to_pulp_sync(self):
         """
         This test checks that the JSON endpoint is setup correctly to allow one Pulp instance
@@ -107,7 +107,45 @@ def test_basic_pulp_to_pulp_sync(self):
         ]
         unit_url = "/".join(url_fragments)
 
+        # Sync using old Pulp content api
         body["url"] = unit_url
         remote = self._create_remote(**body)
         repo2 = self._create_repo_and_sync_with_remote(remote)
         self.assertEqual(get_content_summary(repo2.to_dict()), PYTHON_LG_FIXTURE_SUMMARY)
+
+        # Sync using new PyPI endpoints
+        body["url"] = distro.base_url
+        remote = self._create_remote(**body)
+        repo3 = self._create_repo_and_sync_with_remote(remote)
+        self.assertEqual(get_content_summary(repo3.to_dict()), PYTHON_LG_FIXTURE_SUMMARY)
+
+    def test_full_fixtures_to_pulp_sync(self):
+        """
+        This test checks that Pulp can fully sync another Python Package repository that is not
+        PyPI. This reads the repository's simple page if XMLRPC isn't supported.
+        """
+        remote = self._create_remote(includes="", prereleases=True)
+        repo = self._create_repo_and_sync_with_remote(remote)
+        self.assertEqual(get_content_summary(repo.to_dict()), PYTHON_LG_FIXTURE_SUMMARY)
+
+    def test_full_pulp_to_pulp_sync(self):
+        """
+        This test checks that Pulp can fully sync all packages from another Pulp instance
+        without having to specify the includes field.
+        """
+        remote = self._create_remote(includes=PYTHON_MD_PROJECT_SPECIFIER)
+        repo = self._create_repo_and_sync_with_remote(remote)
+        # Test using live generated simple pages
+        distro = self._create_distribution_from_repo(repo)
+
+        remote = self._create_remote(includes="", url=distro.base_url)
+        repo2 = self._create_repo_and_sync_with_remote(remote)
+        self.assertEqual(get_content_summary(repo2.to_dict()), PYTHON_MD_FIXTURE_SUMMARY)
+
+        # Now test using publication simple pages
+        pub = self._create_publication(repo)
+        distro2 = self._create_distribution_from_publication(pub)
+        remote = self._create_remote(includes="", url=distro2.base_url, prereleases=True)
+
+        repo3 = self._create_repo_and_sync_with_remote(remote)
+        self.assertEqual(get_content_summary(repo3.to_dict()), PYTHON_MD_FIXTURE_SUMMARY)

template_config.yml

@@ -49,6 +49,7 @@ publish_docs_to_pulpprojectdotorg: true
 pulp_scheme: https
 pulp_settings:
   orphan_protection_time: 0
+  pypi_api_hostname: https://pulp:443
 pulpcore_branch: main
 pulpcore_pip_version_specifier: null
 pulpcore_revision: null