From f9ef2ea789a01b0e01311acb3f2d3d5bdb689024 Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Wed, 10 Nov 2021 18:20:18 -0500 Subject: [PATCH] Add simple page parsing for full syncs fixes: #462 --- .github/workflows/scripts/install.sh | 2 +- CHANGES/462.feature | 1 + docs/tech-preview.rst | 2 +- pulp_python/app/tasks/sync.py | 43 ++++++++++------- .../functional/api/test_download_content.py | 46 +++++++++++++++++-- template_config.yml | 1 + 6 files changed, 73 insertions(+), 22 deletions(-) create mode 100644 CHANGES/462.feature diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 80a9d18ec..db093b5f3 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -71,7 +71,7 @@ services: VARSYAML cat >> vars/main.yaml << VARSYAML -pulp_settings: {"orphan_protection_time": 0} +pulp_settings: {"orphan_protection_time": 0, "pypi_api_hostname": "https://pulp:443"} pulp_scheme: https pulp_container_tag: https diff --git a/CHANGES/462.feature b/CHANGES/462.feature new file mode 100644 index 000000000..a9490c8de --- /dev/null +++ b/CHANGES/462.feature @@ -0,0 +1 @@ +Added ability to fully sync repositories that don't support the PyPI XMLRPC endpoints. Full Pulp-to-Pulp syncing is now available. diff --git a/docs/tech-preview.rst b/docs/tech-preview.rst index e237efe62..0ba3d9f0a 100644 --- a/docs/tech-preview.rst +++ b/docs/tech-preview.rst @@ -6,6 +6,6 @@ The following features are currently being released as part of a tech preview * New endpoint “pulp/api/v3/remotes/python/python/from_bandersnatch/” that allows for Python remote creation from a Bandersnatch config file. * PyPI’s json API at content endpoint ‘/pypi/{package-name}/json’. Allows for basic Pulp-to-Pulp syncing. -* Fully mirror Python repositories like PyPI. +* Fully mirror Python repositories provided PyPI and Pulp itself. * ``Twine`` upload packages to indexes at endpoints '/simple` or '/legacy'. * Create pull-through caches of remote sources. diff --git a/pulp_python/app/tasks/sync.py b/pulp_python/app/tasks/sync.py index 0867d18c8..8e19b6e2f 100644 --- a/pulp_python/app/tasks/sync.py +++ b/pulp_python/app/tasks/sync.py @@ -1,5 +1,7 @@ import logging +from aiohttp import ClientResponseError, ClientError +from lxml.etree import LxmlError from gettext import gettext as _ from os import environ @@ -17,12 +19,14 @@ PythonPackageContent, PythonRemote, ) -from pulp_python.app.utils import parse_metadata +from pulp_python.app.utils import parse_metadata, PYPI_LAST_SERIAL +from pypi_simple import parse_repo_index_page from bandersnatch.mirror import Mirror from bandersnatch.master import Master from bandersnatch.configuration import BandersnatchConfig from packaging.requirements import Requirement +from urllib.parse import urljoin logger = logging.getLogger(__name__) @@ -111,8 +115,10 @@ async def run(self): if self.remote.proxy_url: environ['http_proxy'] = self.remote.proxy_url environ['https_proxy'] = self.remote.proxy_url + # Bandersnatch includes leading slash when forming API urls + url = self.remote.url.rstrip("/") # local & global timeouts defaults to 10secs and 5 hours - async with Master(self.remote.url) as master: + async with Master(url) as master: deferred_download = self.remote.policy != Remote.IMMEDIATE workers = self.remote.download_concurrency or self.remote.DEFAULT_DOWNLOAD_CONCURRENCY async with ProgressReport( @@ -179,20 +185,25 @@ async def determine_packages_to_sync(self): self.target_serial = max( [self.synced_serial] + [int(v) for v in self.packages_to_sync.values()] ) - self._filter_packages() - logger.info(f"Trying to reach serial: {self.target_serial}") - pkg_count = len(self.packages_to_sync) - logger.info(f"{pkg_count} packages to sync.") - return - except Exception as e: - """Handle different exceptions if it is XMLRPC error or Mirror error""" - logger.info("Encountered an error in Master {}".format(e)) - pass - """ - If we reach here, then the Mirror most likely doesn't support XMLRPC. - Could raise an exception or try to manually find all the packages from the index page, - Or just keep packages_to_sync empty and have the sync do no work - """ + break + except (ClientError, ClientResponseError, LxmlError): + # Retry if XMLRPC endpoint failed, server might not support it. + continue + else: + logger.info("Failed to get package list using XMLRPC, trying parse simple page.") + url = urljoin(self.python_stage.remote.url, "simple/") + downloader = self.python_stage.remote.get_downloader(url=url) + result = await downloader.run() + with open(result.path) as f: + index = parse_repo_index_page(f.read()) + self.packages_to_sync.update({p: 0 for p in index.projects}) + self.target_serial = result.headers.get(PYPI_LAST_SERIAL, 0) + + self._filter_packages() + if self.target_serial: + logger.info(f"Trying to reach serial: {self.target_serial}") + pkg_count = len(self.packages_to_sync) + logger.info(f"{pkg_count} packages to sync.") async def process_package(self, package): """Filters the package and creates content from it""" diff --git a/pulp_python/tests/functional/api/test_download_content.py b/pulp_python/tests/functional/api/test_download_content.py index 597149b2a..c65327db2 100644 --- a/pulp_python/tests/functional/api/test_download_content.py +++ b/pulp_python/tests/functional/api/test_download_content.py @@ -1,7 +1,6 @@ # coding=utf-8 """Tests that verify download of content served by Pulp.""" import hashlib -import unittest from random import choice from urllib.parse import urljoin @@ -12,6 +11,8 @@ ) from pulp_python.tests.functional.constants import ( PYTHON_FIXTURE_URL, + PYTHON_MD_PROJECT_SPECIFIER, + PYTHON_MD_FIXTURE_SUMMARY, PYTHON_LG_FIXTURE_SUMMARY, PYTHON_LG_PROJECT_SPECIFIER, ) @@ -79,8 +80,8 @@ def test_all(self): self.assertEqual(fixtures_hashes, pulp_hashes) -class PublishPyPiJSON(TestCaseUsingBindings, TestHelpersMixin): - """Test whether a distributed Python repository has a PyPi json endpoint +class PublishPyPIJSON(TestCaseUsingBindings, TestHelpersMixin): + """Test whether a distributed Python repository has a PyPI json endpoint a.k.a Can be consumed by another Pulp instance Test targets the following issue: @@ -88,7 +89,6 @@ class PublishPyPiJSON(TestCaseUsingBindings, TestHelpersMixin): * `Pulp #2886 `_ """ - @unittest.skip("Content can not be synced without https") def test_basic_pulp_to_pulp_sync(self): """ This test checks that the JSON endpoint is setup correctly to allow one Pulp instance @@ -107,7 +107,45 @@ def test_basic_pulp_to_pulp_sync(self): ] unit_url = "/".join(url_fragments) + # Sync using old Pulp content api body["url"] = unit_url remote = self._create_remote(**body) repo2 = self._create_repo_and_sync_with_remote(remote) self.assertEqual(get_content_summary(repo2.to_dict()), PYTHON_LG_FIXTURE_SUMMARY) + + # Sync using new PyPI endpoints + body["url"] = distro.base_url + remote = self._create_remote(**body) + repo3 = self._create_repo_and_sync_with_remote(remote) + self.assertEqual(get_content_summary(repo3.to_dict()), PYTHON_LG_FIXTURE_SUMMARY) + + def test_full_fixtures_to_pulp_sync(self): + """ + This test checks that Pulp can fully sync another Python Package repository that is not + PyPI. This reads the repository's simple page if XMLRPC isn't supported. + """ + remote = self._create_remote(includes="", prereleases=True) + repo = self._create_repo_and_sync_with_remote(remote) + self.assertEqual(get_content_summary(repo.to_dict()), PYTHON_LG_FIXTURE_SUMMARY) + + def test_full_pulp_to_pulp_sync(self): + """ + This test checks that Pulp can fully sync all packages from another Pulp instance + without having to specify the includes field. + """ + remote = self._create_remote(includes=PYTHON_MD_PROJECT_SPECIFIER) + repo = self._create_repo_and_sync_with_remote(remote) + # Test using live generated simple pages + distro = self._create_distribution_from_repo(repo) + + remote = self._create_remote(includes="", url=distro.base_url) + repo2 = self._create_repo_and_sync_with_remote(remote) + self.assertEqual(get_content_summary(repo2.to_dict()), PYTHON_MD_FIXTURE_SUMMARY) + + # Now test using publication simple pages + pub = self._create_publication(repo) + distro2 = self._create_distribution_from_publication(pub) + remote = self._create_remote(includes="", url=distro2.base_url, prereleases=True) + + repo3 = self._create_repo_and_sync_with_remote(remote) + self.assertEqual(get_content_summary(repo3.to_dict()), PYTHON_MD_FIXTURE_SUMMARY) diff --git a/template_config.yml b/template_config.yml index f0f86a080..74cc55d9a 100644 --- a/template_config.yml +++ b/template_config.yml @@ -49,6 +49,7 @@ publish_docs_to_pulpprojectdotorg: true pulp_scheme: https pulp_settings: orphan_protection_time: 0 + pypi_api_hostname: https://pulp:443 pulpcore_branch: main pulpcore_pip_version_specifier: null pulpcore_revision: null