From 8142acfdfda73e136257aa3872004b2f7501ef81 Mon Sep 17 00:00:00 2001 From: Fabricio Aguiar Date: Tue, 24 Aug 2021 17:11:23 -0300 Subject: [PATCH] Update CI + add upgrade tests closes #408 --- .ci/scripts/validate_commit_message.py | 3 +- .github/template_gitref | 2 +- .github/workflows/ci.yml | 75 ++++++++++ .github/workflows/scripts/before_install.sh | 24 ++++ .github/workflows/scripts/check_commit.sh | 2 +- .github/workflows/scripts/script.sh | 50 +++++++ CHANGES/408.misc | 1 + docs/_scripts/base.sh | 9 +- pulp_python/app/utils.py | 23 ++- pulp_python/tests/upgrade/__init__.py | 0 pulp_python/tests/upgrade/post/__init__.py | 0 .../tests/upgrade/post/test_publish.py | 131 ++++++++++++++++++ pulp_python/tests/upgrade/pre/__init__.py | 0 pulp_python/tests/upgrade/pre/test_publish.py | 128 +++++++++++++++++ template_config.yml | 4 +- 15 files changed, 442 insertions(+), 10 deletions(-) create mode 100644 CHANGES/408.misc create mode 100644 pulp_python/tests/upgrade/__init__.py create mode 100644 pulp_python/tests/upgrade/post/__init__.py create mode 100644 pulp_python/tests/upgrade/post/test_publish.py create mode 100644 pulp_python/tests/upgrade/pre/__init__.py create mode 100644 pulp_python/tests/upgrade/pre/test_publish.py diff --git a/.ci/scripts/validate_commit_message.py b/.ci/scripts/validate_commit_message.py index 59073fe63..caf62d86b 100755 --- a/.ci/scripts/validate_commit_message.py +++ b/.ci/scripts/validate_commit_message.py @@ -11,6 +11,7 @@ from pathlib import Path +import os from github import Github @@ -22,7 +23,7 @@ sha = sys.argv[1] message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8") -g = Github() +g = Github(os.environ.get("GITHUB_TOKEN")) repo = g.get_repo("pulp/pulp_python") diff --git a/.github/template_gitref b/.github/template_gitref index 7756632c3..e5dfb51e1 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.04.08-95-gbf7866c +2021.04.08-97-g74b81ba diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 73a6ff365..f1c1d41ab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,6 +33,7 @@ jobs: - name: Check commit message if: github.event_name == 'pull_request' env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} run: sh .github/workflows/scripts/check_commit.sh @@ -133,5 +134,79 @@ jobs: docker exec pulp cat /etc/yum.repos.d/* || true docker exec pulp pip3 list + upgrade: + runs-on: ubuntu-latest + needs: lint + strategy: + fail-fast: false + matrix: + include: + - FROM_PULPCORE_BRANCH: "3.11" + FROM_PULP_PYTHON_BRANCH: "3.2" + + steps: + - uses: actions/checkout@v2 + with: + # by default, it uses a depth of 1 + # this fetches all history so that we can read each commit + fetch-depth: 0 + + - uses: actions/setup-python@v2 + with: + python-version: "3.8" + + - name: Install httpie + run: | + echo ::group::HTTPIE + sudo apt-get update -yq + sudo -E apt-get -yq --no-install-suggests --no-install-recommends install httpie + echo ::endgroup:: + echo "TEST=upgrade" >> $GITHUB_ENV + echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV + echo "FROM_PULP_PYTHON_BRANCH=${{ matrix.FROM_PULP_PYTHON_BRANCH }}" >> $GITHUB_ENV + echo "FROM_PULPCORE_BRANCH=${{ matrix.FROM_PULPCORE_BRANCH }}" >> $GITHUB_ENV + + - name: Install python dependencies + run: | + echo ::group::PYDEPS + pip install wheel + echo ::endgroup:: + + - name: Before Install + run: .github/workflows/scripts/before_install.sh + shell: bash + + - name: Install + run: .github/workflows/scripts/install.sh + env: + PY_COLORS: '1' + ANSIBLE_FORCE_COLOR: '1' + shell: bash + + - name: Install Python client + run: .github/workflows/scripts/install_python_client.sh + + - name: Before Script + run: .github/workflows/scripts/before_script.sh + + - name: Setting secrets + run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" + env: + SECRETS_CONTEXT: ${{ toJson(secrets) }} + + - name: Upgrade test + run: .github/workflows/scripts/script.sh + + - name: After failure + if: failure() + run: | + http --timeout 30 --check-status --pretty format --print hb https://pulp/pulp/api/v3/status/ || true + docker images || true + docker ps -a || true + docker logs pulp || true + docker exec pulp ls -latr /etc/yum.repos.d/ || true + docker exec pulp cat /etc/yum.repos.d/* || true + docker exec pulp pip3 list + diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index 677a640b2..e71a5b012 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -31,6 +31,7 @@ COMMIT_MSG=$(git log --format=%B --no-merges -1) export COMMIT_MSG if [[ "$TEST" == "upgrade" ]]; then + pip install -r functest_requirements.txt git checkout -b ci_upgrade_test cp -R .github /tmp/.github cp -R .ci /tmp/.ci @@ -103,6 +104,21 @@ if [ -n "$PULP_OPENAPI_GENERATOR_PR_NUMBER" ]; then fi +git clone --depth=1 https://github.com/pulp/pulp-cli.git +if [ -n "$PULP_CLI_PR_NUMBER" ]; then + cd pulp-cli + git fetch origin pull/$PULP_CLI_PR_NUMBER/head:$PULP_CLI_PR_NUMBER + git checkout $PULP_CLI_PR_NUMBER + cd .. +fi + +cd pulp-cli +pip install -e . +pulp config create --base-url https://pulp --location tests/cli.toml +mkdir ~/.config/pulp +cp tests/cli.toml ~/.config/pulp/cli.toml +cd .. + git clone --depth=1 https://github.com/pulp/pulpcore.git --branch master @@ -115,6 +131,14 @@ cd .. +if [[ "$TEST" == "upgrade" ]]; then + cd pulpcore + git checkout -b ci_upgrade_test + git fetch --depth=1 origin heads/$FROM_PULPCORE_BRANCH:$FROM_PULPCORE_BRANCH + git checkout $FROM_PULPCORE_BRANCH + cd .. +fi + # Intall requirements for ansible playbooks pip install docker netaddr boto3 ansible diff --git a/.github/workflows/scripts/check_commit.sh b/.github/workflows/scripts/check_commit.sh index 5647c1a48..7780e8be0 100755 --- a/.github/workflows/scripts/check_commit.sh +++ b/.github/workflows/scripts/check_commit.sh @@ -19,7 +19,7 @@ pip3 install pygithub echo ::endgroup:: -for sha in $(curl $GITHUB_CONTEXT | jq '.[].sha' | sed 's/"//g') +for sha in $(curl -H "Authorization: token $GITHUB_TOKEN" $GITHUB_CONTEXT | jq '.[].sha' | sed 's/"//g') do python3 .ci/scripts/validate_commit_message.py $sha VALUE=$? diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index d798cf13c..40309d0ff 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -99,6 +99,56 @@ cmd_prefix bash -c "PULP_DATABASES__default__USER=postgres django-admin test --n export PYTHONPATH=$REPO_ROOT:$REPO_ROOT/../pulpcore${PYTHONPATH:+:${PYTHONPATH}} +if [[ "$TEST" == "upgrade" ]]; then + # Handle app label change: + sed -i "/require_pulp_plugins(/d" pulp_python/tests/functional/utils.py + + # Running pre upgrade tests: + pytest -v -r sx --color=yes --pyargs -capture=no pulp_python.tests.upgrade.pre + + # Checking out ci_upgrade_test branch and upgrading plugins + cmd_prefix bash -c "cd pulpcore; git checkout -f ci_upgrade_test; pip install --upgrade --force-reinstall ." + cmd_prefix bash -c "cd pulp_python; git checkout -f ci_upgrade_test; pip install ." + + # Migrating + cmd_prefix bash -c "django-admin migrate --no-input" + + # Restarting single container services + cmd_prefix bash -c "s6-svc -r /var/run/s6/services/pulpcore-api" + cmd_prefix bash -c "s6-svc -r /var/run/s6/services/pulpcore-content" + cmd_prefix bash -c "s6-svc -d /var/run/s6/services/pulpcore-resource-manager" + cmd_prefix bash -c "s6-svc -d /var/run/s6/services/pulpcore-worker@1" + cmd_prefix bash -c "s6-svc -d /var/run/s6/services/pulpcore-worker@2" + cmd_prefix bash -c "s6-svc -u /var/run/s6/services/new-pulpcore-resource-manager" + cmd_prefix bash -c "s6-svc -u /var/run/s6/services/new-pulpcore-worker@1" + cmd_prefix bash -c "s6-svc -u /var/run/s6/services/new-pulpcore-worker@2" + + echo "Restarting in 60 seconds" + sleep 60 + + # CLI commands to display plugin versions and content data + pulp status + pulp content list + CONTENT_LENGTH=$(pulp content list | jq length) + if [[ "$CONTENT_LENGTH" == "0" ]]; then + echo "Empty content list" + exit 1 + fi + + # Rebuilding bindings + cd ../pulp-openapi-generator + ./generate.sh pulpcore python + pip install ./pulpcore-client + ./generate.sh pulp_python python + pip install ./pulp_python-client + cd $REPO_ROOT + + # Running post upgrade tests + git checkout ci_upgrade_test -- pulp_python/tests/ + pytest -v -r sx --color=yes --pyargs -capture=no pulp_python.tests.upgrade.post + exit +fi + if [[ "$TEST" == "performance" ]]; then if [[ -z ${PERFORMANCE_TEST+x} ]]; then diff --git a/CHANGES/408.misc b/CHANGES/408.misc new file mode 100644 index 000000000..90971a2f5 --- /dev/null +++ b/CHANGES/408.misc @@ -0,0 +1 @@ +Adding upgrade tests diff --git a/docs/_scripts/base.sh b/docs/_scripts/base.sh index 067cfcae3..b80b5d101 100755 --- a/docs/_scripts/base.sh +++ b/docs/_scripts/base.sh @@ -17,10 +17,13 @@ if [ -z "$(pip freeze | grep pulp-cli)" ]; then fi # Set up CLI config file -mkdir ~/.config/pulp -cat > ~/.config/pulp/settings.toml << EOF +if [ ! -f ~/.config/pulp/settings.toml ]; then + echo "Configuring pulp-cli" + mkdir -p ~/.config/pulp + cat > ~/.config/pulp/cli.toml << EOF [cli] -base_url = "$BASE_ADDR" # common to be localhost +base_url = "$BASE_ADDR" verify_ssl = false format = "json" EOF +fi diff --git a/pulp_python/app/utils.py b/pulp_python/app/utils.py index 8e83619dc..2eef63c9d 100644 --- a/pulp_python/app/utils.py +++ b/pulp_python/app/utils.py @@ -150,6 +150,23 @@ def latest_content_version(content_query, version): return latest_content +def json_to_dict(data): + """ + Converts a JSON string into a Python dictionary. + + Args: + data (string): JSON string + + Returns: + dictionary: of JSON string + + """ + if isinstance(data, dict): + return data + + return json.loads(data) + + def python_content_to_info(content): """ Takes in a PythonPackageContent instance and returns a dictionary of the Info fields @@ -175,10 +192,10 @@ def python_content_to_info(content): "package_url": content.project_url or "", # These two are usually identical "project_url": content.project_url or "", # They also usually point to PyPI "release_url": f"{content.project_url}{content.version}/" if content.project_url else "", - "project_urls": json.loads(content.project_urls) or None, + "project_urls": json_to_dict(content.project_urls) or None, "platform": content.platform or "", - "requires_dist": json.loads(content.requires_dist) or None, - "classifiers": json.loads(content.classifiers) or None, + "requires_dist": json_to_dict(content.requires_dist) or None, + "classifiers": json_to_dict(content.classifiers) or None, "yanked": False, # These are no longer used on PyPI, but are still present "yanked_reason": None, } diff --git a/pulp_python/tests/upgrade/__init__.py b/pulp_python/tests/upgrade/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pulp_python/tests/upgrade/post/__init__.py b/pulp_python/tests/upgrade/post/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pulp_python/tests/upgrade/post/test_publish.py b/pulp_python/tests/upgrade/post/test_publish.py new file mode 100644 index 000000000..c6b7d46e5 --- /dev/null +++ b/pulp_python/tests/upgrade/post/test_publish.py @@ -0,0 +1,131 @@ +"""Tests that publish file plugin repositories.""" +import unittest +from random import choice + +from pulp_smash import config +from pulp_smash.pulp3.bindings import monitor_task +from pulp_smash.pulp3.utils import ( + gen_repo, + get_content, + gen_distribution, + get_versions, +) + +from pulp_python.tests.functional.constants import PYTHON_CONTENT_NAME +from pulp_python.tests.functional.utils import ( + gen_python_client, + gen_python_remote, +) +from pulp_python.tests.functional.utils import set_up_module as setUpModule # noqa:F401 + +from pulpcore.client.pulp_python import ( + DistributionsPypiApi, + PublicationsPypiApi, + RepositoriesPythonApi, + RepositoryAddRemoveContent, + RepositorySyncURL, + RemotesPythonApi, + PythonPythonPublication, +) +from pulpcore.client.pulp_python.exceptions import ApiException + + +class PublishAnyRepoVersionTestCase(unittest.TestCase): + """Test whether a particular repository version can be published. + + This test targets the following issues: + + * `Pulp #3324 `_ + * `Pulp Smash #897 `_ + """ + + @classmethod + def setUpClass(cls): + """Create class-wide variables.""" + cls.cfg = config.get_config() + + client = gen_python_client() + cls.repo_api = RepositoriesPythonApi(client) + cls.remote_api = RemotesPythonApi(client) + cls.publications = PublicationsPypiApi(client) + cls.distributions = DistributionsPypiApi(client) + + def setUp(self): + """Create a new repository before each test.""" + pypi_path = "/pypi/pulp_pre_upgrade_test" + url = self.cfg.get_content_host_base_url() + pypi_path + body = gen_python_remote(url=url) + remote = self.remote_api.create(body) + + repo = self.repo_api.create(gen_repo()) + + repository_sync_data = RepositorySyncURL(remote=remote.pulp_href) + sync_response = self.repo_api.sync(repo.pulp_href, repository_sync_data) + monitor_task(sync_response.task) + + self.repo = self.repo_api.read(repo.pulp_href) + + def test_all(self): + """Test whether a particular repository version can be published. + + 1. Create a repository with at least 2 repository versions. + 2. Create a publication by supplying the latest ``repository_version``. + 3. Assert that the publication ``repository_version`` attribute points + to the latest repository version. + 4. Create a publication by supplying the non-latest ``repository_version``. + 5. Create distribution. + 6. Assert that the publication ``repository_version`` attribute points + to the supplied repository version. + 7. Assert that an exception is raised when providing two different + repository versions to be published at same time. + """ + # Step 1 + repo_content = get_content(self.repo.to_dict())[PYTHON_CONTENT_NAME] + print(repo_content) + for file_content in repo_content: + repository_modify_data = RepositoryAddRemoveContent( + remove_content_units=[file_content["pulp_href"]] + ) + modify_response = self.repo_api.modify(self.repo.pulp_href, repository_modify_data) + monitor_task(modify_response.task) + version_hrefs = tuple(ver["pulp_href"] for ver in get_versions(self.repo.to_dict())) + print(version_hrefs) + non_latest = choice(version_hrefs[:-1]) + + # Step 2 + publish_data = PythonPythonPublication(repository=self.repo.pulp_href) + publication = self.create_publication(publish_data) + + # Step 3 + self.assertEqual(publication.repository_version, version_hrefs[-1]) + + # Step 4 + publish_data = PythonPythonPublication(repository_version=non_latest) + publication = self.create_publication(publish_data) + + # Step 5 + body = gen_distribution() + body["base_path"] = "pulp_post_upgrade_test" + body["publication"] = publication.pulp_href + + distribution_response = self.distributions.create(body) + created_resources = monitor_task(distribution_response.task).created_resources + distribution = self.distributions.read(created_resources[0]) + + # Step 6 + self.assertEqual(publication.repository_version, non_latest) + + # Step 7 + with self.assertRaises(ApiException): + body = {"repository": self.repo.pulp_href, "repository_version": non_latest} + self.publications.create(body) + + # Step 8 + self.assertIn("/pypi/pulp_post_upgrade_test/", distribution.base_url) + + def create_publication(self, publish_data): + """Create a new publication from the passed data.""" + publish_response = self.publications.create(publish_data) + created_resources = monitor_task(publish_response.task).created_resources + publication_href = created_resources[0] + return self.publications.read(publication_href) diff --git a/pulp_python/tests/upgrade/pre/__init__.py b/pulp_python/tests/upgrade/pre/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pulp_python/tests/upgrade/pre/test_publish.py b/pulp_python/tests/upgrade/pre/test_publish.py new file mode 100644 index 000000000..899155a26 --- /dev/null +++ b/pulp_python/tests/upgrade/pre/test_publish.py @@ -0,0 +1,128 @@ +"""Tests that publish file plugin repositories.""" +import unittest +from random import choice + +from pulp_smash import config +from pulp_smash.pulp3.bindings import monitor_task +from pulp_smash.pulp3.utils import ( + gen_repo, + get_content, + gen_distribution, + get_versions, +) + +from pulp_python.tests.functional.constants import PYTHON_CONTENT_NAME +from pulp_python.tests.functional.utils import ( + gen_python_client, + gen_python_remote, +) +from pulp_python.tests.functional.utils import set_up_module as setUpModule # noqa:F401 + +from pulpcore.client.pulp_python import ( + DistributionsPypiApi, + PublicationsPypiApi, + RepositoryAddRemoveContent, + RepositoriesPythonApi, + RepositorySyncURL, + RemotesPythonApi, + PythonPythonPublication, +) +from pulpcore.client.pulp_python.exceptions import ApiException + + +class PublishAnyRepoVersionTestCase(unittest.TestCase): + """Test whether a particular repository version can be published. + + This test targets the following issues: + + * `Pulp #3324 `_ + * `Pulp Smash #897 `_ + """ + + @classmethod + def setUpClass(cls): + """Create class-wide variables.""" + cls.cfg = config.get_config() + + client = gen_python_client() + cls.repo_api = RepositoriesPythonApi(client) + cls.remote_api = RemotesPythonApi(client) + cls.publications = PublicationsPypiApi(client) + cls.distributions = DistributionsPypiApi(client) + + def setUp(self): + """Create a new repository before each test.""" + body = gen_python_remote() + remote = self.remote_api.create(body) + + repo = self.repo_api.create(gen_repo()) + + repository_sync_data = RepositorySyncURL(remote=remote.pulp_href) + sync_response = self.repo_api.sync(repo.pulp_href, repository_sync_data) + monitor_task(sync_response.task) + + self.repo = self.repo_api.read(repo.pulp_href) + + def test_all(self): + """Test whether a particular repository version can be published. + + 1. Create a repository with at least 2 repository versions. + 2. Create a publication by supplying the latest ``repository_version``. + 3. Assert that the publication ``repository_version`` attribute points + to the latest repository version. + 4. Create a publication by supplying the non-latest ``repository_version``. + 5. Create distribution. + 6. Assert that the publication ``repository_version`` attribute points + to the supplied repository version. + 7. Assert that an exception is raised when providing two different + repository versions to be published at same time. + """ + # Step 1 + repo_content = get_content(self.repo.to_dict())[PYTHON_CONTENT_NAME][:-1] + for file_content in repo_content: + repository_modify_data = RepositoryAddRemoveContent( + remove_content_units=[file_content["pulp_href"]] + ) + modify_response = self.repo_api.modify(self.repo.pulp_href, repository_modify_data) + monitor_task(modify_response.task) + version_hrefs = tuple(ver["pulp_href"] for ver in get_versions(self.repo.to_dict())) + non_latest = choice(version_hrefs[1:-1]) + + # Step 2 + publish_data = PythonPythonPublication(repository=self.repo.pulp_href) + publication = self.create_publication(publish_data) + + # Step 3 + self.assertEqual(publication.repository_version, version_hrefs[-1]) + + # Step 4 + publish_data = PythonPythonPublication(repository_version=non_latest) + publication = self.create_publication(publish_data) + + # Step 5 + body = gen_distribution() + body["base_path"] = "pulp_pre_upgrade_test" + body["publication"] = publication.pulp_href + + distribution_response = self.distributions.create(body) + created_resources = monitor_task(distribution_response.task).created_resources + distribution = self.distributions.read(created_resources[0]) + + # Step 6 + self.assertEqual(publication.repository_version, non_latest) + + # Step 7 + with self.assertRaises(ApiException): + body = {"repository": self.repo.pulp_href, "repository_version": non_latest} + self.publications.create(body) + + # Step 8 + url = self.cfg.get_content_host_base_url() + "/pulp/content/pulp_pre_upgrade_test/" + self.assertEqual(url, distribution.base_url, url) + + def create_publication(self, publish_data): + """Create a new publication from the passed data.""" + publish_response = self.publications.create(publish_data) + created_resources = monitor_task(publish_response.task).created_resources + publication_href = created_resources[0] + return self.publications.read(publication_href) diff --git a/template_config.yml b/template_config.yml index 88a89d7c9..2331d6397 100644 --- a/template_config.yml +++ b/template_config.yml @@ -61,5 +61,7 @@ test_s3: true travis_addtl_services: [] travis_notifications: None update_redmine: false -upgrade_range: [] +upgrade_range: +- pulp_python_branch: 3.2 + pulpcore_branch: 3.11