diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index d6b3a8c03..eaff66a86 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -1,4 +1,4 @@ -FROM {{ ci_base | default("pulp/pulp-ci-centos:latest") }} +FROM {{ ci_base | default("pulp/pulp-ci-centos:" + pulp_container_tag) }} # Add source directories to container {% for item in plugins %} @@ -23,7 +23,8 @@ RUN pip3 install \ RUN mkdir -p /etc/nginx/pulp/ {% for item in plugins %} -RUN ln /usr/local/lib/python3.6/site-packages/{{ item.name }}/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ item.name }}.conf || true +RUN export plugin_path="$(pip3 show {{ item.name }} | sed -n -e 's/Location: //p')/{{ item.name }}" && \ + ln $plugin_path/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ item.name }}.conf || true {% endfor %} ENTRYPOINT ["/init"] diff --git a/.ci/ansible/settings.py.j2 b/.ci/ansible/settings.py.j2 index 56794bc03..e0e040a7a 100644 --- a/.ci/ansible/settings.py.j2 +++ b/.ci/ansible/settings.py.j2 @@ -1,9 +1,9 @@ -CONTENT_ORIGIN = "http://pulp:80" -ANSIBLE_API_HOSTNAME = "http://pulp:80" -ANSIBLE_CONTENT_HOSTNAME = "http://pulp:80/pulp/content" +CONTENT_ORIGIN = "{{ pulp_scheme }}://pulp:{{ 443 if pulp_scheme == 'https' else 80 }}" +ANSIBLE_API_HOSTNAME = "{{ pulp_scheme }}://pulp:{{ 443 if pulp_scheme == 'https' else 80 }}" +ANSIBLE_CONTENT_HOSTNAME = "{{ pulp_scheme }}://pulp:{{ 443 if pulp_scheme == 'https' else 80 }}/pulp/content" PRIVATE_KEY_PATH = "/etc/pulp/certs/token_private_key.pem" PUBLIC_KEY_PATH = "/etc/pulp/certs/token_public_key.pem" -TOKEN_SERVER = "http://pulp:80/token" +TOKEN_SERVER = "{{ pulp_scheme }}://pulp:{{ 443 if pulp_scheme == 'https' else 80 }}/token/" TOKEN_SIGNATURE_ALGORITHM = "ES256" {% if pulp_settings %} diff --git a/.ci/ansible/start_container.yaml b/.ci/ansible/start_container.yaml index d75a7083a..8661bb88d 100644 --- a/.ci/ansible/start_container.yaml +++ b/.ci/ansible/start_container.yaml @@ -71,7 +71,8 @@ - name: "Wait for Pulp" uri: url: "http://pulp/pulp/api/v3/status/" - follow_redirects: none + follow_redirects: all + validate_certs: no register: result until: result.status == 200 retries: 12 diff --git a/.ci/scripts/cherrypick.sh b/.ci/scripts/cherrypick.sh new file mode 100755 index 000000000..d9f7a8be8 --- /dev/null +++ b/.ci/scripts/cherrypick.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +set -e + +if [ ! -d CHANGES ]; then + echo "Error: no CHANGES directory detected. This script must be run from the project root." + exit 1 +fi + +if [ $# -lt 3 ] +then + echo "Usage: .ci/scripts/cherrypick.sh [commit-hash] [original-issue-id] [backport-issue-id]" + echo " ex: .ci/scripts/cherrypick.sh abcd1234 1234 4567" + echo "" + echo "Note: make sure you are on a fork of the release branch before running this script." + exit +fi + +commit="$(git rev-parse $1)" +issue="$2" +backport="$3" +commit_message=$(git log --format=%B -n 1 $commit) + +if ! echo $commit_message | tr '[:upper:]' '[:lower:]' | grep -q "\[noissue\]" +then + if ! echo $commit_message | tr '[:upper:]' '[:lower:]' | grep -q -E "(fixes|closes).*#$issue" + then + echo "Error: issue $issue not detected in commit message." && exit 1 + fi +fi + +if [ "$4" = "--continue" ] +then + echo "Continue after manually resolving conflicts..." +elif [ "$4" = "" ] +then + if ! git cherry-pick --no-commit "$commit" + then + echo "Please resolve and add merge conflicts and restart this command with appended '--continue'." + exit 1 + fi +else + exit 1 +fi + +for file in $(find CHANGES -name "$issue.*") +do + newfile="${file/$issue/$backport}" + git mv "$file" "$newfile" + sed -i -e "\$a (backported from #$issue)" "$newfile" + git add "$newfile" +done + +commit_message="$(printf "$commit_message" | sed -E 's/(fixes|closes)/backports/i')" +commit_message="$commit_message + +fixes #$backport + +(cherry picked from commit $commit)" +git commit -m "$commit_message" + +printf "\nSuccessfully backported commit $1.\n" diff --git a/.ci/scripts/redmine.py b/.ci/scripts/redmine.py deleted file mode 100644 index f27ad37ce..000000000 --- a/.ci/scripts/redmine.py +++ /dev/null @@ -1,30 +0,0 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -import os -import sys - -from redminelib import Redmine - -REDMINE_API_KEY = os.environ["REDMINE_API_KEY"] -REDMINE_QUERY_URL = sys.argv[1] -CLOSED_CURRENTRELEASE = 11 - -redmine = Redmine(REDMINE_QUERY_URL.split("issues")[0], key=REDMINE_API_KEY) -query_issues = REDMINE_QUERY_URL.split("=")[-1].split(",") - -to_update = [] -for issue in query_issues: - status = redmine.issue.get(int(issue)).status.name - if "CLOSE" not in status and status != "MODIFIED": - raise ValueError("One or more issues are not MODIFIED") - if status == "MODIFIED": # Removing the already closed - to_update.append(int(issue)) - -for issue in to_update: - print(f"Closing #{issue}") - redmine.issue.update(issue, status_id=CLOSED_CURRENTRELEASE) diff --git a/.ci/scripts/update_redmine.sh b/.ci/scripts/update_redmine.sh deleted file mode 100755 index fb15810cc..000000000 --- a/.ci/scripts/update_redmine.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../.. - -set -euv - -export COMMIT_MSG=$(git log --format=%B --no-merges -1) -export RELEASE=$(echo $COMMIT_MSG | awk '{print $2}') -export MILESTONE_URL=$(echo $COMMIT_MSG | grep -o "Redmine Milestone: .*" | awk '{print $3}') -export REDMINE_QUERY_URL=$(echo $COMMIT_MSG | grep -o "Redmine Query: .*" | awk '{print $3}') - -echo "Releasing $RELEASE" -echo "Milestone URL: $MILESTONE_URL" -echo "Query: $REDMINE_QUERY_URL" - -MILESTONE=$(http $MILESTONE_URL | jq -r .version.name) -echo "Milestone: $MILESTONE" - -if [[ "$MILESTONE" != "${RELEASE%.post*}" ]]; then - echo "Milestone $MILESTONE is not equal to Release $RELEASE" - exit 1 -fi - -pip install python-redmine -python3 .ci/scripts/redmine.py $REDMINE_QUERY_URL diff --git a/.ci/scripts/validate_commit_message.py b/.ci/scripts/validate_commit_message.py old mode 100644 new mode 100755 index ce06b8964..caf62d86b --- a/.ci/scripts/validate_commit_message.py +++ b/.ci/scripts/validate_commit_message.py @@ -6,21 +6,24 @@ # For more info visit https://github.com/pulp/plugin_template import re - import subprocess import sys from pathlib import Path + +import os from github import Github -KEYWORDS = ["fixes", "closes"] + NO_ISSUE = "[noissue]" CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc", ".deprecation"] + +KEYWORDS = ["fixes", "closes"] + sha = sys.argv[1] message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8") - -g = Github() +g = Github(os.environ.get("GITHUB_TOKEN")) repo = g.get_repo("pulp/pulp_python") diff --git a/.github/template_gitref b/.github/template_gitref new file mode 100644 index 000000000..6159baa66 --- /dev/null +++ b/.github/template_gitref @@ -0,0 +1 @@ +2021.04.08-97-g74b81ba-dirty diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bc3560d03..e594c8d03 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,19 +5,15 @@ # # For more info visit https://github.com/pulp/plugin_template --- -name: Pulp CI -on: - pull_request: - branches: - - '*' - push: - branches: - - '*' - +name: Python CI +on: {pull_request: {branches: ['*']}} jobs: + + lint: runs-on: ubuntu-latest + steps: - uses: actions/checkout@v2 @@ -28,7 +24,7 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: "3.7" + python-version: "3.6" # dev_requirements contains tools needed for flake8, etc. - name: Install requirements @@ -37,6 +33,7 @@ jobs: - name: Check commit message if: github.event_name == 'pull_request' env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} run: sh .github/workflows/scripts/check_commit.sh @@ -66,7 +63,6 @@ jobs: env: - TEST: pulp - TEST: docs - - TEST: s3 steps: - uses: actions/checkout@v2 @@ -77,7 +73,7 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: "3.7" + python-version: "3.6" - name: Install httpie run: | @@ -99,6 +95,13 @@ jobs: ANSIBLE_FORCE_COLOR: '1' shell: bash + - name: Install Python client + run: .github/workflows/scripts/install_python_client.sh + + - name: Install Ruby client + if: ${{ env.TEST == 'bindings' }} + run: .github/workflows/scripts/install_ruby_client.sh + - name: Before Script run: | .github/workflows/scripts/before_script.sh @@ -112,6 +115,11 @@ jobs: - name: Script run: .github/workflows/scripts/script.sh shell: bash + # env vars useful for post_script.sh when present + env: + GITHUB_PULL_REQUEST: ${{ github.event.number }} + GITHUB_BRANCH: ${{ github.head_ref }} + GITHUB_REPO_SLUG: ${{ github.repository }} - name: After failure if: failure() @@ -124,3 +132,6 @@ jobs: docker exec pulp ls -latr /etc/yum.repos.d/ || true docker exec pulp cat /etc/yum.repos.d/* || true docker exec pulp pip3 list + + + diff --git a/.github/workflows/create-branch.yml b/.github/workflows/create-branch.yml new file mode 100644 index 000000000..14b841c5d --- /dev/null +++ b/.github/workflows/create-branch.yml @@ -0,0 +1,79 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template +--- +name: Create New Release Branch +on: + workflow_dispatch: + inputs: + name: + description: "Branch name (e.g. 3.14)" + required: true + +env: + RELEASE_WORKFLOW: true + +jobs: + create-branch: + runs-on: ubuntu-latest + + strategy: + fail-fast: false + + steps: + - uses: actions/checkout@v2 + with: + # by default, it uses a depth of 1 + # this fetches all history so that we can read each commit + fetch-depth: 0 + + - uses: actions/setup-python@v2 + with: + python-version: "3.6" + + - name: Install python dependencies + run: | + echo ::group::PYDEPS + pip install bump2version + echo ::endgroup:: + + - name: Setting secrets + run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" + env: + SECRETS_CONTEXT: ${{ toJson(secrets) }} + + - name: Verify that branch name matches current version string on master branch + run: | + X_Y_VERSION=$(grep version setup.py | sed -rn 's/version="(.*)\.0\.dev",/\1/p' | awk '{$1=$1};1') + if [[ "$X_Y_VERSION" != "${{ github.event.inputs.name }}" ]] + then + echo "Branch name doesn't match the current version string $X_Y_VERSION." + exit 1 + fi + + - name: Create ${{ github.event.inputs.name }} release branch + run: | + git checkout -b ${{ github.event.inputs.name }} + git push origin ${{ github.event.inputs.name }} + + - name: Bump version on master branch + run: | + git checkout master + bump2version --no-commit minor + + - name: Make a PR with version bump + uses: peter-evans/create-pull-request@v3 + with: + committer: pulpbot + author: pulpbot + branch: minor-version-bump + base: master + title: Bump minor version + body: '[noissue]' + commit-message: | + Bump minor version + [noissue] + delete-branch: true diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 18d8b66b1..2de3e0e48 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -5,7 +5,7 @@ # # For more info visit https://github.com/pulp/plugin_template --- -name: Pulp Nightly CI/CD +name: Python Nightly CI/CD on: schedule: # * is a special character in YAML so you have to quote this string @@ -22,7 +22,7 @@ jobs: env: - TEST: pulp - TEST: docs - - TEST: s3 + - TEST: generate-bindings steps: - uses: actions/checkout@v2 @@ -33,7 +33,7 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: "3.7" + python-version: "3.6" - name: Install httpie run: | @@ -48,6 +48,11 @@ jobs: run: .github/workflows/scripts/before_install.sh shell: bash + - uses: ruby/setup-ruby@v1 + if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} + with: + ruby-version: "2.6" + - name: Install run: .github/workflows/scripts/install.sh env: @@ -64,10 +69,49 @@ jobs: env: SECRETS_CONTEXT: ${{ toJson(secrets) }} + - name: Install Python client + run: .github/workflows/scripts/install_python_client.sh + + - name: Install Ruby client + if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} + run: .github/workflows/scripts/install_ruby_client.sh + - name: Script run: .github/workflows/scripts/script.sh shell: bash + - name: Upload python client packages + if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} + uses: actions/upload-artifact@v2 + with: + name: python-client.tar + path: python-client.tar + + - name: Upload ruby client packages + if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} + uses: actions/upload-artifact@v2 + with: + name: ruby-client.tar + path: ruby-client.tar + + - name: Upload built docs + if: ${{ env.TEST == 'docs' }} + uses: actions/upload-artifact@v2 + with: + name: docs.tar + path: docs/docs.tar + + - name: After failure + if: failure() + run: | + http --timeout 30 --check-status --pretty format --print hb http://pulp/pulp/api/v3/status/ || true + docker images || true + docker ps -a || true + docker logs pulp || true + docker exec pulp ls -latr /etc/yum.repos.d/ || true + docker exec pulp cat /etc/yum.repos.d/* || true + docker exec pulp pip3 list + publish: runs-on: ubuntu-latest needs: test @@ -84,7 +128,7 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: "3.7" + python-version: "3.6" - uses: actions/setup-ruby@v1 with: @@ -115,6 +159,13 @@ jobs: ANSIBLE_FORCE_COLOR: '1' shell: bash + - name: Install Python client + run: .github/workflows/scripts/install_python_client.sh + + - name: Install Ruby client + if: ${{ env.TEST == 'bindings' }} || env.TEST == 'generate-bindings' }} + run: .github/workflows/scripts/install_ruby_client.sh + - name: Before Script run: .github/workflows/scripts/before_script.sh @@ -124,15 +175,29 @@ jobs: SECRETS_CONTEXT: ${{ toJson(secrets) }} - - name: Publish nightly client to rubygems - run: .ci/scripts/publish_client_gem.sh - shell: bash + - name: Download Ruby client + uses: actions/download-artifact@v2 + with: + name: python-client.tar + + - name: Untar Ruby client packages + run: tar -xvf ruby-client.tar + + - name: Publish client to rubygems + run: bash .github/workflows/scripts/publish_client_gem.sh - - name: Publish nightly client to pypi - run: .ci/scripts/publish_client_pypi.sh - shell: bash + - name: Download Python client + uses: actions/download-artifact@v2 + with: + name: python-client.tar + + - name: Untar python client packages + run: tar -xvf python-client.tar + + - name: Publish client to pypi + run: bash .github/workflows/scripts/publish_client_pypi.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 55fe87e67..f6932e6d9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,14 +5,70 @@ # # For more info visit https://github.com/pulp/plugin_template --- -name: Pulp Release CI/CD +name: Release Pipeline on: - push: - tags: - - '*' + workflow_dispatch: + inputs: + release: + description: "Release tag (e.g. 3.2.1)" + required: true + before_script: + description: | + Bash code to run before script.sh is executed. This should only be used when re-running + a workflow to correct some aspect of the docs. e.g.: git checkout origin/3.14 CHANGES.rst + required: false + +env: + RELEASE_WORKFLOW: true jobs: + build-artifacts: + runs-on: ubuntu-latest + + strategy: + fail-fast: false + + steps: + - uses: actions/checkout@v2 + with: + # by default, it uses a depth of 1 + # this fetches all history so that we can read each commit + fetch-depth: 0 + + - uses: actions/setup-python@v2 + with: + python-version: "3.6" + + - name: Install python dependencies + run: | + echo ::group::PYDEPS + pip install bandersnatch bump2version gitpython python-redmine towncrier==19.9.0 wheel + echo ::endgroup:: + + - name: Configure Git with pulpbot name and email + run: | + git config --global user.name 'pulpbot' + git config --global user.email 'pulp-infra@redhat.com' + + - name: Setting secrets + run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" + env: + SECRETS_CONTEXT: ${{ toJson(secrets) }} + + - name: Create the release commit, tag it, create a post-release commit, and build plugin package + run: python .github/workflows/scripts/release.py ${{ github.event.inputs.release }} + + - name: 'Tar files' + run: tar -cvf pulp_python.tar $GITHUB_WORKSPACE + + - name: 'Upload Artifact' + uses: actions/upload-artifact@v2 + with: + name: pulp_python.tar + path: pulp_python.tar test: + needs: build-artifacts + runs-on: ubuntu-latest strategy: @@ -21,18 +77,33 @@ jobs: env: - TEST: pulp - TEST: docs - - TEST: s3 + - TEST: generate-bindings steps: - - uses: actions/checkout@v2 + - uses: actions/download-artifact@v2 with: - # by default, it uses a depth of 1 - # this fetches all history so that we can read each commit - fetch-depth: 0 + name: pulp_python.tar - uses: actions/setup-python@v2 with: - python-version: "3.7" + python-version: "3.6" + - uses: actions/setup-ruby@v1 + with: + ruby-version: "2.6" + + - name: Untar repository + run: | + shopt -s dotglob + tar -xf pulp_python.tar + mv home/runner/work/pulp_python/pulp_python/* ./ + + # update to the branch's latest ci files rather than the ones from the release tag. this is + # helpful when there was a problem with the ci files during the release which needs to be + # fixed after the release tag has been created + - name: Update ci files + run: | + git checkout "origin/${GITHUB_REF##*/}" -- .ci + git checkout "origin/${GITHUB_REF##*/}" -- .github - name: Install httpie run: | @@ -48,7 +119,9 @@ jobs: shell: bash - name: Install - run: .github/workflows/scripts/install.sh + run: | + export PLUGIN_VERSION=${{ github.event.inputs.release }} + .github/workflows/scripts/install.sh env: PY_COLORS: '1' ANSIBLE_FORCE_COLOR: '1' @@ -63,79 +136,153 @@ jobs: env: SECRETS_CONTEXT: ${{ toJson(secrets) }} + - name: Install Python client + run: .github/workflows/scripts/install_python_client.sh + + - name: Install Ruby client + if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} + run: .github/workflows/scripts/install_ruby_client.sh + + - name: Additional before_script + run: ${{ github.event.inputs.before_script }} + shell: bash + - name: Script + if: ${{ env.TEST != 'generate-bindings' }} run: .github/workflows/scripts/script.sh shell: bash + - name: Upload python client packages + if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} + uses: actions/upload-artifact@v2 + with: + name: python-client.tar + path: python-client.tar + + - name: Upload ruby client packages + if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} + uses: actions/upload-artifact@v2 + with: + name: ruby-client.tar + path: ruby-client.tar + + - name: Upload built docs + if: ${{ env.TEST == 'docs' }} + uses: actions/upload-artifact@v2 + with: + name: docs.tar + path: docs/docs.tar + + - name: After failure + if: failure() + run: | + http --timeout 30 --check-status --pretty format --print hb http://pulp/pulp/api/v3/status/ || true + docker images || true + docker ps -a || true + docker logs pulp || true + docker exec pulp ls -latr /etc/yum.repos.d/ || true + docker exec pulp cat /etc/yum.repos.d/* || true + docker exec pulp pip3 list + + publish: runs-on: ubuntu-latest needs: test env: TEST: publish + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - uses: actions/checkout@v2 + - uses: actions/download-artifact@v2 with: - # by default, it uses a depth of 1 - # this fetches all history so that we can read each commit - fetch-depth: 0 + name: pulp_python.tar - uses: actions/setup-python@v2 with: - python-version: "3.7" + python-version: "3.6" - uses: actions/setup-ruby@v1 with: ruby-version: "2.6" - - name: Install httpie + - name: Configure Git with pulpbot name and email run: | - echo ::group::HTTPIE - sudo apt-get update -yq - sudo -E apt-get -yq --no-install-suggests --no-install-recommends install httpie - echo ::endgroup:: - echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV + git config --global user.name 'pulpbot' + git config --global user.email 'pulp-infra@redhat.com' - - name: Install python dependencies + - name: Untar repository run: | - echo ::group::PYDEPS - pip install wheel - echo ::endgroup:: - - - name: Before Install - run: .github/workflows/scripts/before_install.sh - shell: bash + shopt -s dotglob + tar -xf pulp_python.tar + mv home/runner/work/pulp_python/pulp_python/* ./ - - name: Install - run: .github/workflows/scripts/install.sh - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - shell: bash - - - name: Before Script + # update to the branch's latest ci files rather than the ones from the release tag. this is + # helpful when there was a problem with the ci files during the release which needs to be + # fixed after the release tag has been created + - name: Update ci files run: | - .github/workflows/scripts/before_script.sh + git checkout "origin/${GITHUB_REF##*/}" -- .ci + git checkout "origin/${GITHUB_REF##*/}" -- .github - name: Setting secrets run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" env: SECRETS_CONTEXT: ${{ toJson(secrets) }} + + - name: Install python dependencies + run: | + echo ::group::PYDEPS + pip install gitpython python-redmine + echo ::endgroup:: + + - name: Push branch and tag to GitHub + run: bash .github/workflows/scripts/push_branch_and_tag_to_github.sh ${{ github.event.inputs.release }} - name: Deploy plugin to pypi - run: bash .ci/scripts/publish_plugin_pypi.sh - - name: Publish client to rubygems - run: bash .ci/scripts/publish_client_gem.sh + run: bash .github/workflows/scripts/publish_plugin_pypi.sh ${{ github.event.inputs.release }} + - name: Download Python client + uses: actions/download-artifact@v2 + with: + name: python-client.tar + + - name: Untar python client packages + run: tar -xvf python-client.tar + - name: Publish client to pypi - run: bash .ci/scripts/publish_client_pypi.sh + run: bash .github/workflows/scripts/publish_client_pypi.sh + - name: Download Ruby client + uses: actions/download-artifact@v2 + with: + name: ruby-client.tar - - name: After failure - if: failure() - run: | - http --timeout 30 --check-status --pretty format --print hb http://pulp/pulp/api/v3/status/ || true - docker images || true - docker ps -a || true - docker logs pulp || true - docker exec pulp ls -latr /etc/yum.repos.d/ || true - docker exec pulp cat /etc/yum.repos.d/* || true - docker exec pulp pip3 list + - name: Untar Ruby client packages + run: tar -xvf ruby-client.tar + + - name: Publish client to rubygems + run: bash .github/workflows/scripts/publish_client_gem.sh + + + + - name: Create release on GitHub + run: bash .github/workflows/scripts/create_release_from_tag.sh ${{ github.event.inputs.release }} + + - name: Cleanup repository before making changelog PR + run: rm -rf .lock generation pulp_python_client* *-client.tar pulp_python.tar todo web docs.tar + + - name: Stage changelog for master branch + run: python .github/workflows/scripts/stage-changelog-for-master.py ${{ github.event.inputs.release }} + + - name: Create Pull Request for Changelog + uses: peter-evans/create-pull-request@v3 + with: + committer: pulpbot + author: pulpbot + branch: changelog/${{ github.event.inputs.release }} + base: master + title: 'Building changelog for ${{ github.event.inputs.release }}' + body: '[noissue]' + commit-message: | + Building changelog for ${{ github.event.inputs.release }} + + [noissue] + delete-branch: true diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index b425fc393..4d537c011 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -27,10 +27,23 @@ else BRANCH="${GITHUB_REF##refs/tags/}" fi +COMMIT_MSG=$(git log --format=%B --no-merges -1) +export COMMIT_MSG + +if [[ "$TEST" == "upgrade" ]]; then + git checkout -b ci_upgrade_test + cp -R .github /tmp/.github + cp -R .ci /tmp/.ci + git checkout $FROM_PULP_PYTHON_BRANCH + rm -rf .ci .github + cp -R /tmp/.github . + cp -R /tmp/.ci . +fi + if [[ "$TEST" == "plugin-from-pypi" ]]; then COMPONENT_VERSION=$(http https://pypi.org/pypi/pulp-python/json | jq -r '.info.version') else - COMPONENT_VERSION=$(sed -ne "s/\s*version=['\"]\(.*\)['\"][\s,]*/\1/p" setup.py) + COMPONENT_VERSION=$(sed -ne "s/\s*version.*=.*['\"]\(.*\)['\"][\s,]*/\1/p" setup.py) fi mkdir .ci/ansible/vars || true echo "---" > .ci/ansible/vars/main.yaml @@ -41,9 +54,6 @@ echo "component_version: '${COMPONENT_VERSION}'" >> .ci/ansible/vars/main.yaml export PRE_BEFORE_INSTALL=$PWD/.github/workflows/scripts/pre_before_install.sh export POST_BEFORE_INSTALL=$PWD/.github/workflows/scripts/post_before_install.sh -COMMIT_MSG=$(git log --format=%B --no-merges -1) -export COMMIT_MSG - if [ -f $PRE_BEFORE_INSTALL ]; then source $PRE_BEFORE_INSTALL fi @@ -68,8 +78,22 @@ else export CI_BASE_IMAGE= fi + cd .. + +git clone --depth=1 https://github.com/pulp/pulp-smash.git + +if [ -n "$PULP_SMASH_PR_NUMBER" ]; then + cd pulp-smash + git fetch --depth=1 origin pull/$PULP_SMASH_PR_NUMBER/head:$PULP_SMASH_PR_NUMBER + git checkout $PULP_SMASH_PR_NUMBER + cd .. +fi + +pip install --upgrade --force-reinstall ./pulp-smash + + git clone --depth=1 https://github.com/pulp/pulp-openapi-generator.git if [ -n "$PULP_OPENAPI_GENERATOR_PR_NUMBER" ]; then cd pulp-openapi-generator @@ -78,14 +102,9 @@ if [ -n "$PULP_OPENAPI_GENERATOR_PR_NUMBER" ]; then cd .. fi -cd pulp-openapi-generator -sed -i -e 's/localhost:24817/pulp/g' generate.sh -sed -i -e 's/:24817/pulp/g' generate.sh -cd .. - -git clone --depth=1 https://github.com/pulp/pulpcore.git --branch master +git clone --depth=1 https://github.com/pulp/pulpcore.git --branch 3.11 cd pulpcore if [ -n "$PULPCORE_PR_NUMBER" ]; then @@ -96,22 +115,19 @@ cd .. -git clone --depth=1 https://github.com/pulp/pulp-smash.git - -if [ -n "$PULP_SMASH_PR_NUMBER" ]; then - cd pulp-smash - git fetch --depth=1 origin pull/$PULP_SMASH_PR_NUMBER/head:$PULP_SMASH_PR_NUMBER - git checkout $PULP_SMASH_PR_NUMBER - cd .. -fi - -pip install --upgrade --force-reinstall ./pulp-smash - # Intall requirements for ansible playbooks pip install docker netaddr boto3 ansible -ansible-galaxy collection install amazon.aws +for i in {1..3} +do + ansible-galaxy collection install amazon.aws && s=0 && break || s=$? && sleep 3 +done +if [[ $s -gt 0 ]] +then + echo "Failed to install amazon.aws" + exit $s +fi cd pulp_python diff --git a/.github/workflows/scripts/before_script.sh b/.github/workflows/scripts/before_script.sh index e6d02dfca..7f15509c2 100755 --- a/.github/workflows/scripts/before_script.sh +++ b/.github/workflows/scripts/before_script.sh @@ -29,7 +29,7 @@ tail -v -n +1 .ci/ansible/vars/main.yaml echo "PULP CONFIG:" tail -v -n +1 .ci/ansible/settings/settings.* ~/.config/pulp_smash/settings.json -if [[ "$TEST" == 'pulp' || "$TEST" == 'performance' || "$TEST" == 's3' || "$TEST" == "plugin-from-pypi" ]]; then +if [[ "$TEST" == 'pulp' || "$TEST" == 'performance' || "$TEST" == 'upgrade' || "$TEST" == 's3' || "$TEST" == "plugin-from-pypi" ]]; then # Many functional tests require these cmd_prefix dnf install -yq lsof which dnf-plugins-core fi diff --git a/.github/workflows/scripts/check_commit.sh b/.github/workflows/scripts/check_commit.sh index 5647c1a48..7780e8be0 100755 --- a/.github/workflows/scripts/check_commit.sh +++ b/.github/workflows/scripts/check_commit.sh @@ -19,7 +19,7 @@ pip3 install pygithub echo ::endgroup:: -for sha in $(curl $GITHUB_CONTEXT | jq '.[].sha' | sed 's/"//g') +for sha in $(curl -H "Authorization: token $GITHUB_TOKEN" $GITHUB_CONTEXT | jq '.[].sha' | sed 's/"//g') do python3 .ci/scripts/validate_commit_message.py $sha VALUE=$? diff --git a/.github/workflows/scripts/create_release_from_tag.sh b/.github/workflows/scripts/create_release_from_tag.sh new file mode 100755 index 000000000..328454e0d --- /dev/null +++ b/.github/workflows/scripts/create_release_from_tag.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -e + +curl -s -X POST https://api.github.com/repos/$GITHUB_REPOSITORY/releases \ +-H "Authorization: token $RELEASE_TOKEN" \ +-d @- << EOF +{ + "tag_name": "$1", + "name": "$1" +} +EOF diff --git a/.github/workflows/scripts/docs-publisher.py b/.github/workflows/scripts/docs-publisher.py new file mode 100755 index 000000000..bb43c8b45 --- /dev/null +++ b/.github/workflows/scripts/docs-publisher.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python + +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +import argparse +import subprocess +import os +import re +from shutil import rmtree +import tempfile +import requests +import json +from packaging import version + +WORKING_DIR = os.environ["WORKSPACE"] + +VERSION_REGEX = r"(\s*)(version)(\s*)(=)(\s*)(['\"])(.*)(['\"])(.*)" +RELEASE_REGEX = r"(\s*)(release)(\s*)(=)(\s*)(['\"])(.*)(['\"])(.*)" + +USERNAME = "doc_builder_pulp_python" +HOSTNAME = "8.43.85.236" + +SITE_ROOT = "/var/www/docs.pulpproject.org/pulp_python/" + + +def make_directory_with_rsync(remote_paths_list): + """ + Ensure the remote directory path exists. + + :param remote_paths_list: The list of parameters. e.g. ['en', 'latest'] to be en/latest on the + remote. + :type remote_paths_list: a list of strings, with each string representing a directory. + """ + try: + tempdir_path = tempfile.mkdtemp() + cwd = os.getcwd() + os.chdir(tempdir_path) + os.makedirs(os.sep.join(remote_paths_list)) + remote_path_arg = "%s@%s:%s%s" % ( + USERNAME, + HOSTNAME, + SITE_ROOT, + remote_paths_list[0], + ) + local_path_arg = tempdir_path + os.sep + remote_paths_list[0] + os.sep + rsync_command = ["rsync", "-avzh", local_path_arg, remote_path_arg] + exit_code = subprocess.call(rsync_command) + if exit_code != 0: + raise RuntimeError("An error occurred while creating remote directories.") + finally: + rmtree(tempdir_path) + os.chdir(cwd) + + +def ensure_dir(target_dir, clean=True): + """ + Ensure that the directory specified exists and is empty. + + By default this will delete the directory if it already exists + + :param target_dir: The directory to process + :type target_dir: str + :param clean: Whether or not the directory should be removed and recreated + :type clean: bool + """ + if clean: + rmtree(target_dir, ignore_errors=True) + try: + os.makedirs(target_dir) + except OSError: + pass + + +def main(): + """ + Builds documentation using the 'make html' command and rsyncs to docs.pulpproject.org. + """ + parser = argparse.ArgumentParser() + parser.add_argument("--build-type", required=True, help="Build type: nightly or beta.") + parser.add_argument("--branch", required=True, help="Branch or tag name.") + opts = parser.parse_args() + if opts.build_type not in ["nightly", "tag"]: + raise RuntimeError("Build type must be either 'nightly' or 'tag'.") + + build_type = opts.build_type + + branch = opts.branch + + ga_build = False + + publish_at_root = False + + if (not re.search("[a-zA-Z]", branch) or "post" in branch) and len(branch.split(".")) > 2: + ga_build = True + # Only publish docs at the root if this is the latest version + r = requests.get("https://pypi.org/pypi/pulp-python/json") + latest_version = version.parse(json.loads(r.text)["info"]["version"]) + docs_version = version.parse(branch) + # This is to mitigate delays on PyPI which doesn't update metadata in timely manner. + # It doesn't prevent incorrect docs being published at root if 2 releases are done close + # to each other, within PyPI delay. E.g. Release 3.11.0 an then 3.10.1 immediately after. + if docs_version >= latest_version: + publish_at_root = True + # Post releases should use the x.y.z part of the version string to form a path + if "post" in branch: + branch = ".".join(branch.split(".")[:-1]) + + # rsync the docs + print("rsync the docs") + docs_directory = os.sep.join([WORKING_DIR, "docs"]) + local_path_arg = os.sep.join([docs_directory, "_build", "html"]) + os.sep + if build_type != "tag": + # This is a nightly build + remote_path_arg = "%s@%s:%sen/%s/%s/" % ( + USERNAME, + HOSTNAME, + SITE_ROOT, + branch, + build_type, + ) + make_directory_with_rsync(["en", branch, build_type]) + rsync_command = ["rsync", "-avzh", "--delete", local_path_arg, remote_path_arg] + exit_code = subprocess.call(rsync_command, cwd=docs_directory) + if exit_code != 0: + raise RuntimeError("An error occurred while pushing docs.") + elif ga_build: + # This is a GA build. + # publish to the root of docs.pulpproject.org + if publish_at_root: + version_components = branch.split(".") + x_y_version = "{}.{}".format(version_components[0], version_components[1]) + remote_path_arg = "%s@%s:%s" % (USERNAME, HOSTNAME, SITE_ROOT) + rsync_command = [ + "rsync", + "-avzh", + "--delete", + "--exclude", + "en", + "--omit-dir-times", + local_path_arg, + remote_path_arg, + ] + exit_code = subprocess.call(rsync_command, cwd=docs_directory) + if exit_code != 0: + raise RuntimeError("An error occurred while pushing docs.") + # publish to docs.pulpproject.org/en/3.y/ + make_directory_with_rsync(["en", x_y_version]) + remote_path_arg = "%s@%s:%sen/%s/" % ( + USERNAME, + HOSTNAME, + SITE_ROOT, + x_y_version, + ) + rsync_command = [ + "rsync", + "-avzh", + "--delete", + "--omit-dir-times", + local_path_arg, + remote_path_arg, + ] + exit_code = subprocess.call(rsync_command, cwd=docs_directory) + if exit_code != 0: + raise RuntimeError("An error occurred while pushing docs.") + # publish to docs.pulpproject.org/en/3.y.z/ + make_directory_with_rsync(["en", branch]) + remote_path_arg = "%s@%s:%sen/%s/" % (USERNAME, HOSTNAME, SITE_ROOT, branch) + rsync_command = [ + "rsync", + "-avzh", + "--delete", + "--omit-dir-times", + local_path_arg, + remote_path_arg, + ] + exit_code = subprocess.call(rsync_command, cwd=docs_directory) + if exit_code != 0: + raise RuntimeError("An error occurred while pushing docs.") + else: + # This is a pre-release + make_directory_with_rsync(["en", branch]) + remote_path_arg = "%s@%s:%sen/%s/%s/" % ( + USERNAME, + HOSTNAME, + SITE_ROOT, + branch, + build_type, + ) + rsync_command = [ + "rsync", + "-avzh", + "--delete", + "--exclude", + "nightly", + "--exclude", + "testing", + local_path_arg, + remote_path_arg, + ] + exit_code = subprocess.call(rsync_command, cwd=docs_directory) + if exit_code != 0: + raise RuntimeError("An error occurred while pushing docs.") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index a63cd0345..fd8ac9d62 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -15,13 +15,6 @@ set -euv source .github/workflows/scripts/utils.sh -if [ "${GITHUB_REF##refs/tags/}" = "${GITHUB_REF}" ] -then - TAG_BUILD=0 -else - TAG_BUILD=1 -fi - if [[ "$TEST" = "docs" || "$TEST" = "publish" ]]; then pip install -r ../pulpcore/doc_requirements.txt pip install -r doc_requirements.txt @@ -34,10 +27,12 @@ cd .ci/ansible/ TAG=ci_build if [[ "$TEST" == "plugin-from-pypi" ]]; then PLUGIN_NAME=pulp_python +elif [[ "${RELEASE_WORKFLOW:-false}" == "true" ]]; then + PLUGIN_NAME=./pulp_python/dist/pulp_python-$PLUGIN_VERSION-py3-none-any.whl else PLUGIN_NAME=./pulp_python fi -if [ "${TAG_BUILD}" = "1" ]; then +if [[ "${RELEASE_WORKFLOW:-false}" == "true" ]]; then # Install the plugin only and use published PyPI packages for the rest # Quoting ${TAG} ensures Ansible casts the tag as a string. cat >> vars/main.yaml << VARSYAML @@ -46,7 +41,7 @@ image: tag: "${TAG}" plugins: - name: pulpcore - source: pulpcore + source: pulpcore<3.12 - name: pulp_python source: "${PLUGIN_NAME}" services: @@ -75,6 +70,10 @@ fi cat >> vars/main.yaml << VARSYAML pulp_settings: null +pulp_scheme: http + +pulp_container_tag: python36 + VARSYAML ansible-playbook build_container.yaml diff --git a/.github/workflows/scripts/install_python_client.sh b/.github/workflows/scripts/install_python_client.sh new file mode 100755 index 000000000..226a40f6b --- /dev/null +++ b/.github/workflows/scripts/install_python_client.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +set -euv + +export PULP_URL="${PULP_URL:-http://pulp}" + +# make sure this script runs at the repo root +cd "$(dirname "$(realpath -e "$0")")"/../../.. + +pip install twine wheel + +export REPORTED_VERSION=$(http $PULP_URL/pulp/api/v3/status/ | jq --arg plugin python --arg legacy_plugin pulp_python -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version') +export DESCRIPTION="$(git describe --all --exact-match `git rev-parse HEAD`)" +if [[ $DESCRIPTION == 'tags/'$REPORTED_VERSION ]]; then + export VERSION=${REPORTED_VERSION} +else + export EPOCH="$(date +%s)" + export VERSION=${REPORTED_VERSION}${EPOCH} +fi + +export response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/pulp-python-client/$VERSION/) + +if [ "$response" == "200" ]; +then + echo "pulp_python client $VERSION has already been released. Installing from PyPI." + pip install pulp-python-client==$VERSION + mkdir -p dist + tar cvf python-client.tar ./dist + exit +fi + +cd ../pulp-openapi-generator +rm -rf pulp_python-client +./generate.sh pulp_python python $VERSION +cd pulp_python-client +python setup.py sdist bdist_wheel --python-tag py3 +find . -name "*.whl" -exec pip install {} \; +tar cvf ../../pulp_python/python-client.tar ./dist +exit $? diff --git a/.github/workflows/scripts/install_ruby_client.sh b/.github/workflows/scripts/install_ruby_client.sh new file mode 100755 index 000000000..f3a0448ce --- /dev/null +++ b/.github/workflows/scripts/install_ruby_client.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +set -euv + +# make sure this script runs at the repo root +cd "$(dirname "$(realpath -e "$0")")"/../../.. + +export PULP_URL="${PULP_URL:-http://pulp}" + +export REPORTED_VERSION=$(http $PULP_URL/pulp/api/v3/status/ | jq --arg plugin python --arg legacy_plugin pulp_python -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version') +export DESCRIPTION="$(git describe --all --exact-match `git rev-parse HEAD`)" +if [[ $DESCRIPTION == 'tags/'$REPORTED_VERSION ]]; then + export VERSION=${REPORTED_VERSION} +else + export EPOCH="$(date +%s)" + export VERSION=${REPORTED_VERSION}${EPOCH} +fi + +export response=$(curl --write-out %{http_code} --silent --output /dev/null https://rubygems.org/gems/pulp_python_client/versions/$VERSION) + +if [ "$response" == "200" ]; +then + echo "pulp_python client $VERSION has already been released. Installing from RubyGems.org." + gem install pulp_python_client -v $VERSION + touch pulp_python_client-$VERSION.gem + tar cvf ruby-client.tar ./pulp_python_client-$VERSION.gem + exit +fi + +cd ../pulp-openapi-generator +rm -rf pulp_python-client +./generate.sh pulp_python ruby $VERSION +cd pulp_python-client +gem build pulp_python_client +gem install --both ./pulp_python_client-$VERSION.gem +tar cvf ../../pulp_python/ruby-client.tar ./pulp_python_client-$VERSION.gem diff --git a/.github/workflows/scripts/publish_client_gem.sh b/.github/workflows/scripts/publish_client_gem.sh new file mode 100755 index 000000000..2cd1d0523 --- /dev/null +++ b/.github/workflows/scripts/publish_client_gem.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +set -euv + +# make sure this script runs at the repo root +cd "$(dirname "$(realpath -e "$0")")"/../../.. + + +mkdir ~/.gem || true +touch ~/.gem/credentials +echo "--- +:rubygems_api_key: $RUBYGEMS_API_KEY" > ~/.gem/credentials +sudo chmod 600 ~/.gem/credentials + +export VERSION=$(ls pulp_python_client* | sed -rn 's/pulp_python_client-(.*)\.gem/\1/p') + +if [[ -z "$VERSION" ]]; then + echo "No client package found." + exit +fi + +export response=$(curl --write-out %{http_code} --silent --output /dev/null https://rubygems.org/gems/pulp_python_client/versions/$VERSION) + +if [ "$response" == "200" ]; +then + echo "pulp_python client $VERSION has already been released. Skipping." + exit +fi + +GEM_FILE="$(ls pulp_python_client-*)" +gem push ${GEM_FILE} diff --git a/.github/workflows/scripts/publish_client_pypi.sh b/.github/workflows/scripts/publish_client_pypi.sh new file mode 100755 index 000000000..83dbcaf60 --- /dev/null +++ b/.github/workflows/scripts/publish_client_pypi.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +set -euv + +# make sure this script runs at the repo root +cd "$(dirname "$(realpath -e "$0")")"/../../.. + +pip install twine + +export VERSION=$(ls dist | sed -rn 's/pulp_python-client-(.*)\.tar.gz/\1/p') + +if [[ -z "$VERSION" ]]; then + echo "No client package found." + exit +fi + +export response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/pulp-python-client/$VERSION/) + +if [ "$response" == "200" ]; +then + echo "pulp_python client $VERSION has already been released. Skipping." + exit +fi + +twine check dist/pulp_python_client-$VERSION-py3-none-any.whl || exit 1 +twine check dist/pulp_python-client-$VERSION.tar.gz || exit 1 +twine upload dist/pulp_python_client-$VERSION-py3-none-any.whl -u pulp -p $PYPI_PASSWORD +twine upload dist/pulp_python-client-$VERSION.tar.gz -u pulp -p $PYPI_PASSWORD + +exit $? diff --git a/.github/workflows/scripts/publish_docs.sh b/.github/workflows/scripts/publish_docs.sh new file mode 100755 index 000000000..4f978d71e --- /dev/null +++ b/.github/workflows/scripts/publish_docs.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +set -euv + +# make sure this script runs at the repo root +cd "$(dirname "$(realpath -e "$0")")"/../../.. + +mkdir ~/.ssh +echo "$PULP_DOCS_KEY" > ~/.ssh/pulp-infra +chmod 600 ~/.ssh/pulp-infra + +echo "docs.pulpproject.org,8.43.85.236 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGXG+8vjSQvnAkq33i0XWgpSrbco3rRqNZr0SfVeiqFI7RN/VznwXMioDDhc+hQtgVhd6TYBOrV07IMcKj+FAzg=" >> /home/runner/.ssh/known_hosts +chmod 644 /home/runner/.ssh/known_hosts + +pip3 install -r doc_requirements.txt + +export PYTHONUNBUFFERED=1 +export DJANGO_SETTINGS_MODULE=pulpcore.app.settings +export PULP_SETTINGS=$PWD/.ci/ansible/settings/settings.py +export WORKSPACE=$PWD + +eval "$(ssh-agent -s)" #start the ssh agent +ssh-add ~/.ssh/pulp-infra + +python3 .github/workflows/scripts/docs-publisher.py --build-type $1 --branch $2 diff --git a/.github/workflows/scripts/publish_plugin_pypi.sh b/.github/workflows/scripts/publish_plugin_pypi.sh new file mode 100755 index 000000000..981f77073 --- /dev/null +++ b/.github/workflows/scripts/publish_plugin_pypi.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +# make sure this script runs at the repo root +cd "$(dirname "$(realpath -e "$0")")"/../../.. + +set -euv + +export response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/pulp-python/$1/) +if [ "$response" == "200" ]; +then + echo "pulp_python $1 has already been released. Skipping." + exit +fi + +pip install twine + +twine check dist/pulp_python-$1-py3-none-any.whl || exit 1 +twine check dist/pulp-python-$1.tar.gz || exit 1 +twine upload dist/pulp_python-$1-py3-none-any.whl -u pulp -p $PYPI_PASSWORD +twine upload dist/pulp-python-$1.tar.gz -u pulp -p $PYPI_PASSWORD + +exit $? diff --git a/.github/workflows/scripts/push_branch_and_tag_to_github.sh b/.github/workflows/scripts/push_branch_and_tag_to_github.sh new file mode 100755 index 000000000..2f13c78d8 --- /dev/null +++ b/.github/workflows/scripts/push_branch_and_tag_to_github.sh @@ -0,0 +1,21 @@ +#!/bin/sh +set -e + +BRANCH_NAME=$(echo $GITHUB_REF | sed -rn 's/refs\/heads\/(.*)/\1/p') + +ref_string=$(git show-ref --tags | grep refs/tags/$1) + +SHA=${ref_string:0:40} + +remote_repo=https://pulpbot:${RELEASE_TOKEN}@github.com/${GITHUB_REPOSITORY}.git + +git push "${remote_repo}" $BRANCH_NAME + +curl -s -X POST https://api.github.com/repos/$GITHUB_REPOSITORY/git/refs \ +-H "Authorization: token $RELEASE_TOKEN" \ +-d @- << EOF +{ + "ref": "refs/tags/$1", + "sha": "$SHA" +} +EOF diff --git a/.github/workflows/scripts/release.py b/.github/workflows/scripts/release.py new file mode 100755 index 000000000..a47f7f3bf --- /dev/null +++ b/.github/workflows/scripts/release.py @@ -0,0 +1,202 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +import argparse +import asyncio +import re +import os +import shutil +import textwrap + +from bandersnatch.mirror import BandersnatchMirror +from bandersnatch.master import Master +from bandersnatch.configuration import BandersnatchConfig + +from git import Repo + +from packaging.requirements import Requirement + + +async def get_package_from_pypi(package_name, plugin_path): + """ + Download a package from PyPI. + + :param name: name of the package to download from PyPI + :return: String path to the package + """ + config = BandersnatchConfig().config + config["mirror"]["master"] = "https://pypi.org" + config["mirror"]["workers"] = "1" + config["mirror"]["directory"] = plugin_path + if not config.has_section("plugins"): + config.add_section("plugins") + config["plugins"]["enabled"] = "blocklist_release\n" + if not config.has_section("allowlist"): + config.add_section("allowlist") + config["plugins"]["enabled"] += "allowlist_release\nallowlist_project\n" + config["allowlist"]["packages"] = "\n".join([package_name]) + os.makedirs(os.path.join(plugin_path, "dist"), exist_ok=True) + async with Master("https://pypi.org/") as master: + mirror = BandersnatchMirror(homedir=plugin_path, master=master) + name = Requirement(package_name).name + result = await mirror.synchronize([name]) + package_found = False + + for package in result[name]: + current_path = os.path.join(plugin_path, package) + destination_path = os.path.join(plugin_path, "dist", os.path.basename(package)) + shutil.move(current_path, destination_path) + package_found = True + return package_found + + +def create_release_commits(repo, release_version, plugin_path): + """Build changelog, set version, commit, bump to next dev version, commit.""" + # First commit: changelog + os.system(f"towncrier --yes --version {release_version}") + git = repo.git + git.add("CHANGES.rst") + git.add("CHANGES/*") + git.commit("-m", f"Building changelog for {release_version}\n\n[noissue]") + + # Second commit: release version + os.system("bump2version release --allow-dirty") + + git.add(f"{plugin_path}/{plugin_name}/*") + git.add(f"{plugin_path}/docs/conf.py") + git.add(f"{plugin_path}/setup.py") + git.add(f"{plugin_path}/requirements.txt") + git.add(f"{plugin_path}/.bumpversion.cfg") + + git.commit("-m", f"Release {release_version}\n\n[noissue]") + + sha = repo.head.object.hexsha + short_sha = git.rev_parse(sha, short=7) + + os.system("bump2version patch --allow-dirty") + + new_dev_version = None + with open(f"{plugin_path}/setup.py") as fp: + for line in fp.readlines(): + if "version=" in line: + new_dev_version = re.split("\"|'", line)[1] + if not new_dev_version: + raise RuntimeError("Could not detect new dev version ... aborting.") + + git.add(f"{plugin_path}/{plugin_name}/*") + git.add(f"{plugin_path}/docs/conf.py") + git.add(f"{plugin_path}/setup.py") + git.add(f"{plugin_path}/requirements.txt") + git.add(f"{plugin_path}/.bumpversion.cfg") + git.commit("-m", f"Bump to {new_dev_version}\n\n[noissue]") + + print(f"Release commit == {short_sha}") + print(f"All changes were committed on branch: release_{release_version}") + return sha + + +def create_tag_and_build_package(repo, desired_tag, commit_sha, plugin_path): + """Create a tag if one is needed and build a package if one is not on PyPI.""" + # Remove auth header config + with repo.config_writer() as conf: + conf.remove_section('http "https://github.com/"') + conf.release() + + # Determine if a tag exists and if it matches the specified commit sha + tag = None + for existing_tag in repo.tags: + if existing_tag.name == desired_tag: + if existing_tag.commit.hexsha == commit_sha: + tag = existing_tag + else: + raise RuntimeError( + "The '{desired_tag}' tag already exists, but the commit sha does not match " + "'{commit_sha}'." + ) + + # Create a tag if one does not exist + if not tag: + tag = repo.create_tag(desired_tag, ref=commit_sha) + + # Checkout the desired tag and reset the tree + repo.head.reference = tag.commit + repo.head.reset(index=True, working_tree=True) + + # Check if Package is available on PyPI + loop = asyncio.get_event_loop() # noqa + # fmt: off + package_found = asyncio.run( + get_package_from_pypi("pulp-python=={tag.name}", plugin_path) + ) # noqa + # fmt: on + if not package_found: + os.system("python3 setup.py sdist bdist_wheel --python-tag py3") + + +helper = textwrap.dedent( + """\ + Start the release process. + + Example: + setup.py on plugin before script: + version="2.0.0.dev" + + $ python .ci/scripts/release.py + + setup.py on plugin after script: + version="2.0.1.dev" + + + """ +) +parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, description=helper) + +parser.add_argument( + "release_version", + type=str, + help="The version string for the release.", +) + +args = parser.parse_args() + +release_version_arg = args.release_version + +release_path = os.path.dirname(os.path.abspath(__file__)) +plugin_path = release_path.split("/.github")[0] + +plugin_name = "pulp_python" +version = None +with open(f"{plugin_path}/setup.py") as fp: + for line in fp.readlines(): + if "version=" in line: + version = re.split("\"|'", line)[1] + if not version: + raise RuntimeError("Could not detect existing version ... aborting.") +release_version = version.replace(".dev", "") + +print(f"\n\nRepo path: {plugin_path}") +repo = Repo(plugin_path) + +release_commit = None +if release_version != release_version_arg: + # Look for a commit with the requested release version + for commit in repo.iter_commits(): + if f"Release {release_version_arg}\n" in commit.message: + release_commit = commit + release_version = release_version_arg + break + if not release_commit: + raise RuntimeError( + f"The release version {release_version_arg} does not match the .dev version at HEAD. " + "A release commit for such version does not exist." + ) + +if not release_commit: + release_commit_sha = create_release_commits(repo, release_version, plugin_path) +else: + release_commit_sha = release_commit.hexsha +create_tag_and_build_package(repo, release_version, release_commit_sha, plugin_path) diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index f5452575e..9c116015a 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -26,9 +26,12 @@ export FUNC_TEST_SCRIPT=$PWD/.github/workflows/scripts/func_test_script.sh export DJANGO_SETTINGS_MODULE=pulpcore.app.settings export PULP_SETTINGS=$PWD/.ci/ansible/settings/settings.py -if [[ "$TEST" = "docs" || "$TEST" = "publish" ]]; then +export PULP_URL="http://pulp" + +if [[ "$TEST" = "docs" ]]; then cd docs - make PULP_URL="http://pulp" html + make PULP_URL="$PULP_URL" diagrams html + tar -cvf docs.tar ./_build cd .. echo "Validating OpenAPI schema..." @@ -42,44 +45,44 @@ if [[ "$TEST" = "docs" || "$TEST" = "publish" ]]; then exit fi +if [[ "${RELEASE_WORKFLOW:-false}" == "true" ]]; then + REPORTED_VERSION=$(http $PULP_URL/pulp/api/v3/status/ | jq --arg plugin python --arg legacy_plugin pulp_python -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version') + response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/pulp-python/$REPORTED_VERSION/) + if [ "$response" == "200" ]; + then + echo "pulp_python $REPORTED_VERSION has already been released. Skipping running tests." + exit + fi +fi + if [[ "$TEST" == "plugin-from-pypi" ]]; then COMPONENT_VERSION=$(http https://pypi.org/pypi/pulp-python/json | jq -r '.info.version') git checkout ${COMPONENT_VERSION} -- pulp_python/tests/ fi cd ../pulp-openapi-generator - ./generate.sh pulpcore python pip install ./pulpcore-client -./generate.sh pulp_python python -pip install ./pulp_python-client -cd $REPO_ROOT - -if [[ "$TEST" = 'bindings' || "$TEST" = 'publish' ]]; then - python $REPO_ROOT/.ci/assets/bindings/test_bindings.py - cd ../pulp-openapi-generator - if [ ! -f $REPO_ROOT/.ci/assets/bindings/test_bindings.rb ] - then - exit - fi - - rm -rf ./pulpcore-client - +rm -rf ./pulpcore-client +if [[ "$TEST" = 'bindings' ]]; then ./generate.sh pulpcore ruby 0 cd pulpcore-client - gem build pulpcore_client + gem build pulpcore_client.gemspec gem install --both ./pulpcore_client-0.gem - cd .. - rm -rf ./pulp_python-client +fi +cd $REPO_ROOT - ./generate.sh pulp_python ruby 0 +if [[ "$TEST" = 'bindings' ]]; then + python $REPO_ROOT/.ci/assets/bindings/test_bindings.py +fi - cd pulp_python-client - gem build pulp_python_client - gem install --both ./pulp_python_client-0.gem - cd .. - ruby $REPO_ROOT/.ci/assets/bindings/test_bindings.rb - exit +if [[ "$TEST" = 'bindings' ]]; then + if [ ! -f $REPO_ROOT/.ci/assets/bindings/test_bindings.rb ]; then + exit + else + ruby $REPO_ROOT/.ci/assets/bindings/test_bindings.rb + exit + fi fi cat unittest_requirements.txt | cmd_stdin_prefix bash -c "cat > /tmp/unittest_requirements.txt" @@ -95,6 +98,8 @@ cmd_prefix bash -c "PULP_DATABASES__default__USER=postgres django-admin test --n # Run functional tests export PYTHONPATH=$REPO_ROOT:$REPO_ROOT/../pulpcore${PYTHONPATH:+:${PYTHONPATH}} + + if [[ "$TEST" == "performance" ]]; then if [[ -z ${PERFORMANCE_TEST+x} ]]; then pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_python.tests.performance diff --git a/.github/workflows/scripts/secrets.py b/.github/workflows/scripts/secrets.py old mode 100644 new mode 100755 diff --git a/.github/workflows/scripts/stage-changelog-for-master.py b/.github/workflows/scripts/stage-changelog-for-master.py new file mode 100755 index 000000000..649f70578 --- /dev/null +++ b/.github/workflows/scripts/stage-changelog-for-master.py @@ -0,0 +1,64 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +import argparse +import os +import textwrap + +from git import Repo +from git.exc import GitCommandError + + +helper = textwrap.dedent( + """\ + Stage the changelog for a release on master branch. + + Example: + $ python .github/workflows/scripts/stage-changelog-for-master.py 3.4.0 + + """ +) + +parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, description=helper) + +parser.add_argument( + "release_version", + type=str, + help="The version string for the release.", +) + +args = parser.parse_args() + +release_version_arg = args.release_version + +release_path = os.path.dirname(os.path.abspath(__file__)) +plugin_path = release_path.split("/.github")[0] + +print(f"\n\nRepo path: {plugin_path}") +repo = Repo(plugin_path) + +changelog_commit = None +# Look for a commit with the requested release version +for commit in repo.iter_commits(): + if f"Building changelog for {release_version_arg}\n" in commit.message: + changelog_commit = commit + break + +if not changelog_commit: + raise RuntimeError("Changelog commit for {release_version_arg} was not found.") + +git = repo.git +git.stash() +git.checkout("origin/master") +try: + git.cherry_pick(changelog_commit.hexsha) +except GitCommandError: + git.add("CHANGES/") + # Don't try opening an editor for the commit message + with git.custom_environment(GIT_EDITOR="true"): + git.cherry_pick("--continue") +git.reset("origin/master") diff --git a/.github/workflows/scripts/update_ci.sh b/.github/workflows/scripts/update_ci.sh new file mode 100755 index 000000000..c83c94541 --- /dev/null +++ b/.github/workflows/scripts/update_ci.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +set -eu + +if [ ! -d ../plugin_template ]; then + echo "Checking out plugin_template" + git clone https://github.com/pulp/plugin_template.git ../plugin_template +fi + + +if [ ! -f "template_config.yml" ]; then + echo "No template_config.yml detected." + exit 1 +fi + +pushd ../plugin_template +./plugin-template --github pulp_python +popd + +if [[ `git status --porcelain` ]]; then + git add -A + git commit -m "Update CI files" -m "[noissue]" +else + echo "No updates needed" +fi diff --git a/.github/workflows/update_ci.yml b/.github/workflows/update_ci.yml new file mode 100644 index 000000000..2b432b256 --- /dev/null +++ b/.github/workflows/update_ci.yml @@ -0,0 +1,48 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template +--- +name: CI Update +on: + schedule: + # * is a special character in YAML so you have to quote this string + # runs at 2:30 UTC daily + - cron: '30 2 * * *' + + workflow_dispatch: + +jobs: + update: + runs-on: ubuntu-latest + + strategy: + fail-fast: false + + steps: + - uses: actions/checkout@v2 + + - name: Configure Git with pulpbot name and email + run: | + git config --global user.name 'pulpbot' + git config --global user.email 'pulp-infra@redhat.com' + + - name: Run update + run: | + .github/workflows/scripts/update_ci.sh + + - name: Create Pull Request for CI files + uses: peter-evans/create-pull-request@v3 + with: + token: ${{ secrets.RELEASE_TOKEN }} + committer: pulpbot + author: pulpbot + title: 'Update CI files' + body: '[noissue]' + commit-message: | + Update CI files + + [noissue] + delete-branch: true diff --git a/docs/Makefile b/docs/Makefile index 2f09cf033..593e28dc6 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -2,18 +2,21 @@ # # You can set these variables from the command line. -SPHINXOPTS = -W -n -SPHINXBUILD = sphinx-build -PAPER = -BUILDDIR = _build -PULP_URL = "http://localhost:24817" +SPHINXOPTS = -W -n +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build +STATIC_BUILD_DIR = _static +DIAGRAM_BUILD_DIR = _diagrams +DIAGRAM_SOURCE_DIR = diagrams_src +PULP_URL = "http://localhost:24817" # Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . # the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext @@ -41,9 +44,22 @@ help: clean: -rm -rf $(BUILDDIR)/* + -rm -rf $(DIAGRAM_BUILD_DIR)/* + +diagrams: +ifneq ($(wildcard $(DIAGRAM_SOURCE_DIR)), ) + mkdir -p $(DIAGRAM_BUILD_DIR) + python3 -m plantuml $(DIAGRAM_SOURCE_DIR)/*.dot + # cp + rm = mv workaround https://pulp.plan.io/issues/4791#note-3 + cp $(DIAGRAM_SOURCE_DIR)/*.png $(DIAGRAM_BUILD_DIR)/ + rm $(DIAGRAM_SOURCE_DIR)/*.png +else + @echo "Did not find $(DIAGRAM_SOURCE_DIR)." +endif html: - curl --fail -o _static/api.json "$(PULP_URL)/pulp/api/v3/docs/api.json?plugin=pulp_python&include_html=1" + mkdir -p $(STATIC_BUILD_DIR) + curl --fail -o $(STATIC_BUILD_DIR)/api.json "$(PULP_URL)/pulp/api/v3/docs/api.json?plugin=pulp_python&include_html=1" $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." @@ -72,7 +88,7 @@ htmlhelp: $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." + ".hhp project file in $(BUILDDIR)/htmlhelp." epub: $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub @@ -84,7 +100,7 @@ latex: @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." + "(use \`make latexpdf' here to do that automatically)." latexpdf: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @@ -107,7 +123,7 @@ texinfo: @echo @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." @echo "Run \`make' in that directory to run these through makeinfo" \ - "(use \`make info' here to do that automatically)." + "(use \`make info' here to do that automatically)." info: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @@ -129,9 +145,9 @@ linkcheck: $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." + "or in $(BUILDDIR)/linkcheck/output.txt." doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." + "results in $(BUILDDIR)/doctest/output.txt." diff --git a/docs/_scripts/base.sh b/docs/_scripts/base.sh index 067cfcae3..b80b5d101 100755 --- a/docs/_scripts/base.sh +++ b/docs/_scripts/base.sh @@ -17,10 +17,13 @@ if [ -z "$(pip freeze | grep pulp-cli)" ]; then fi # Set up CLI config file -mkdir ~/.config/pulp -cat > ~/.config/pulp/settings.toml << EOF +if [ ! -f ~/.config/pulp/settings.toml ]; then + echo "Configuring pulp-cli" + mkdir -p ~/.config/pulp + cat > ~/.config/pulp/cli.toml << EOF [cli] -base_url = "$BASE_ADDR" # common to be localhost +base_url = "$BASE_ADDR" verify_ssl = false format = "json" EOF +fi diff --git a/pulp_python/tests/upgrade/__init__.py b/pulp_python/tests/upgrade/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pulp_python/tests/upgrade/pre/__init__.py b/pulp_python/tests/upgrade/pre/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pulp_python/tests/upgrade/pre/test_publish.py b/pulp_python/tests/upgrade/pre/test_publish.py new file mode 100644 index 000000000..899155a26 --- /dev/null +++ b/pulp_python/tests/upgrade/pre/test_publish.py @@ -0,0 +1,128 @@ +"""Tests that publish file plugin repositories.""" +import unittest +from random import choice + +from pulp_smash import config +from pulp_smash.pulp3.bindings import monitor_task +from pulp_smash.pulp3.utils import ( + gen_repo, + get_content, + gen_distribution, + get_versions, +) + +from pulp_python.tests.functional.constants import PYTHON_CONTENT_NAME +from pulp_python.tests.functional.utils import ( + gen_python_client, + gen_python_remote, +) +from pulp_python.tests.functional.utils import set_up_module as setUpModule # noqa:F401 + +from pulpcore.client.pulp_python import ( + DistributionsPypiApi, + PublicationsPypiApi, + RepositoryAddRemoveContent, + RepositoriesPythonApi, + RepositorySyncURL, + RemotesPythonApi, + PythonPythonPublication, +) +from pulpcore.client.pulp_python.exceptions import ApiException + + +class PublishAnyRepoVersionTestCase(unittest.TestCase): + """Test whether a particular repository version can be published. + + This test targets the following issues: + + * `Pulp #3324 `_ + * `Pulp Smash #897 `_ + """ + + @classmethod + def setUpClass(cls): + """Create class-wide variables.""" + cls.cfg = config.get_config() + + client = gen_python_client() + cls.repo_api = RepositoriesPythonApi(client) + cls.remote_api = RemotesPythonApi(client) + cls.publications = PublicationsPypiApi(client) + cls.distributions = DistributionsPypiApi(client) + + def setUp(self): + """Create a new repository before each test.""" + body = gen_python_remote() + remote = self.remote_api.create(body) + + repo = self.repo_api.create(gen_repo()) + + repository_sync_data = RepositorySyncURL(remote=remote.pulp_href) + sync_response = self.repo_api.sync(repo.pulp_href, repository_sync_data) + monitor_task(sync_response.task) + + self.repo = self.repo_api.read(repo.pulp_href) + + def test_all(self): + """Test whether a particular repository version can be published. + + 1. Create a repository with at least 2 repository versions. + 2. Create a publication by supplying the latest ``repository_version``. + 3. Assert that the publication ``repository_version`` attribute points + to the latest repository version. + 4. Create a publication by supplying the non-latest ``repository_version``. + 5. Create distribution. + 6. Assert that the publication ``repository_version`` attribute points + to the supplied repository version. + 7. Assert that an exception is raised when providing two different + repository versions to be published at same time. + """ + # Step 1 + repo_content = get_content(self.repo.to_dict())[PYTHON_CONTENT_NAME][:-1] + for file_content in repo_content: + repository_modify_data = RepositoryAddRemoveContent( + remove_content_units=[file_content["pulp_href"]] + ) + modify_response = self.repo_api.modify(self.repo.pulp_href, repository_modify_data) + monitor_task(modify_response.task) + version_hrefs = tuple(ver["pulp_href"] for ver in get_versions(self.repo.to_dict())) + non_latest = choice(version_hrefs[1:-1]) + + # Step 2 + publish_data = PythonPythonPublication(repository=self.repo.pulp_href) + publication = self.create_publication(publish_data) + + # Step 3 + self.assertEqual(publication.repository_version, version_hrefs[-1]) + + # Step 4 + publish_data = PythonPythonPublication(repository_version=non_latest) + publication = self.create_publication(publish_data) + + # Step 5 + body = gen_distribution() + body["base_path"] = "pulp_pre_upgrade_test" + body["publication"] = publication.pulp_href + + distribution_response = self.distributions.create(body) + created_resources = monitor_task(distribution_response.task).created_resources + distribution = self.distributions.read(created_resources[0]) + + # Step 6 + self.assertEqual(publication.repository_version, non_latest) + + # Step 7 + with self.assertRaises(ApiException): + body = {"repository": self.repo.pulp_href, "repository_version": non_latest} + self.publications.create(body) + + # Step 8 + url = self.cfg.get_content_host_base_url() + "/pulp/content/pulp_pre_upgrade_test/" + self.assertEqual(url, distribution.base_url, url) + + def create_publication(self, publish_data): + """Create a new publication from the passed data.""" + publish_response = self.publications.create(publish_data) + created_resources = monitor_task(publish_response.task).created_resources + publication_href = created_resources[0] + return self.publications.read(publication_href) diff --git a/template_config.yml b/template_config.yml index 3590e1ddc..ed0f64606 100644 --- a/template_config.yml +++ b/template_config.yml @@ -1,13 +1,19 @@ # This config represents the latest values used when running the plugin-template. Any settings that # were not present before running plugin-template have been added with their default values. +# generated with plugin_template@2021.04.08-97-g74b81ba + additional_plugins: [] +additional_repos: [] black: false check_commit_message: true check_gettext: true check_manifest: true +check_openapi_schema: true check_stray_pulpcore_imports: true cherry_pick_automation: false +ci_trigger: '{pull_request: {branches: [''*'']}}' +core_import_allowed: [] coverage: false deploy_client_to_pypi: true deploy_client_to_rubygems: true @@ -16,7 +22,9 @@ deploy_daily_client_to_rubygems: true deploy_to_pypi: true docker_fixtures: false docs_test: true +flake8: true issue_tracker: github +noissue_marker: '[noissue]' plugin_app_label: python plugin_camel: PulpPython plugin_camel_short: Python @@ -27,15 +35,22 @@ plugin_dash_short: python plugin_default_branch: master plugin_name: pulp_python plugin_snake: pulp_python +post_job_template: null +pre_job_template: null publish_docs_to_pulpprojectdotorg: false +pulp_scheme: http pulp_settings: null -pulpcore_branch: master -pulpcore_pip_version_specifier: null +pulpcore_branch: 3.11 +pulpcore_pip_version_specifier: <3.12 pulpprojectdotorg_key_id: null pydocstyle: true pypi_username: pulp +python_version: '3.6' redmine_project: null +release_email: pulp-infra@redhat.com +release_user: pulpbot stable_branch: null +sync_ci: true test_bindings: false test_cli: false test_fips_nightly: false @@ -45,4 +60,5 @@ test_s3: false travis_addtl_services: [] travis_notifications: None update_redmine: false +upgrade_range: []