diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index d6b3a8c03..eaff66a86 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -1,4 +1,4 @@ -FROM {{ ci_base | default("pulp/pulp-ci-centos:latest") }} +FROM {{ ci_base | default("pulp/pulp-ci-centos:" + pulp_container_tag) }} # Add source directories to container {% for item in plugins %} @@ -23,7 +23,8 @@ RUN pip3 install \ RUN mkdir -p /etc/nginx/pulp/ {% for item in plugins %} -RUN ln /usr/local/lib/python3.6/site-packages/{{ item.name }}/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ item.name }}.conf || true +RUN export plugin_path="$(pip3 show {{ item.name }} | sed -n -e 's/Location: //p')/{{ item.name }}" && \ + ln $plugin_path/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ item.name }}.conf || true {% endfor %} ENTRYPOINT ["/init"] diff --git a/.ci/ansible/settings.py.j2 b/.ci/ansible/settings.py.j2 index 3de131929..e0e040a7a 100644 --- a/.ci/ansible/settings.py.j2 +++ b/.ci/ansible/settings.py.j2 @@ -1,9 +1,9 @@ -CONTENT_ORIGIN = "http://pulp:80" -ANSIBLE_API_HOSTNAME = "http://pulp:80" -ANSIBLE_CONTENT_HOSTNAME = "http://pulp:80/pulp/content" +CONTENT_ORIGIN = "{{ pulp_scheme }}://pulp:{{ 443 if pulp_scheme == 'https' else 80 }}" +ANSIBLE_API_HOSTNAME = "{{ pulp_scheme }}://pulp:{{ 443 if pulp_scheme == 'https' else 80 }}" +ANSIBLE_CONTENT_HOSTNAME = "{{ pulp_scheme }}://pulp:{{ 443 if pulp_scheme == 'https' else 80 }}/pulp/content" PRIVATE_KEY_PATH = "/etc/pulp/certs/token_private_key.pem" PUBLIC_KEY_PATH = "/etc/pulp/certs/token_public_key.pem" -TOKEN_SERVER = "http://pulp:80/token/" +TOKEN_SERVER = "{{ pulp_scheme }}://pulp:{{ 443 if pulp_scheme == 'https' else 80 }}/token/" TOKEN_SIGNATURE_ALGORITHM = "ES256" {% if pulp_settings %} diff --git a/.ci/ansible/start_container.yaml b/.ci/ansible/start_container.yaml index d75a7083a..8661bb88d 100644 --- a/.ci/ansible/start_container.yaml +++ b/.ci/ansible/start_container.yaml @@ -71,7 +71,8 @@ - name: "Wait for Pulp" uri: url: "http://pulp/pulp/api/v3/status/" - follow_redirects: none + follow_redirects: all + validate_certs: no register: result until: result.status == 200 retries: 12 diff --git a/.ci/scripts/cherrypick.sh b/.ci/scripts/cherrypick.sh index 44469f64c..d9f7a8be8 100755 --- a/.ci/scripts/cherrypick.sh +++ b/.ci/scripts/cherrypick.sh @@ -9,6 +9,11 @@ set -e +if [ ! -d CHANGES ]; then + echo "Error: no CHANGES directory detected. This script must be run from the project root." + exit 1 +fi + if [ $# -lt 3 ] then echo "Usage: .ci/scripts/cherrypick.sh [commit-hash] [original-issue-id] [backport-issue-id]" diff --git a/.ci/scripts/validate_commit_message.py b/.ci/scripts/validate_commit_message.py index 59073fe63..caf62d86b 100755 --- a/.ci/scripts/validate_commit_message.py +++ b/.ci/scripts/validate_commit_message.py @@ -11,6 +11,7 @@ from pathlib import Path +import os from github import Github @@ -22,7 +23,7 @@ sha = sys.argv[1] message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8") -g = Github() +g = Github(os.environ.get("GITHUB_TOKEN")) repo = g.get_repo("pulp/pulp_python") diff --git a/.github/template_gitref b/.github/template_gitref new file mode 100644 index 000000000..e5dfb51e1 --- /dev/null +++ b/.github/template_gitref @@ -0,0 +1 @@ +2021.04.08-97-g74b81ba diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e5848796f..e594c8d03 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,16 +5,15 @@ # # For more info visit https://github.com/pulp/plugin_template --- -name: Pulp CI -on: - pull_request: - branches: - - '*' - +name: Python CI +on: {pull_request: {branches: ['*']}} jobs: + + lint: runs-on: ubuntu-latest + steps: - uses: actions/checkout@v2 @@ -25,7 +24,7 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: "3.7" + python-version: "3.6" # dev_requirements contains tools needed for flake8, etc. - name: Install requirements @@ -34,6 +33,7 @@ jobs: - name: Check commit message if: github.event_name == 'pull_request' env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} run: sh .github/workflows/scripts/check_commit.sh @@ -63,7 +63,6 @@ jobs: env: - TEST: pulp - TEST: docs - - TEST: s3 steps: - uses: actions/checkout@v2 @@ -74,7 +73,7 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: "3.7" + python-version: "3.6" - name: Install httpie run: | @@ -116,6 +115,11 @@ jobs: - name: Script run: .github/workflows/scripts/script.sh shell: bash + # env vars useful for post_script.sh when present + env: + GITHUB_PULL_REQUEST: ${{ github.event.number }} + GITHUB_BRANCH: ${{ github.head_ref }} + GITHUB_REPO_SLUG: ${{ github.repository }} - name: After failure if: failure() @@ -128,3 +132,6 @@ jobs: docker exec pulp ls -latr /etc/yum.repos.d/ || true docker exec pulp cat /etc/yum.repos.d/* || true docker exec pulp pip3 list + + + diff --git a/.github/workflows/create-branch.yml b/.github/workflows/create-branch.yml new file mode 100644 index 000000000..5486d5bd2 --- /dev/null +++ b/.github/workflows/create-branch.yml @@ -0,0 +1,79 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template +--- +name: Create New Release Branch +on: + workflow_dispatch: + inputs: + name: + description: "Branch name (e.g. 3.14)" + required: true + +env: + RELEASE_WORKFLOW: true + +jobs: + create-branch: + runs-on: ubuntu-latest + + strategy: + fail-fast: false + + steps: + - uses: actions/checkout@v2 + with: + # by default, it uses a depth of 1 + # this fetches all history so that we can read each commit + fetch-depth: 0 + + - uses: actions/setup-python@v2 + with: + python-version: "3.8" + + - name: Install python dependencies + run: | + echo ::group::PYDEPS + pip install bump2version + echo ::endgroup:: + + - name: Setting secrets + run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" + env: + SECRETS_CONTEXT: ${{ toJson(secrets) }} + + - name: Verify that branch name matches current version string on master branch + run: | + X_Y_VERSION=$(grep version setup.py | sed -rn 's/version="(.*)\.0\.dev",/\1/p' | awk '{$1=$1};1') + if [[ "$X_Y_VERSION" != "${{ github.event.inputs.name }}" ]] + then + echo "Branch name doesn't match the current version string $X_Y_VERSION." + exit 1 + fi + + - name: Create ${{ github.event.inputs.name }} release branch + run: | + git checkout -b ${{ github.event.inputs.name }} + git push origin ${{ github.event.inputs.name }} + + - name: Bump version on master branch + run: | + git checkout master + bump2version --no-commit minor + + - name: Make a PR with version bump + uses: peter-evans/create-pull-request@v3 + with: + committer: pulpbot + author: pulpbot + branch: minor-version-bump + base: master + title: Bump minor version + body: '[noissue]' + commit-message: | + Bump minor version + [noissue] + delete-branch: true diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 8b4114832..993004d78 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -5,7 +5,7 @@ # # For more info visit https://github.com/pulp/plugin_template --- -name: Pulp Nightly CI/CD +name: Python Nightly CI/CD on: schedule: # * is a special character in YAML so you have to quote this string @@ -22,7 +22,6 @@ jobs: env: - TEST: pulp - TEST: docs - - TEST: s3 - TEST: generate-bindings steps: @@ -34,7 +33,7 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: "3.7" + python-version: "3.6" - name: Install httpie run: | @@ -49,6 +48,11 @@ jobs: run: .github/workflows/scripts/before_install.sh shell: bash + - uses: ruby/setup-ruby@v1 + if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} + with: + ruby-version: "2.6" + - name: Install run: .github/workflows/scripts/install.sh env: @@ -97,6 +101,17 @@ jobs: name: docs.tar path: docs/docs.tar + - name: After failure + if: failure() + run: | + http --timeout 30 --check-status --pretty format --print hb http://pulp/pulp/api/v3/status/ || true + docker images || true + docker ps -a || true + docker logs pulp || true + docker exec pulp ls -latr /etc/yum.repos.d/ || true + docker exec pulp cat /etc/yum.repos.d/* || true + docker exec pulp pip3 list + publish: runs-on: ubuntu-latest needs: test @@ -113,7 +128,7 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: "3.7" + python-version: "3.6" - uses: actions/setup-ruby@v1 with: @@ -196,3 +211,14 @@ jobs: tar -xvf docs.tar -C ./docs .github/workflows/scripts/publish_docs.sh nightly ${GITHUB_REF##*/} + + - name: After failure + if: failure() + run: | + http --timeout 30 --check-status --pretty format --print hb http://pulp/pulp/api/v3/status/ || true + docker images || true + docker ps -a || true + docker logs pulp || true + docker exec pulp ls -latr /etc/yum.repos.d/ || true + docker exec pulp cat /etc/yum.repos.d/* || true + docker exec pulp pip3 list diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 86fb5302f..60958260f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,6 +12,11 @@ on: release: description: "Release tag (e.g. 3.2.1)" required: true + before_script: + description: | + Bash code to run before script.sh is executed. This should only be used when re-running + a workflow to correct some aspect of the docs. e.g.: git checkout origin/3.14 CHANGES.rst + required: false env: RELEASE_WORKFLOW: true @@ -32,12 +37,12 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: "3.7" + python-version: "3.8" - name: Install python dependencies run: | echo ::group::PYDEPS - pip install bandersnatch bump2version gitpython python-redmine towncrier wheel + pip install bandersnatch bump2version gitpython python-redmine towncrier==19.9.0 wheel echo ::endgroup:: - name: Configure Git with pulpbot name and email @@ -72,7 +77,6 @@ jobs: env: - TEST: pulp - TEST: docs - - TEST: s3 - TEST: generate-bindings steps: @@ -82,7 +86,7 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: "3.7" + python-version: "3.6" - uses: actions/setup-ruby@v1 with: ruby-version: "2.6" @@ -139,6 +143,10 @@ jobs: if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} run: .github/workflows/scripts/install_ruby_client.sh + - name: Additional before_script + run: ${{ github.event.inputs.before_script }} + shell: bash + - name: Script if: ${{ env.TEST != 'generate-bindings' }} run: .github/workflows/scripts/script.sh @@ -192,12 +200,17 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: "3.7" + python-version: "3.8" - uses: actions/setup-ruby@v1 with: ruby-version: "2.6" + - name: Configure Git with pulpbot name and email + run: | + git config --global user.name 'pulpbot' + git config --global user.email 'pulp-infra@redhat.com' + - name: Untar repository run: | shopt -s dotglob @@ -217,6 +230,12 @@ jobs: env: SECRETS_CONTEXT: ${{ toJson(secrets) }} + - name: Install python dependencies + run: | + echo ::group::PYDEPS + pip install gitpython python-redmine + echo ::endgroup:: + - name: Push branch and tag to GitHub run: bash .github/workflows/scripts/push_branch_and_tag_to_github.sh ${{ github.event.inputs.release }} - name: Download built docs @@ -251,5 +270,28 @@ jobs: - name: Publish client to rubygems run: bash .github/workflows/scripts/publish_client_gem.sh + + - name: Create release on GitHub run: bash .github/workflows/scripts/create_release_from_tag.sh ${{ github.event.inputs.release }} + + - name: Cleanup repository before making changelog PR + run: rm -rf .lock generation pulp_python_client* *-client.tar pulp_python.tar todo web docs.tar + + - name: Stage changelog for master branch + run: python .github/workflows/scripts/stage-changelog-for-master.py ${{ github.event.inputs.release }} + + - name: Create Pull Request for Changelog + uses: peter-evans/create-pull-request@v3 + with: + committer: pulpbot + author: pulpbot + branch: changelog/${{ github.event.inputs.release }} + base: master + title: 'Building changelog for ${{ github.event.inputs.release }}' + body: '[noissue]' + commit-message: | + Building changelog for ${{ github.event.inputs.release }} + + [noissue] + delete-branch: true diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index acfbf2ba3..9f04d3124 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -27,6 +27,9 @@ else BRANCH="${GITHUB_REF##refs/tags/}" fi +COMMIT_MSG=$(git log --format=%B --no-merges -1) +export COMMIT_MSG + if [[ "$TEST" == "upgrade" ]]; then git checkout -b ci_upgrade_test cp -R .github /tmp/.github @@ -35,14 +38,12 @@ if [[ "$TEST" == "upgrade" ]]; then rm -rf .ci .github cp -R /tmp/.github . cp -R /tmp/.ci . - # Pin deps - sed -i "s/~/=/g" requirements.txt fi if [[ "$TEST" == "plugin-from-pypi" ]]; then COMPONENT_VERSION=$(http https://pypi.org/pypi/pulp-python/json | jq -r '.info.version') else - COMPONENT_VERSION=$(sed -ne "s/\s*version=['\"]\(.*\)['\"][\s,]*/\1/p" setup.py) + COMPONENT_VERSION=$(sed -ne "s/\s*version.*=.*['\"]\(.*\)['\"][\s,]*/\1/p" setup.py) fi mkdir .ci/ansible/vars || true echo "---" > .ci/ansible/vars/main.yaml @@ -53,9 +54,6 @@ echo "component_version: '${COMPONENT_VERSION}'" >> .ci/ansible/vars/main.yaml export PRE_BEFORE_INSTALL=$PWD/.github/workflows/scripts/pre_before_install.sh export POST_BEFORE_INSTALL=$PWD/.github/workflows/scripts/post_before_install.sh -COMMIT_MSG=$(git log --format=%B --no-merges -1) -export COMMIT_MSG - if [ -f $PRE_BEFORE_INSTALL ]; then source $PRE_BEFORE_INSTALL fi @@ -106,7 +104,7 @@ fi -git clone --depth=1 https://github.com/pulp/pulpcore.git --branch master +git clone --depth=1 https://github.com/pulp/pulpcore.git --branch 3.13 cd pulpcore if [ -n "$PULPCORE_PR_NUMBER" ]; then diff --git a/.github/workflows/scripts/check_commit.sh b/.github/workflows/scripts/check_commit.sh index 5647c1a48..7780e8be0 100755 --- a/.github/workflows/scripts/check_commit.sh +++ b/.github/workflows/scripts/check_commit.sh @@ -19,7 +19,7 @@ pip3 install pygithub echo ::endgroup:: -for sha in $(curl $GITHUB_CONTEXT | jq '.[].sha' | sed 's/"//g') +for sha in $(curl -H "Authorization: token $GITHUB_TOKEN" $GITHUB_CONTEXT | jq '.[].sha' | sed 's/"//g') do python3 .ci/scripts/validate_commit_message.py $sha VALUE=$? diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index b3ad3b7b9..de4203d82 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -41,7 +41,7 @@ image: tag: "${TAG}" plugins: - name: pulpcore - source: pulpcore + source: pulpcore~=3.13.0 - name: pulp_python source: "${PLUGIN_NAME}" services: @@ -70,6 +70,10 @@ fi cat >> vars/main.yaml << VARSYAML pulp_settings: null +pulp_scheme: http + +pulp_container_tag: python36 + VARSYAML ansible-playbook build_container.yaml diff --git a/.github/workflows/scripts/install_python_client.sh b/.github/workflows/scripts/install_python_client.sh index ecad00fe3..226a40f6b 100755 --- a/.github/workflows/scripts/install_python_client.sh +++ b/.github/workflows/scripts/install_python_client.sh @@ -16,7 +16,7 @@ cd "$(dirname "$(realpath -e "$0")")"/../../.. pip install twine wheel -export REPORTED_VERSION=$(http pulp/pulp/api/v3/status/ | jq --arg plugin python --arg legacy_plugin pulp_python -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version') +export REPORTED_VERSION=$(http $PULP_URL/pulp/api/v3/status/ | jq --arg plugin python --arg legacy_plugin pulp_python -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version') export DESCRIPTION="$(git describe --all --exact-match `git rev-parse HEAD`)" if [[ $DESCRIPTION == 'tags/'$REPORTED_VERSION ]]; then export VERSION=${REPORTED_VERSION} @@ -37,10 +37,10 @@ then fi cd ../pulp-openapi-generator - +rm -rf pulp_python-client ./generate.sh pulp_python python $VERSION cd pulp_python-client python setup.py sdist bdist_wheel --python-tag py3 -pip install dist/pulp_python_client-$VERSION-py3-none-any.whl +find . -name "*.whl" -exec pip install {} \; tar cvf ../../pulp_python/python-client.tar ./dist exit $? diff --git a/.github/workflows/scripts/install_ruby_client.sh b/.github/workflows/scripts/install_ruby_client.sh index 4e459a07c..f3a0448ce 100755 --- a/.github/workflows/scripts/install_ruby_client.sh +++ b/.github/workflows/scripts/install_ruby_client.sh @@ -14,7 +14,7 @@ cd "$(dirname "$(realpath -e "$0")")"/../../.. export PULP_URL="${PULP_URL:-http://pulp}" -export REPORTED_VERSION=$(http pulp/pulp/api/v3/status/ | jq --arg plugin python --arg legacy_plugin pulp_python -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version') +export REPORTED_VERSION=$(http $PULP_URL/pulp/api/v3/status/ | jq --arg plugin python --arg legacy_plugin pulp_python -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version') export DESCRIPTION="$(git describe --all --exact-match `git rev-parse HEAD`)" if [[ $DESCRIPTION == 'tags/'$REPORTED_VERSION ]]; then export VERSION=${REPORTED_VERSION} @@ -35,7 +35,7 @@ then fi cd ../pulp-openapi-generator - +rm -rf pulp_python-client ./generate.sh pulp_python ruby $VERSION cd pulp_python-client gem build pulp_python_client diff --git a/.github/workflows/scripts/publish_plugin_pypi.sh b/.github/workflows/scripts/publish_plugin_pypi.sh index 29c2a8c50..981f77073 100755 --- a/.github/workflows/scripts/publish_plugin_pypi.sh +++ b/.github/workflows/scripts/publish_plugin_pypi.sh @@ -12,8 +12,6 @@ cd "$(dirname "$(realpath -e "$0")")"/../../.. set -euv -export PULP_URL="${PULP_URL:-http://pulp}" - export response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/pulp-python/$1/) if [ "$response" == "200" ]; then diff --git a/.github/workflows/scripts/release.py b/.github/workflows/scripts/release.py index 5c900bee1..a47f7f3bf 100755 --- a/.github/workflows/scripts/release.py +++ b/.github/workflows/scripts/release.py @@ -128,10 +128,11 @@ def create_tag_and_build_package(repo, desired_tag, commit_sha, plugin_path): # Check if Package is available on PyPI loop = asyncio.get_event_loop() # noqa + # fmt: off package_found = asyncio.run( get_package_from_pypi("pulp-python=={tag.name}", plugin_path) ) # noqa - + # fmt: on if not package_found: os.system("python3 setup.py sdist bdist_wheel --python-tag py3") diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index 04d52edfd..9c116015a 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -46,7 +46,7 @@ if [[ "$TEST" = "docs" ]]; then fi if [[ "${RELEASE_WORKFLOW:-false}" == "true" ]]; then - REPORTED_VERSION=$(http pulp/pulp/api/v3/status/ | jq --arg plugin python --arg legacy_plugin pulp_python -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version') + REPORTED_VERSION=$(http $PULP_URL/pulp/api/v3/status/ | jq --arg plugin python --arg legacy_plugin pulp_python -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version') response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/pulp-python/$REPORTED_VERSION/) if [ "$response" == "200" ]; then diff --git a/.github/workflows/scripts/stage-changelog-for-master.py b/.github/workflows/scripts/stage-changelog-for-master.py new file mode 100755 index 000000000..649f70578 --- /dev/null +++ b/.github/workflows/scripts/stage-changelog-for-master.py @@ -0,0 +1,64 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +import argparse +import os +import textwrap + +from git import Repo +from git.exc import GitCommandError + + +helper = textwrap.dedent( + """\ + Stage the changelog for a release on master branch. + + Example: + $ python .github/workflows/scripts/stage-changelog-for-master.py 3.4.0 + + """ +) + +parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, description=helper) + +parser.add_argument( + "release_version", + type=str, + help="The version string for the release.", +) + +args = parser.parse_args() + +release_version_arg = args.release_version + +release_path = os.path.dirname(os.path.abspath(__file__)) +plugin_path = release_path.split("/.github")[0] + +print(f"\n\nRepo path: {plugin_path}") +repo = Repo(plugin_path) + +changelog_commit = None +# Look for a commit with the requested release version +for commit in repo.iter_commits(): + if f"Building changelog for {release_version_arg}\n" in commit.message: + changelog_commit = commit + break + +if not changelog_commit: + raise RuntimeError("Changelog commit for {release_version_arg} was not found.") + +git = repo.git +git.stash() +git.checkout("origin/master") +try: + git.cherry_pick(changelog_commit.hexsha) +except GitCommandError: + git.add("CHANGES/") + # Don't try opening an editor for the commit message + with git.custom_environment(GIT_EDITOR="true"): + git.cherry_pick("--continue") +git.reset("origin/master") diff --git a/.github/workflows/scripts/update_ci.sh b/.github/workflows/scripts/update_ci.sh new file mode 100755 index 000000000..c83c94541 --- /dev/null +++ b/.github/workflows/scripts/update_ci.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +set -eu + +if [ ! -d ../plugin_template ]; then + echo "Checking out plugin_template" + git clone https://github.com/pulp/plugin_template.git ../plugin_template +fi + + +if [ ! -f "template_config.yml" ]; then + echo "No template_config.yml detected." + exit 1 +fi + +pushd ../plugin_template +./plugin-template --github pulp_python +popd + +if [[ `git status --porcelain` ]]; then + git add -A + git commit -m "Update CI files" -m "[noissue]" +else + echo "No updates needed" +fi diff --git a/.github/workflows/update_ci.yml b/.github/workflows/update_ci.yml new file mode 100644 index 000000000..2b432b256 --- /dev/null +++ b/.github/workflows/update_ci.yml @@ -0,0 +1,48 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template +--- +name: CI Update +on: + schedule: + # * is a special character in YAML so you have to quote this string + # runs at 2:30 UTC daily + - cron: '30 2 * * *' + + workflow_dispatch: + +jobs: + update: + runs-on: ubuntu-latest + + strategy: + fail-fast: false + + steps: + - uses: actions/checkout@v2 + + - name: Configure Git with pulpbot name and email + run: | + git config --global user.name 'pulpbot' + git config --global user.email 'pulp-infra@redhat.com' + + - name: Run update + run: | + .github/workflows/scripts/update_ci.sh + + - name: Create Pull Request for CI files + uses: peter-evans/create-pull-request@v3 + with: + token: ${{ secrets.RELEASE_TOKEN }} + committer: pulpbot + author: pulpbot + title: 'Update CI files' + body: '[noissue]' + commit-message: | + Update CI files + + [noissue] + delete-branch: true diff --git a/CHANGES/418.bugfix b/CHANGES/418.bugfix new file mode 100644 index 000000000..ed48855d5 --- /dev/null +++ b/CHANGES/418.bugfix @@ -0,0 +1,2 @@ +Fixed improper metadata serving when using publications with S3 storage +(backported from #413) diff --git a/CHANGES/419.feature b/CHANGES/419.feature new file mode 100644 index 000000000..c3811cabe --- /dev/null +++ b/CHANGES/419.feature @@ -0,0 +1,2 @@ +Python package content can now be filtered by their sha256 +(backported from #404) diff --git a/CHANGES/420.bugfix b/CHANGES/420.bugfix new file mode 100644 index 000000000..b332cc583 --- /dev/null +++ b/CHANGES/420.bugfix @@ -0,0 +1,2 @@ +Fixed twine upload failing when using remote storage backends +(backported from #400) diff --git a/README.md b/README.md index 4808f9b18..b164c4170 100644 --- a/README.md +++ b/README.md @@ -6,5 +6,5 @@ A Pulp plugin to support hosting your own pip compatible Python packages. -For more information, please see the [documentation](https://pulp-python.readthedocs.io/en/latest/) or the +For more information, please see the [documentation](https://docs.pulpproject.org/pulp_python/) or the [Pulp project page](https://pulpproject.org). diff --git a/pulp_python/app/models.py b/pulp_python/app/models.py index 207d8b0c5..a6bd4d922 100644 --- a/pulp_python/app/models.py +++ b/pulp_python/app/models.py @@ -2,7 +2,10 @@ from aiohttp.web import json_response from django.contrib.postgres.fields import ArrayField, JSONField +from django.core.exceptions import ObjectDoesNotExist from django.db import models +from dynaconf import settings +from yarl import URL from pulpcore.plugin.models import ( Content, @@ -63,6 +66,38 @@ def content_handler(self, path): name = path.parts[1] elif path.match("pypi/*/json"): name = path.parts[1] + elif len(path.parts) and path.parts[0] == "simple": + # Temporary fix for PublishedMetadata not being properly served from remote storage + # https://github.com/pulp/pulp_python/issues/413 + if settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem": + try: + publication = self.publication or Publication.objects.filter( + repository_version=self.repository.latest_version()).latest("pulp_created") + except ObjectDoesNotExist: + return None + rel_path = f"{path}/index.html" + try: + ca = publication.published_artifact.select_related( + "content_artifact", + "content_artifact__artifact", + ).get(relative_path=rel_path).content_artifact + except ObjectDoesNotExist: + return None + file = ca.artifact.file + content_disposition = f"attachment;filename={ca.relative_path}" + parameters = { + "ResponseContentDisposition": content_disposition, + "ResponseContentType": "text/html" + } + url = URL(file.storage.url(file.name, parameters=parameters), encoded=True) + # Trick the content app to stream the metadata from the remote storage + remote = PythonRemote(name="Redirect", url=str(url), policy="streamed") + remote.get_remote_artifact_url = lambda x: str(url) + setattr(self, "publication", None) + setattr(self, "repository", None) + setattr(self, "remote", remote) + return None + if name: package_content = PythonPackageContent.objects.filter( pk__in=self.publication.repository_version.content, diff --git a/pulp_python/app/serializers.py b/pulp_python/app/serializers.py index 2dc4ee226..fd7f779a7 100644 --- a/pulp_python/app/serializers.py +++ b/pulp_python/app/serializers.py @@ -1,8 +1,8 @@ from gettext import gettext as _ -import os import shutil import tempfile +from django.core.files.storage import default_storage as storage from packaging.requirements import Requirement from rest_framework import serializers @@ -237,11 +237,12 @@ def deferred_validate(self, data): # Copy file to a temp directory under the user provided filename, we do this # because pkginfo validates that the filename has a valid extension before # reading it - with tempfile.TemporaryDirectory() as td: - temp_path = os.path.join(td, filename) + with tempfile.NamedTemporaryFile('wb', suffix=filename) as temp_file: artifact = data["artifact"] - shutil.copy2(artifact.file.path, temp_path) - metadata = DIST_TYPES[packagetype](temp_path) + artifact_file = storage.open(artifact.file.name) + shutil.copyfileobj(artifact_file, temp_file) + temp_file.flush() + metadata = DIST_TYPES[packagetype](temp_file.name) metadata.packagetype = packagetype break else: diff --git a/pulp_python/app/tasks/publish.py b/pulp_python/app/tasks/publish.py index 8d8d363a9..c4fb0478b 100644 --- a/pulp_python/app/tasks/publish.py +++ b/pulp_python/app/tasks/publish.py @@ -1,7 +1,6 @@ from gettext import gettext as _ import logging import os -import tempfile from django.core.files import File from packaging.utils import canonicalize_name @@ -30,12 +29,11 @@ def publish(repository_version_pk): version=repository_version.number, )) - with tempfile.TemporaryDirectory("."): - with python_models.PythonPublication.create(repository_version, pass_through=True) as pub: - write_simple_api(pub) + with python_models.PythonPublication.create(repository_version, pass_through=True) as pub: + write_simple_api(pub) - log.info(_('Publication: {pk} created').format(pk=pub.pk)) - return pub + log.info(_('Publication: {pk} created').format(pk=pub.pk)) + return pub def write_simple_api(publication): diff --git a/pulp_python/app/tasks/upload.py b/pulp_python/app/tasks/upload.py index 2629ee066..39b11d5ff 100644 --- a/pulp_python/app/tasks/upload.py +++ b/pulp_python/app/tasks/upload.py @@ -1,4 +1,3 @@ -import os import pkginfo import shutil import tempfile @@ -7,6 +6,7 @@ from datetime import datetime, timezone from django.db import transaction from django.contrib.sessions.models import Session +from django.core.files.storage import default_storage as storage from pulpcore.plugin.models import Artifact, CreatedResource, ContentArtifact from pulp_python.app.models import PythonPackageContent, PythonRepository @@ -99,11 +99,12 @@ def create_content(artifact_sha256, filename): # Copy file to a temp directory under the user provided filename, we do this # because pkginfo validates that the filename has a valid extension before # reading it - with tempfile.TemporaryDirectory() as td: - temp_path = os.path.join(td, filename) - artifact = Artifact.objects.get(sha256=artifact_sha256) - shutil.copy2(artifact.file.path, temp_path) - metadata = DIST_TYPES[packagetype](temp_path) + artifact = Artifact.objects.get(sha256=artifact_sha256) + artifact_file = storage.open(artifact.file.name) + with tempfile.NamedTemporaryFile('wb', suffix=filename) as temp_file: + shutil.copyfileobj(artifact_file, temp_file) + temp_file.flush() + metadata = DIST_TYPES[packagetype](temp_file.name) metadata.packagetype = packagetype data = parse_project_metadata(vars(metadata)) diff --git a/pulp_python/app/viewsets.py b/pulp_python/app/viewsets.py index 737f1df68..8e148392b 100644 --- a/pulp_python/app/viewsets.py +++ b/pulp_python/app/viewsets.py @@ -99,6 +99,7 @@ class Meta: 'requires_python': ['exact', 'in', "contains"], 'filename': ['exact', 'in', 'contains'], 'keywords': ['in', 'contains'], + 'sha256': ['exact', 'in'], } diff --git a/pulp_python/tests/functional/api/test_pypi_apis.py b/pulp_python/tests/functional/api/test_pypi_apis.py index f0e2d35be..6910d7223 100644 --- a/pulp_python/tests/functional/api/test_pypi_apis.py +++ b/pulp_python/tests/functional/api/test_pypi_apis.py @@ -179,7 +179,6 @@ def test_twine_upload(self): "-p", password, ), - capture_output=True, check=True, ) tasks = task_api.list(reserved_resources_record=repo.pulp_href).results @@ -203,7 +202,6 @@ def test_twine_upload(self): "-p", password, ), - capture_output=True, check=True, ) @@ -221,9 +219,9 @@ def test_twine_upload(self): password, "--skip-existing", ), - capture_output=True, + stdout=subprocess.PIPE, check=True, - text=True + universal_newlines=True ) self.assertEqual(output.stdout.count("Skipping"), 2) diff --git a/template_config.yml b/template_config.yml index 40fba39f1..f18351b58 100644 --- a/template_config.yml +++ b/template_config.yml @@ -1,6 +1,8 @@ # This config represents the latest values used when running the plugin-template. Any settings that # were not present before running plugin-template have been added with their default values. +# generated with plugin_template@2021.04.08-97-g74b81ba + additional_plugins: [] additional_repos: [] black: false @@ -10,6 +12,8 @@ check_manifest: true check_openapi_schema: true check_stray_pulpcore_imports: true cherry_pick_automation: false +ci_trigger: '{pull_request: {branches: [''*'']}}' +core_import_allowed: [] coverage: false deploy_client_to_pypi: true deploy_client_to_rubygems: true @@ -18,7 +22,9 @@ deploy_daily_client_to_rubygems: true deploy_to_pypi: true docker_fixtures: false docs_test: true +flake8: true issue_tracker: github +noissue_marker: '[noissue]' plugin_app_label: python plugin_camel: PulpPython plugin_camel_short: Python @@ -29,16 +35,22 @@ plugin_dash_short: python plugin_default_branch: master plugin_name: pulp_python plugin_snake: pulp_python +post_job_template: null +pre_job_template: null publish_docs_to_pulpprojectdotorg: true +pulp_scheme: http pulp_settings: null pulpcore_branch: 3.13 pulpcore_pip_version_specifier: ~=3.13.0 pulpprojectdotorg_key_id: null pydocstyle: true pypi_username: pulp +python_version: '3.6' redmine_project: null +release_email: pulp-infra@redhat.com release_user: pulpbot stable_branch: null +sync_ci: true test_bindings: false test_cli: false test_fips_nightly: false