diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index d90e19c03..eaff66a86 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -1,4 +1,4 @@ -FROM {{ ci_base | default("pulp/pulp-ci-centos:latest") }} +FROM {{ ci_base | default("pulp/pulp-ci-centos:" + pulp_container_tag) }} # Add source directories to container {% for item in plugins %} @@ -23,7 +23,8 @@ RUN pip3 install \ RUN mkdir -p /etc/nginx/pulp/ {% for item in plugins %} -RUN ln /usr/local/lib/python3.8/site-packages/{{ item.name }}/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ item.name }}.conf || true +RUN export plugin_path="$(pip3 show {{ item.name }} | sed -n -e 's/Location: //p')/{{ item.name }}" && \ + ln $plugin_path/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ item.name }}.conf || true {% endfor %} ENTRYPOINT ["/init"] diff --git a/.ci/ansible/settings.py.j2 b/.ci/ansible/settings.py.j2 index 3de131929..e0e040a7a 100644 --- a/.ci/ansible/settings.py.j2 +++ b/.ci/ansible/settings.py.j2 @@ -1,9 +1,9 @@ -CONTENT_ORIGIN = "http://pulp:80" -ANSIBLE_API_HOSTNAME = "http://pulp:80" -ANSIBLE_CONTENT_HOSTNAME = "http://pulp:80/pulp/content" +CONTENT_ORIGIN = "{{ pulp_scheme }}://pulp:{{ 443 if pulp_scheme == 'https' else 80 }}" +ANSIBLE_API_HOSTNAME = "{{ pulp_scheme }}://pulp:{{ 443 if pulp_scheme == 'https' else 80 }}" +ANSIBLE_CONTENT_HOSTNAME = "{{ pulp_scheme }}://pulp:{{ 443 if pulp_scheme == 'https' else 80 }}/pulp/content" PRIVATE_KEY_PATH = "/etc/pulp/certs/token_private_key.pem" PUBLIC_KEY_PATH = "/etc/pulp/certs/token_public_key.pem" -TOKEN_SERVER = "http://pulp:80/token/" +TOKEN_SERVER = "{{ pulp_scheme }}://pulp:{{ 443 if pulp_scheme == 'https' else 80 }}/token/" TOKEN_SIGNATURE_ALGORITHM = "ES256" {% if pulp_settings %} diff --git a/.ci/ansible/smash-config.json b/.ci/ansible/smash-config.json index 9af43820b..308fe38b9 100644 --- a/.ci/ansible/smash-config.json +++ b/.ci/ansible/smash-config.json @@ -12,13 +12,13 @@ "hostname": "pulp", "roles": { "api": { - "port": 80, - "scheme": "http", + "port": 443, + "scheme": "https", "service": "nginx" }, "content": { - "port": 80, - "scheme": "http", + "port": 443, + "scheme": "https", "service": "pulp_content_app" }, "pulp resource manager": {}, diff --git a/.ci/ansible/start_container.yaml b/.ci/ansible/start_container.yaml index d75a7083a..8661bb88d 100644 --- a/.ci/ansible/start_container.yaml +++ b/.ci/ansible/start_container.yaml @@ -71,7 +71,8 @@ - name: "Wait for Pulp" uri: url: "http://pulp/pulp/api/v3/status/" - follow_redirects: none + follow_redirects: all + validate_certs: no register: result until: result.status == 200 retries: 12 diff --git a/.ci/scripts/cherrypick.sh b/.ci/scripts/cherrypick.sh index 44469f64c..d9f7a8be8 100755 --- a/.ci/scripts/cherrypick.sh +++ b/.ci/scripts/cherrypick.sh @@ -9,6 +9,11 @@ set -e +if [ ! -d CHANGES ]; then + echo "Error: no CHANGES directory detected. This script must be run from the project root." + exit 1 +fi + if [ $# -lt 3 ] then echo "Usage: .ci/scripts/cherrypick.sh [commit-hash] [original-issue-id] [backport-issue-id]" diff --git a/.github/template_gitref b/.github/template_gitref new file mode 100644 index 000000000..7756632c3 --- /dev/null +++ b/.github/template_gitref @@ -0,0 +1 @@ +2021.04.08-95-gbf7866c diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e5848796f..73a6ff365 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,16 +5,15 @@ # # For more info visit https://github.com/pulp/plugin_template --- -name: Pulp CI -on: - pull_request: - branches: - - '*' - +name: Python CI +on: {pull_request: {branches: ['*']}} jobs: + + lint: runs-on: ubuntu-latest + steps: - uses: actions/checkout@v2 @@ -25,7 +24,7 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: "3.7" + python-version: "3.8" # dev_requirements contains tools needed for flake8, etc. - name: Install requirements @@ -74,7 +73,7 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: "3.7" + python-version: "3.8" - name: Install httpie run: | @@ -116,15 +115,23 @@ jobs: - name: Script run: .github/workflows/scripts/script.sh shell: bash + # env vars useful for post_script.sh when present + env: + GITHUB_PULL_REQUEST: ${{ github.event.number }} + GITHUB_BRANCH: ${{ github.head_ref }} + GITHUB_REPO_SLUG: ${{ github.repository }} - name: After failure if: failure() run: | echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" - http --timeout 30 --check-status --pretty format --print hb http://pulp/pulp/api/v3/status/ || true + http --timeout 30 --check-status --pretty format --print hb https://pulp/pulp/api/v3/status/ || true docker images || true docker ps -a || true docker logs pulp || true docker exec pulp ls -latr /etc/yum.repos.d/ || true docker exec pulp cat /etc/yum.repos.d/* || true docker exec pulp pip3 list + + + diff --git a/.github/workflows/create-branch.yml b/.github/workflows/create-branch.yml new file mode 100644 index 000000000..5486d5bd2 --- /dev/null +++ b/.github/workflows/create-branch.yml @@ -0,0 +1,79 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template +--- +name: Create New Release Branch +on: + workflow_dispatch: + inputs: + name: + description: "Branch name (e.g. 3.14)" + required: true + +env: + RELEASE_WORKFLOW: true + +jobs: + create-branch: + runs-on: ubuntu-latest + + strategy: + fail-fast: false + + steps: + - uses: actions/checkout@v2 + with: + # by default, it uses a depth of 1 + # this fetches all history so that we can read each commit + fetch-depth: 0 + + - uses: actions/setup-python@v2 + with: + python-version: "3.8" + + - name: Install python dependencies + run: | + echo ::group::PYDEPS + pip install bump2version + echo ::endgroup:: + + - name: Setting secrets + run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" + env: + SECRETS_CONTEXT: ${{ toJson(secrets) }} + + - name: Verify that branch name matches current version string on master branch + run: | + X_Y_VERSION=$(grep version setup.py | sed -rn 's/version="(.*)\.0\.dev",/\1/p' | awk '{$1=$1};1') + if [[ "$X_Y_VERSION" != "${{ github.event.inputs.name }}" ]] + then + echo "Branch name doesn't match the current version string $X_Y_VERSION." + exit 1 + fi + + - name: Create ${{ github.event.inputs.name }} release branch + run: | + git checkout -b ${{ github.event.inputs.name }} + git push origin ${{ github.event.inputs.name }} + + - name: Bump version on master branch + run: | + git checkout master + bump2version --no-commit minor + + - name: Make a PR with version bump + uses: peter-evans/create-pull-request@v3 + with: + committer: pulpbot + author: pulpbot + branch: minor-version-bump + base: master + title: Bump minor version + body: '[noissue]' + commit-message: | + Bump minor version + [noissue] + delete-branch: true diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index f12868f44..73c4babfd 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -5,7 +5,7 @@ # # For more info visit https://github.com/pulp/plugin_template --- -name: Pulp Nightly CI/CD +name: Python Nightly CI/CD on: schedule: # * is a special character in YAML so you have to quote this string @@ -34,7 +34,7 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: "3.7" + python-version: "3.8" - name: Install httpie run: | @@ -102,6 +102,17 @@ jobs: name: docs.tar path: docs/docs.tar + - name: After failure + if: failure() + run: | + http --timeout 30 --check-status --pretty format --print hb https://pulp/pulp/api/v3/status/ || true + docker images || true + docker ps -a || true + docker logs pulp || true + docker exec pulp ls -latr /etc/yum.repos.d/ || true + docker exec pulp cat /etc/yum.repos.d/* || true + docker exec pulp pip3 list + publish: runs-on: ubuntu-latest needs: test @@ -118,7 +129,7 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: "3.7" + python-version: "3.8" - uses: actions/setup-ruby@v1 with: @@ -201,3 +212,14 @@ jobs: tar -xvf docs.tar -C ./docs .github/workflows/scripts/publish_docs.sh nightly ${GITHUB_REF##*/} + + - name: After failure + if: failure() + run: | + http --timeout 30 --check-status --pretty format --print hb https://pulp/pulp/api/v3/status/ || true + docker images || true + docker ps -a || true + docker logs pulp || true + docker exec pulp ls -latr /etc/yum.repos.d/ || true + docker exec pulp cat /etc/yum.repos.d/* || true + docker exec pulp pip3 list diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 86fb5302f..c72e8b120 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,6 +12,11 @@ on: release: description: "Release tag (e.g. 3.2.1)" required: true + before_script: + description: | + Bash code to run before script.sh is executed. This should only be used when re-running + a workflow to correct some aspect of the docs. e.g.: git checkout origin/3.14 CHANGES.rst + required: false env: RELEASE_WORKFLOW: true @@ -32,12 +37,12 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: "3.7" + python-version: "3.8" - name: Install python dependencies run: | echo ::group::PYDEPS - pip install bandersnatch bump2version gitpython python-redmine towncrier wheel + pip install bandersnatch bump2version gitpython python-redmine towncrier==19.9.0 wheel echo ::endgroup:: - name: Configure Git with pulpbot name and email @@ -82,7 +87,7 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: "3.7" + python-version: "3.8" - uses: actions/setup-ruby@v1 with: ruby-version: "2.6" @@ -139,6 +144,10 @@ jobs: if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} run: .github/workflows/scripts/install_ruby_client.sh + - name: Additional before_script + run: ${{ github.event.inputs.before_script }} + shell: bash + - name: Script if: ${{ env.TEST != 'generate-bindings' }} run: .github/workflows/scripts/script.sh @@ -168,7 +177,7 @@ jobs: - name: After failure if: failure() run: | - http --timeout 30 --check-status --pretty format --print hb http://pulp/pulp/api/v3/status/ || true + http --timeout 30 --check-status --pretty format --print hb https://pulp/pulp/api/v3/status/ || true docker images || true docker ps -a || true docker logs pulp || true @@ -192,12 +201,17 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: "3.7" + python-version: "3.8" - uses: actions/setup-ruby@v1 with: ruby-version: "2.6" + - name: Configure Git with pulpbot name and email + run: | + git config --global user.name 'pulpbot' + git config --global user.email 'pulp-infra@redhat.com' + - name: Untar repository run: | shopt -s dotglob @@ -217,6 +231,12 @@ jobs: env: SECRETS_CONTEXT: ${{ toJson(secrets) }} + - name: Install python dependencies + run: | + echo ::group::PYDEPS + pip install gitpython python-redmine + echo ::endgroup:: + - name: Push branch and tag to GitHub run: bash .github/workflows/scripts/push_branch_and_tag_to_github.sh ${{ github.event.inputs.release }} - name: Download built docs @@ -251,5 +271,28 @@ jobs: - name: Publish client to rubygems run: bash .github/workflows/scripts/publish_client_gem.sh + + - name: Create release on GitHub run: bash .github/workflows/scripts/create_release_from_tag.sh ${{ github.event.inputs.release }} + + - name: Cleanup repository before making changelog PR + run: rm -rf .lock generation pulp_python_client* *-client.tar pulp_python.tar todo web docs.tar + + - name: Stage changelog for master branch + run: python .github/workflows/scripts/stage-changelog-for-master.py ${{ github.event.inputs.release }} + + - name: Create Pull Request for Changelog + uses: peter-evans/create-pull-request@v3 + with: + committer: pulpbot + author: pulpbot + branch: changelog/${{ github.event.inputs.release }} + base: master + title: 'Building changelog for ${{ github.event.inputs.release }}' + body: '[noissue]' + commit-message: | + Building changelog for ${{ github.event.inputs.release }} + + [noissue] + delete-branch: true diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index acfbf2ba3..677a640b2 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -27,6 +27,9 @@ else BRANCH="${GITHUB_REF##refs/tags/}" fi +COMMIT_MSG=$(git log --format=%B --no-merges -1) +export COMMIT_MSG + if [[ "$TEST" == "upgrade" ]]; then git checkout -b ci_upgrade_test cp -R .github /tmp/.github @@ -35,14 +38,12 @@ if [[ "$TEST" == "upgrade" ]]; then rm -rf .ci .github cp -R /tmp/.github . cp -R /tmp/.ci . - # Pin deps - sed -i "s/~/=/g" requirements.txt fi if [[ "$TEST" == "plugin-from-pypi" ]]; then COMPONENT_VERSION=$(http https://pypi.org/pypi/pulp-python/json | jq -r '.info.version') else - COMPONENT_VERSION=$(sed -ne "s/\s*version=['\"]\(.*\)['\"][\s,]*/\1/p" setup.py) + COMPONENT_VERSION=$(sed -ne "s/\s*version.*=.*['\"]\(.*\)['\"][\s,]*/\1/p" setup.py) fi mkdir .ci/ansible/vars || true echo "---" > .ci/ansible/vars/main.yaml @@ -53,9 +54,6 @@ echo "component_version: '${COMPONENT_VERSION}'" >> .ci/ansible/vars/main.yaml export PRE_BEFORE_INSTALL=$PWD/.github/workflows/scripts/pre_before_install.sh export POST_BEFORE_INSTALL=$PWD/.github/workflows/scripts/post_before_install.sh -COMMIT_MSG=$(git log --format=%B --no-merges -1) -export COMMIT_MSG - if [ -f $PRE_BEFORE_INSTALL ]; then source $PRE_BEFORE_INSTALL fi diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index b3ad3b7b9..ab575d573 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -69,11 +69,45 @@ VARSYAML fi cat >> vars/main.yaml << VARSYAML -pulp_settings: null +pulp_settings: {"orphan_protection_time": 0} +pulp_scheme: https + +pulp_container_tag: https + VARSYAML +if [ "$TEST" = "s3" ]; then + export MINIO_ACCESS_KEY=AKIAIT2Z5TDYPX3ARJBA + export MINIO_SECRET_KEY=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 + sed -i -e '/^services:/a \ + - name: minio\ + image: minio/minio\ + env:\ + MINIO_ACCESS_KEY: "'$MINIO_ACCESS_KEY'"\ + MINIO_SECRET_KEY: "'$MINIO_SECRET_KEY'"\ + command: "server /data"' vars/main.yaml + sed -i -e '$a s3_test: true\ +minio_access_key: "'$MINIO_ACCESS_KEY'"\ +minio_secret_key: "'$MINIO_SECRET_KEY'"' vars/main.yaml +fi + ansible-playbook build_container.yaml ansible-playbook start_container.yaml +echo ::group::SSL +# Copy pulp CA +sudo docker cp pulp:/etc/pulp/certs/pulp_webserver.crt /usr/local/share/ca-certificates/pulp_webserver.crt + +# Hack: adding pulp CA to certifi.where() +CERTIFI=$(python -c 'import certifi; print(certifi.where())') +cat /usr/local/share/ca-certificates/pulp_webserver.crt | sudo tee -a $CERTIFI + +# Hack: adding pulp CA to default CA file +CERT=$(python -c 'import ssl; print(ssl.get_default_verify_paths().openssl_cafile)') +cat $CERTIFI | sudo tee -a $CERT + +# Updating certs +sudo update-ca-certificates +echo ::endgroup:: echo ::group::PIP_LIST cmd_prefix bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" diff --git a/.github/workflows/scripts/install_python_client.sh b/.github/workflows/scripts/install_python_client.sh index ecad00fe3..1c42e2ea9 100755 --- a/.github/workflows/scripts/install_python_client.sh +++ b/.github/workflows/scripts/install_python_client.sh @@ -9,14 +9,14 @@ set -euv -export PULP_URL="${PULP_URL:-http://pulp}" +export PULP_URL="${PULP_URL:-https://pulp}" # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. pip install twine wheel -export REPORTED_VERSION=$(http pulp/pulp/api/v3/status/ | jq --arg plugin python --arg legacy_plugin pulp_python -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version') +export REPORTED_VERSION=$(http $PULP_URL/pulp/api/v3/status/ | jq --arg plugin python --arg legacy_plugin pulp_python -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version') export DESCRIPTION="$(git describe --all --exact-match `git rev-parse HEAD`)" if [[ $DESCRIPTION == 'tags/'$REPORTED_VERSION ]]; then export VERSION=${REPORTED_VERSION} @@ -37,10 +37,10 @@ then fi cd ../pulp-openapi-generator - +rm -rf pulp_python-client ./generate.sh pulp_python python $VERSION cd pulp_python-client python setup.py sdist bdist_wheel --python-tag py3 -pip install dist/pulp_python_client-$VERSION-py3-none-any.whl +find . -name "*.whl" -exec pip install {} \; tar cvf ../../pulp_python/python-client.tar ./dist exit $? diff --git a/.github/workflows/scripts/install_ruby_client.sh b/.github/workflows/scripts/install_ruby_client.sh index 4e459a07c..1dd84089a 100755 --- a/.github/workflows/scripts/install_ruby_client.sh +++ b/.github/workflows/scripts/install_ruby_client.sh @@ -12,9 +12,9 @@ set -euv # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. -export PULP_URL="${PULP_URL:-http://pulp}" +export PULP_URL="${PULP_URL:-https://pulp}" -export REPORTED_VERSION=$(http pulp/pulp/api/v3/status/ | jq --arg plugin python --arg legacy_plugin pulp_python -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version') +export REPORTED_VERSION=$(http $PULP_URL/pulp/api/v3/status/ | jq --arg plugin python --arg legacy_plugin pulp_python -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version') export DESCRIPTION="$(git describe --all --exact-match `git rev-parse HEAD`)" if [[ $DESCRIPTION == 'tags/'$REPORTED_VERSION ]]; then export VERSION=${REPORTED_VERSION} @@ -35,7 +35,7 @@ then fi cd ../pulp-openapi-generator - +rm -rf pulp_python-client ./generate.sh pulp_python ruby $VERSION cd pulp_python-client gem build pulp_python_client diff --git a/.github/workflows/scripts/post_docs_test.sh b/.github/workflows/scripts/post_docs_test.sh index 39c9256bf..0e97e3108 100644 --- a/.github/workflows/scripts/post_docs_test.sh +++ b/.github/workflows/scripts/post_docs_test.sh @@ -1,7 +1,7 @@ #!/usr/bin/env sh -export BASE_ADDR=http://pulp:80 -export CONTENT_ADDR=http://pulp:80 +export BASE_ADDR=https://pulp +export CONTENT_ADDR=https://pulp cd docs/_scripts/ bash ./quickstart.sh diff --git a/.github/workflows/scripts/publish_plugin_pypi.sh b/.github/workflows/scripts/publish_plugin_pypi.sh index 29c2a8c50..981f77073 100755 --- a/.github/workflows/scripts/publish_plugin_pypi.sh +++ b/.github/workflows/scripts/publish_plugin_pypi.sh @@ -12,8 +12,6 @@ cd "$(dirname "$(realpath -e "$0")")"/../../.. set -euv -export PULP_URL="${PULP_URL:-http://pulp}" - export response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/pulp-python/$1/) if [ "$response" == "200" ]; then diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index f26e7e8d3..d798cf13c 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -26,7 +26,7 @@ export FUNC_TEST_SCRIPT=$PWD/.github/workflows/scripts/func_test_script.sh export DJANGO_SETTINGS_MODULE=pulpcore.app.settings export PULP_SETTINGS=$PWD/.ci/ansible/settings/settings.py -export PULP_URL="http://pulp" +export PULP_URL="https://pulp" if [[ "$TEST" = "docs" ]]; then cd docs @@ -46,7 +46,7 @@ if [[ "$TEST" = "docs" ]]; then fi if [[ "${RELEASE_WORKFLOW:-false}" == "true" ]]; then - REPORTED_VERSION=$(http pulp/pulp/api/v3/status/ | jq --arg plugin python --arg legacy_plugin pulp_python -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version') + REPORTED_VERSION=$(http $PULP_URL/pulp/api/v3/status/ | jq --arg plugin python --arg legacy_plugin pulp_python -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version') response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/pulp-python/$REPORTED_VERSION/) if [ "$response" == "200" ]; then diff --git a/.github/workflows/scripts/stage-changelog-for-master.py b/.github/workflows/scripts/stage-changelog-for-master.py new file mode 100755 index 000000000..649f70578 --- /dev/null +++ b/.github/workflows/scripts/stage-changelog-for-master.py @@ -0,0 +1,64 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +import argparse +import os +import textwrap + +from git import Repo +from git.exc import GitCommandError + + +helper = textwrap.dedent( + """\ + Stage the changelog for a release on master branch. + + Example: + $ python .github/workflows/scripts/stage-changelog-for-master.py 3.4.0 + + """ +) + +parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, description=helper) + +parser.add_argument( + "release_version", + type=str, + help="The version string for the release.", +) + +args = parser.parse_args() + +release_version_arg = args.release_version + +release_path = os.path.dirname(os.path.abspath(__file__)) +plugin_path = release_path.split("/.github")[0] + +print(f"\n\nRepo path: {plugin_path}") +repo = Repo(plugin_path) + +changelog_commit = None +# Look for a commit with the requested release version +for commit in repo.iter_commits(): + if f"Building changelog for {release_version_arg}\n" in commit.message: + changelog_commit = commit + break + +if not changelog_commit: + raise RuntimeError("Changelog commit for {release_version_arg} was not found.") + +git = repo.git +git.stash() +git.checkout("origin/master") +try: + git.cherry_pick(changelog_commit.hexsha) +except GitCommandError: + git.add("CHANGES/") + # Don't try opening an editor for the commit message + with git.custom_environment(GIT_EDITOR="true"): + git.cherry_pick("--continue") +git.reset("origin/master") diff --git a/.github/workflows/scripts/update_ci.sh b/.github/workflows/scripts/update_ci.sh new file mode 100755 index 000000000..c83c94541 --- /dev/null +++ b/.github/workflows/scripts/update_ci.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +set -eu + +if [ ! -d ../plugin_template ]; then + echo "Checking out plugin_template" + git clone https://github.com/pulp/plugin_template.git ../plugin_template +fi + + +if [ ! -f "template_config.yml" ]; then + echo "No template_config.yml detected." + exit 1 +fi + +pushd ../plugin_template +./plugin-template --github pulp_python +popd + +if [[ `git status --porcelain` ]]; then + git add -A + git commit -m "Update CI files" -m "[noissue]" +else + echo "No updates needed" +fi diff --git a/.github/workflows/update_ci.yml b/.github/workflows/update_ci.yml new file mode 100644 index 000000000..2b432b256 --- /dev/null +++ b/.github/workflows/update_ci.yml @@ -0,0 +1,48 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template +--- +name: CI Update +on: + schedule: + # * is a special character in YAML so you have to quote this string + # runs at 2:30 UTC daily + - cron: '30 2 * * *' + + workflow_dispatch: + +jobs: + update: + runs-on: ubuntu-latest + + strategy: + fail-fast: false + + steps: + - uses: actions/checkout@v2 + + - name: Configure Git with pulpbot name and email + run: | + git config --global user.name 'pulpbot' + git config --global user.email 'pulp-infra@redhat.com' + + - name: Run update + run: | + .github/workflows/scripts/update_ci.sh + + - name: Create Pull Request for CI files + uses: peter-evans/create-pull-request@v3 + with: + token: ${{ secrets.RELEASE_TOKEN }} + committer: pulpbot + author: pulpbot + title: 'Update CI files' + body: '[noissue]' + commit-message: | + Update CI files + + [noissue] + delete-branch: true diff --git a/CHANGES/413.bugfix b/CHANGES/413.bugfix new file mode 100644 index 000000000..f2fbc8158 --- /dev/null +++ b/CHANGES/413.bugfix @@ -0,0 +1 @@ +Fixed improper metadata serving when using publications with S3 storage diff --git a/pulp_python/app/models.py b/pulp_python/app/models.py index 207d8b0c5..0524febeb 100644 --- a/pulp_python/app/models.py +++ b/pulp_python/app/models.py @@ -2,7 +2,10 @@ from aiohttp.web import json_response from django.contrib.postgres.fields import ArrayField, JSONField +from django.core.exceptions import ObjectDoesNotExist from django.db import models +from dynaconf import settings +from yarl import URL from pulpcore.plugin.models import ( Content, @@ -63,6 +66,34 @@ def content_handler(self, path): name = path.parts[1] elif path.match("pypi/*/json"): name = path.parts[1] + elif len(path.parts) and path.parts[0] == "simple": + # Temporary fix for PublishedMetadata not being properly served from remote storage + if settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem": + publication = self.publication or Publication.objects.filter( + repository_version=self.repository.latest_version()).latest("pulp_created") + rel_path = f"{path}/index.html" + try: + ca = publication.published_artifact.select_related( + "content_artifact", + "content_artifact__artifact", + ).get(relative_path=rel_path).content_artifact + except ObjectDoesNotExist: + return None + file = ca.artifact.file + content_disposition = f"attachment;filename={ca.relative_path}" + parameters = { + "ResponseContentDisposition": content_disposition, + "ResponseContentType": "text/html" + } + url = URL(file.storage.url(file.name, parameters=parameters), encoded=True) + # Trick the content app to stream the metadata from the remote storage + remote = PythonRemote(name="Redirect", url=str(url), policy="streamed") + remote.get_remote_artifact_url = lambda x: str(url) + setattr(self, "publication", None) + setattr(self, "repository", None) + setattr(self, "remote", remote) + return None + if name: package_content = PythonPackageContent.objects.filter( pk__in=self.publication.repository_version.content, diff --git a/pulp_python/tests/functional/api/test_consume_content.py b/pulp_python/tests/functional/api/test_consume_content.py index b9346b0aa..3f101648e 100644 --- a/pulp_python/tests/functional/api/test_consume_content.py +++ b/pulp_python/tests/functional/api/test_consume_content.py @@ -1,8 +1,8 @@ # coding=utf-8 """Tests that perform actions over content unit.""" from pulp_smash import cli -from pulp_smash.pulp3.bindings import monitor_task -from pulp_smash.pulp3.utils import delete_orphans, modify_repo +from pulp_smash.pulp3.bindings import delete_orphans, monitor_task +from pulp_smash.pulp3.utils import modify_repo from pulp_python.tests.functional.constants import ( PYTHON_FIXTURE_URL, @@ -75,7 +75,7 @@ def test_workflow_01(self): pub = self._create_publication(repo) distro = self._create_distribution_from_publication(pub) - self.addCleanup(delete_orphans, cfg) + self.addCleanup(delete_orphans) self.check_consume(distro.to_dict()) def test_workflow_02(self): diff --git a/pulp_python/tests/functional/api/test_crud_content_unit.py b/pulp_python/tests/functional/api/test_crud_content_unit.py index 183c98b67..125979d42 100644 --- a/pulp_python/tests/functional/api/test_crud_content_unit.py +++ b/pulp_python/tests/functional/api/test_crud_content_unit.py @@ -1,7 +1,6 @@ # coding=utf-8 """Tests that perform actions over content unit.""" -from pulp_smash.pulp3.bindings import monitor_task, PulpTaskError -from pulp_smash.pulp3.utils import delete_orphans +from pulp_smash.pulp3.bindings import delete_orphans, monitor_task, PulpTaskError from pulp_python.tests.functional.utils import ( gen_artifact, diff --git a/template_config.yml b/template_config.yml index 872cb70ba..88a89d7c9 100644 --- a/template_config.yml +++ b/template_config.yml @@ -1,6 +1,8 @@ # This config represents the latest values used when running the plugin-template. Any settings that # were not present before running plugin-template have been added with their default values. +# generated with plugin_template@2021.04.08-95-gbf7866c + additional_plugins: [] additional_repos: [] black: false @@ -10,6 +12,8 @@ check_manifest: true check_openapi_schema: true check_stray_pulpcore_imports: true cherry_pick_automation: false +ci_trigger: '{pull_request: {branches: [''*'']}}' +core_import_allowed: [] coverage: false deploy_client_to_pypi: true deploy_client_to_rubygems: true @@ -18,7 +22,9 @@ deploy_daily_client_to_rubygems: true deploy_to_pypi: true docker_fixtures: false docs_test: true +flake8: true issue_tracker: github +noissue_marker: '[noissue]' plugin_app_label: python plugin_camel: PulpPython plugin_camel_short: Python @@ -29,22 +35,29 @@ plugin_dash_short: python plugin_default_branch: master plugin_name: pulp_python plugin_snake: pulp_python +post_job_template: null +pre_job_template: null publish_docs_to_pulpprojectdotorg: true -pulp_settings: null +pulp_scheme: https +pulp_settings: + orphan_protection_time: 0 pulpcore_branch: master pulpcore_pip_version_specifier: null pulpprojectdotorg_key_id: null pydocstyle: true pypi_username: pulp +python_version: '3.8' redmine_project: null +release_email: pulp-infra@redhat.com release_user: pulpbot stable_branch: null +sync_ci: true test_bindings: false test_cli: false test_fips_nightly: false test_performance: false test_released_plugin_with_next_pulpcore_release: false -test_s3: false +test_s3: true travis_addtl_services: [] travis_notifications: None update_redmine: false