Update CI + add upgrade tests

.ci/scripts/validate_commit_message.py

@@ -11,6 +11,7 @@
 from pathlib import Path
 
 
+import os
 from github import Github
 
 
@@ -22,7 +23,7 @@
 
 sha = sys.argv[1]
 message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8")
-g = Github()
+g = Github(os.environ.get("GITHUB_TOKEN"))
 repo = g.get_repo("pulp/pulp_python")
 
 

.github/template_gitref

@@ -1 +1 @@
-2021.04.08-95-gbf7866c
+2021.04.08-97-g74b81ba

.github/workflows/ci.yml

@@ -33,6 +33,7 @@ jobs:
       - name: Check commit message
         if: github.event_name == 'pull_request'
         env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }}
         run: sh .github/workflows/scripts/check_commit.sh
 
@@ -133,5 +134,79 @@ jobs:
           docker exec pulp cat /etc/yum.repos.d/* || true
           docker exec pulp pip3 list
 
+  upgrade:
+    runs-on: ubuntu-latest
+    needs: lint
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - FROM_PULPCORE_BRANCH: "3.11"
+            FROM_PULP_PYTHON_BRANCH: "3.2"
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          # by default, it uses a depth of 1
+          # this fetches all history so that we can read each commit
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v2
+        with:
+          python-version: "3.8"
+
+      - name: Install httpie
+        run: |
+          echo ::group::HTTPIE
+          sudo apt-get update -yq
+          sudo -E apt-get -yq --no-install-suggests --no-install-recommends install httpie
+          echo ::endgroup::
+          echo "TEST=upgrade" >> $GITHUB_ENV
+          echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV
+          echo "FROM_PULP_PYTHON_BRANCH=${{ matrix.FROM_PULP_PYTHON_BRANCH }}" >> $GITHUB_ENV
+          echo "FROM_PULPCORE_BRANCH=${{ matrix.FROM_PULPCORE_BRANCH }}" >> $GITHUB_ENV
+
+      - name: Install python dependencies
+        run: |
+          echo ::group::PYDEPS
+          pip install wheel
+          echo ::endgroup::
+
+      - name: Before Install
+        run: .github/workflows/scripts/before_install.sh
+        shell: bash
+
+      - name: Install
+        run: .github/workflows/scripts/install.sh
+        env:
+          PY_COLORS: '1'
+          ANSIBLE_FORCE_COLOR: '1'
+        shell: bash
+
+      - name: Install Python client
+        run: .github/workflows/scripts/install_python_client.sh
+
+      - name: Before Script
+        run: .github/workflows/scripts/before_script.sh
+
+      - name: Setting secrets
+        run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT"
+        env:
+          SECRETS_CONTEXT: ${{ toJson(secrets) }}
+
+      - name: Upgrade test
+        run: .github/workflows/scripts/script.sh
+
+      - name: After failure
+        if: failure()
+        run: |
+          http --timeout 30 --check-status --pretty format --print hb https://pulp/pulp/api/v3/status/ || true
+          docker images || true
+          docker ps -a || true
+          docker logs pulp || true
+          docker exec pulp ls -latr /etc/yum.repos.d/ || true
+          docker exec pulp cat /etc/yum.repos.d/* || true
+          docker exec pulp pip3 list
+
 
 

.github/workflows/scripts/before_install.sh

@@ -31,6 +31,7 @@ COMMIT_MSG=$(git log --format=%B --no-merges -1)
 export COMMIT_MSG
 
 if [[ "$TEST" == "upgrade" ]]; then
+  pip install -r functest_requirements.txt
   git checkout -b ci_upgrade_test
   cp -R .github /tmp/.github
   cp -R .ci /tmp/.ci
@@ -103,6 +104,21 @@ if [ -n "$PULP_OPENAPI_GENERATOR_PR_NUMBER" ]; then
 fi
 
 
+git clone --depth=1 https://github.com/pulp/pulp-cli.git
+if [ -n "$PULP_CLI_PR_NUMBER" ]; then
+  cd pulp-cli
+  git fetch origin pull/$PULP_CLI_PR_NUMBER/head:$PULP_CLI_PR_NUMBER
+  git checkout $PULP_CLI_PR_NUMBER
+  cd ..
+fi
+
+cd pulp-cli
+pip install -e .
+pulp config create --base-url https://pulp --location tests/cli.toml
+mkdir ~/.config/pulp
+cp tests/cli.toml ~/.config/pulp/cli.toml
+cd ..
+
 
 git clone --depth=1 https://github.com/pulp/pulpcore.git --branch master
 
@@ -115,6 +131,14 @@ cd ..
 
 
 
+if [[ "$TEST" == "upgrade" ]]; then
+  cd pulpcore
+  git checkout -b ci_upgrade_test
+  git fetch --depth=1 origin heads/$FROM_PULPCORE_BRANCH:$FROM_PULPCORE_BRANCH
+  git checkout $FROM_PULPCORE_BRANCH
+  cd ..
+fi
+
 
 # Intall requirements for ansible playbooks
 pip install docker netaddr boto3 ansible

.github/workflows/scripts/check_commit.sh

@@ -19,7 +19,7 @@ pip3 install pygithub
 
 echo ::endgroup::
 
-for sha in $(curl $GITHUB_CONTEXT | jq '.[].sha' | sed 's/"//g')
+for sha in $(curl -H "Authorization: token $GITHUB_TOKEN" $GITHUB_CONTEXT | jq '.[].sha' | sed 's/"//g')
 do
   python3 .ci/scripts/validate_commit_message.py $sha
   VALUE=$?

.github/workflows/scripts/script.sh

@@ -99,6 +99,56 @@ cmd_prefix bash -c "PULP_DATABASES__default__USER=postgres django-admin test --n
 export PYTHONPATH=$REPO_ROOT:$REPO_ROOT/../pulpcore${PYTHONPATH:+:${PYTHONPATH}}
 
 
+if [[ "$TEST" == "upgrade" ]]; then
+  # Handle app label change:
+  sed -i "/require_pulp_plugins(/d" pulp_python/tests/functional/utils.py
+
+  # Running pre upgrade tests:
+  pytest -v -r sx --color=yes --pyargs -capture=no pulp_python.tests.upgrade.pre
+
+  # Checking out ci_upgrade_test branch and upgrading plugins
+  cmd_prefix bash -c "cd pulpcore; git checkout -f ci_upgrade_test; pip install --upgrade --force-reinstall ."
+  cmd_prefix bash -c "cd pulp_python; git checkout -f ci_upgrade_test; pip install ."
+
+  # Migrating
+  cmd_prefix bash -c "django-admin migrate --no-input"
+
+  # Restarting single container services
+  cmd_prefix bash -c "s6-svc -r /var/run/s6/services/pulpcore-api"
+  cmd_prefix bash -c "s6-svc -r /var/run/s6/services/pulpcore-content"
+  cmd_prefix bash -c "s6-svc -d /var/run/s6/services/pulpcore-resource-manager"
+  cmd_prefix bash -c "s6-svc -d /var/run/s6/services/pulpcore-worker@1"
+  cmd_prefix bash -c "s6-svc -d /var/run/s6/services/pulpcore-worker@2"
+  cmd_prefix bash -c "s6-svc -u /var/run/s6/services/new-pulpcore-resource-manager"
+  cmd_prefix bash -c "s6-svc -u /var/run/s6/services/new-pulpcore-worker@1"
+  cmd_prefix bash -c "s6-svc -u /var/run/s6/services/new-pulpcore-worker@2"
+
+  echo "Restarting in 60 seconds"
+  sleep 60
+
+  # CLI commands to display plugin versions and content data
+  pulp status
+  pulp content list
+  CONTENT_LENGTH=$(pulp content list | jq length)
+  if [[ "$CONTENT_LENGTH" == "0" ]]; then
+    echo "Empty content list"
+    exit 1
+  fi
+
+  # Rebuilding bindings
+  cd ../pulp-openapi-generator
+  ./generate.sh pulpcore python
+  pip install ./pulpcore-client
+  ./generate.sh pulp_python python
+  pip install ./pulp_python-client
+  cd $REPO_ROOT
+
+  # Running post upgrade tests
+  git checkout ci_upgrade_test -- pulp_python/tests/
+  pytest -v -r sx --color=yes --pyargs -capture=no pulp_python.tests.upgrade.post
+  exit
+fi
+
 
 if [[ "$TEST" == "performance" ]]; then
   if [[ -z ${PERFORMANCE_TEST+x} ]]; then

CHANGES/408.misc

@@ -0,0 +1 @@
+Adding upgrade tests

docs/_scripts/base.sh

@@ -17,10 +17,13 @@ if [ -z "$(pip freeze | grep pulp-cli)" ]; then
 fi
 
 # Set up CLI config file
-mkdir ~/.config/pulp
-cat > ~/.config/pulp/settings.toml << EOF
+if [ ! -f ~/.config/pulp/settings.toml ]; then
+  echo "Configuring pulp-cli"
+  mkdir -p ~/.config/pulp
+  cat > ~/.config/pulp/cli.toml << EOF
 [cli]
-base_url = "$BASE_ADDR" # common to be localhost
+base_url = "$BASE_ADDR"
 verify_ssl = false
 format = "json"
 EOF
+fi

pulp_python/app/utils.py

@@ -150,6 +150,23 @@ def latest_content_version(content_query, version):
     return latest_content
 
 
+def json_to_dict(data):
+    """
+    Converts a JSON string into a Python dictionary.
+
+    Args:
+        data (string): JSON string
+
+    Returns:
+        dictionary: of JSON string
+
+    """
+    if isinstance(data, dict):
+        return data
+
+    return json.loads(data)
+
+
 def python_content_to_info(content):
     """
     Takes in a PythonPackageContent instance and returns a dictionary of the Info fields
@@ -175,10 +192,10 @@ def python_content_to_info(content):
         "package_url": content.project_url or "",  # These two are usually identical
         "project_url": content.project_url or "",  # They also usually point to PyPI
         "release_url": f"{content.project_url}{content.version}/" if content.project_url else "",
-        "project_urls": json.loads(content.project_urls) or None,
+        "project_urls": json_to_dict(content.project_urls) or None,
         "platform": content.platform or "",
-        "requires_dist": json.loads(content.requires_dist) or None,
-        "classifiers": json.loads(content.classifiers) or None,
+        "requires_dist": json_to_dict(content.requires_dist) or None,
+        "classifiers": json_to_dict(content.classifiers) or None,
         "yanked": False,  # These are no longer used on PyPI, but are still present
         "yanked_reason": None,
     }