diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index fb9ed4a1c9..a446f33940 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -44,4 +44,8 @@ RUN PULP_STATIC_ROOT=/var/lib/operator/static/ PULP_CONTENT_ORIGIN=localhost \ RUN mkdir /var/lib/pulp/.config USER root:root +# Lots of plugins try to use this path, and throw warnings if they cannot access it. +RUN mkdir /.pytest_cache +RUN chown pulp:pulp /.pytest_cache + ENTRYPOINT ["/init"] diff --git a/.ci/ansible/build_container.yaml b/.ci/ansible/build_container.yaml index 0a188cba9b..0ffd21d4fb 100644 --- a/.ci/ansible/build_container.yaml +++ b/.ci/ansible/build_container.yaml @@ -1,6 +1,7 @@ # Ansible playbook to create the pulp service containers image --- -- hosts: "localhost" +- name: "Build CI Container Image" + hosts: "localhost" gather_facts: false vars_files: - "vars/main.yaml" @@ -9,6 +10,7 @@ ansible.builtin.template: src: "Containerfile.j2" dest: "Containerfile" + - name: "Build pulp image" # We build from the ../.. (parent dir of pulpcore git repo) Docker build # "context" so that repos like pulp-smash are accessible to Docker @@ -18,9 +20,20 @@ # 1-off-builds and CI purposes (which has no cache across CI runs.) # Run build.yaml with -e cache=false if your builds are using outdated # layers. - command: "docker build --network host --no-cache={{ not cache | default(true) | bool }} -t {{ image.name }}:{{ image.tag }} -f {{ playbook_dir }}/Containerfile ../../.." + ansible.builtin.command: + argv: + - "docker" + - "build" + - "--network" + - "host" + - "--no-cache={{ not cache | default(true) | bool }}" + - "-t" + - "{{ image.name }}:{{ image.tag }}" + - "-f" + - "{{ playbook_dir }}/Containerfile" + - "../../.." - name: "Clean image cache" - docker_prune: - images : true + community.docker.docker_prune: + images: true ... diff --git a/.ci/ansible/settings.py.j2 b/.ci/ansible/settings.py.j2 index dfe2851de4..ab8ebb3dce 100644 --- a/.ci/ansible/settings.py.j2 +++ b/.ci/ansible/settings.py.j2 @@ -10,10 +10,6 @@ REDIS_HOST = "localhost" REDIS_PORT = 6379 ANALYTICS = False -{% if api_root is defined %} -API_ROOT = {{ api_root | repr }} -{% endif %} - {% if pulp_settings %} {% for key, value in pulp_settings.items() %} {{ key | upper }} = {{ value | repr }} diff --git a/.ci/ansible/start_container.yaml b/.ci/ansible/start_container.yaml index acdc22ad32..e0891b7ab5 100644 --- a/.ci/ansible/start_container.yaml +++ b/.ci/ansible/start_container.yaml @@ -1,6 +1,7 @@ # Ansible playbook to start the pulp service container and its supporting services --- -- hosts: "localhost" +- name: "Start CI Containers" + hosts: "localhost" gather_facts: false vars_files: - "vars/main.yaml" @@ -14,16 +15,16 @@ - "settings" - name: "Generate Pulp Settings" - template: + ansible.builtin.template: src: "settings.py.j2" dest: "settings/settings.py" - name: "Setup docker networking" - docker_network: + community.docker.docker_network: name: "pulp_ci_bridge" - name: "Start Service Containers" - docker_container: + community.docker.docker_container: name: "{{ item.name }}" image: "{{ item.image }}" auto_remove: true @@ -39,12 +40,12 @@ loop: "{{ services | default([]) }}" - name: "Retrieve Docker Network Info" - docker_network_info: + community.docker.docker_network_info: name: "pulp_ci_bridge" register: "pulp_ci_bridge_info" - name: "Update /etc/hosts" - lineinfile: + ansible.builtin.lineinfile: path: "/etc/hosts" regexp: "\\s{{ item.value.Name }}\\s*$" line: "{{ item.value.IPv4Address | ansible.utils.ipaddr('address') }}\t{{ item.value.Name }}" @@ -55,16 +56,27 @@ amazon.aws.s3_bucket: aws_access_key: "{{ minio_access_key }}" aws_secret_key: "{{ minio_secret_key }}" - s3_url: "http://minio:9000" + endpoint_url: "http://minio:9000" region: "eu-central-1" name: "pulp3" state: "present" when: "s3_test | default(false)" - - block: + - name: "Wait on Services" + block: + - name: "Wait for azurite" + ansible.builtin.uri: + url: "http://ci-azurite:10000/" + status_code: + - 200 + - 400 + when: "azure_test | default(false)" + retries: 2 + delay: 5 + - name: "Wait for Pulp" - uri: - url: "http://pulp{{ lookup('env', 'PULP_API_ROOT') | default('\/pulp\/', True) }}api/v3/status/" + ansible.builtin.uri: + url: "http://pulp{{ pulp_scenario_settings.api_root | default(pulp_settings.api_root | default('\/pulp\/', True), True) }}api/v3/status/" follow_redirects: "all" validate_certs: "no" register: "result" @@ -73,11 +85,12 @@ delay: 5 rescue: - name: "Output pulp container log" - command: "docker logs pulp" + ansible.builtin.command: + cmd: "docker logs pulp" failed_when: true - name: "Check version of component being tested" - assert: + ansible.builtin.assert: that: - "(result.json.versions | items2dict(key_name='component', value_name='version'))[item.app_label] | canonical_semver == (component_version | canonical_semver)" fail_msg: | @@ -86,14 +99,15 @@ loop: "{{ 'plugins' | ansible.builtin.extract(lookup('ansible.builtin.file', '../../template_config.yml') | from_yaml) }}" - name: "Set pulp password in .netrc" - copy: + ansible.builtin.copy: dest: "~/.netrc" content: | machine pulp login admin password password -- hosts: "pulp" +- name: "Prepare Pulp Application Container" + hosts: "pulp" gather_facts: false tasks: - name: "Create directory for pulp-smash config" @@ -108,6 +122,6 @@ dest: "/var/lib/pulp/.config/pulp_smash/settings.json" - name: "Set pulp admin password" - command: + ansible.builtin.command: cmd: "pulpcore-manager reset-admin-password --password password" ... diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index a0d257207a..5f224fd4fb 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -29,7 +29,7 @@ if [ "$TEST" = "s3" ]; then COMPONENT_SOURCE="${COMPONENT_SOURCE}[s3] git+https://github.com/gerrod3/botocore.git@fix-100-continue" fi if [ "$TEST" = "azure" ]; then - COMPONENT_SOURCE="${COMPONENT_SOURCE}[azure]" + COMPONENT_SOURCE="${COMPONENT_SOURCE}[azure,uvloop]" fi if [[ "$TEST" = "pulp" ]]; then @@ -39,10 +39,6 @@ if [[ "$TEST" = "lowerbounds" ]]; then python3 .ci/scripts/calc_constraints.py requirements.txt > lowerbounds_constraints.txt fi -export PULP_API_ROOT=$(test "${TEST}" = "s3" && echo "/rerouted/djnd/" || echo "/pulp/") - -echo "PULP_API_ROOT=${PULP_API_ROOT}" >> "$GITHUB_ENV" - # Compose the scenario definition. mkdir -p .ci/ansible/vars @@ -54,9 +50,8 @@ legacy_component_name: "pulpcore" component_name: "core" component_version: "${COMPONENT_VERSION}" pulp_env: {"PULP_CA_BUNDLE": "/etc/pulp/certs/pulp_webserver.crt"} -pulp_settings: {"allowed_export_paths": ["/tmp"], "allowed_import_paths": ["/tmp"], "orphan_protection_time": 0} +pulp_settings: {"allowed_export_paths": ["/tmp"], "allowed_import_paths": ["/tmp"], "api_root": "/pulp/", "orphan_protection_time": 0} pulp_scheme: "https" -api_root: "${PULP_API_ROOT}" image: name: "pulp" tag: "ci_build" @@ -97,7 +92,7 @@ if [ "$TEST" = "s3" ]; then s3_test: true minio_access_key: "${MINIO_ACCESS_KEY}" minio_secret_key: "${MINIO_SECRET_KEY}" -pulp_scenario_settings: {"AWS_ACCESS_KEY_ID": "AKIAIT2Z5TDYPX3ARJBA", "AWS_DEFAULT_ACL": "@none None", "AWS_S3_ADDRESSING_STYLE": "path", "AWS_S3_ENDPOINT_URL": "http://minio:9000", "AWS_S3_REGION_NAME": "eu-central-1", "AWS_S3_SIGNATURE_VERSION": "s3v4", "AWS_SECRET_ACCESS_KEY": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "AWS_STORAGE_BUCKET_NAME": "pulp3", "DEFAULT_FILE_STORAGE": "storages.backends.s3boto3.S3Boto3Storage", "MEDIA_ROOT": "", "domain_enabled": true, "hide_guarded_distributions": true} +pulp_scenario_settings: {"AWS_ACCESS_KEY_ID": "AKIAIT2Z5TDYPX3ARJBA", "AWS_DEFAULT_ACL": "@none None", "AWS_S3_ADDRESSING_STYLE": "path", "AWS_S3_ENDPOINT_URL": "http://minio:9000", "AWS_S3_REGION_NAME": "eu-central-1", "AWS_S3_SIGNATURE_VERSION": "s3v4", "AWS_SECRET_ACCESS_KEY": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "AWS_STORAGE_BUCKET_NAME": "pulp3", "DEFAULT_FILE_STORAGE": "storages.backends.s3boto3.S3Boto3Storage", "MEDIA_ROOT": "", "api_root": "/rerouted/djnd/", "domain_enabled": true, "hide_guarded_distributions": true} pulp_scenario_env: {} VARSYAML fi diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index feb76f2d75..8395faaad6 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -15,7 +15,7 @@ REPO_ROOT="$PWD" source .github/workflows/scripts/utils.sh -PIP_REQUIREMENTS=("pulp-cli") +PIP_REQUIREMENTS=("pulp-cli" "yq") # This must be the **only** call to "pip install" on the test runner. pip install ${PIP_REQUIREMENTS[*]} @@ -23,7 +23,7 @@ pip install ${PIP_REQUIREMENTS[*]} if [[ "$TEST" = "s3" ]]; then for i in {1..3} do - ansible-galaxy collection install "amazon.aws:8.1.0" && s=0 && break || s=$? && sleep 3 + ansible-galaxy collection install "amazon.aws:11.1.0" && s=0 && break || s=$? && sleep 3 done if [[ $s -gt 0 ]] then @@ -36,18 +36,15 @@ fi PULP_CLI_VERSION="$(pip freeze | sed -n -e 's/pulp-cli==//p')" git clone --depth 1 --branch "$PULP_CLI_VERSION" https://github.com/pulp/pulp-cli.git ../pulp-cli -cd .ci/ansible/ +PULP_API_ROOT="$(yq -r '.pulp_scenario_settings.api_root // .pulp_settings.api_root // "/pulp/"' < .ci/ansible/vars/main.yaml)" pulp config create --base-url https://pulp --api-root "${PULP_API_ROOT}" --username "admin" --password "password" -cp ~/.config/pulp/cli.toml "${REPO_ROOT}/../pulp-cli/tests/cli.toml" +cp ~/.config/pulp/cli.toml "../pulp-cli/tests/cli.toml" + +cd .ci/ansible/ ansible-playbook build_container.yaml ansible-playbook start_container.yaml - -# Plugins often write to ~/.config/pulp/cli.toml from the host -chmod 777 ~/.config/pulp -chmod 666 ~/.config/pulp/cli.toml -sudo chown -R 700:700 ~/.config echo ::group::SSL # Copy pulp CA sudo docker cp pulp:/etc/pulp/certs/pulp_webserver.crt /usr/local/share/ca-certificates/pulp_webserver.crt @@ -78,7 +75,3 @@ if [[ " s3 " =~ " ${TEST} " ]]; then cmd_prefix bash -c "s6-rc -d change redis" echo "The Redis service was disabled for $TEST" fi - -# Lots of plugins try to use this path, and throw warnings if they cannot access it. -cmd_prefix mkdir /.pytest_cache -cmd_prefix chown pulp:pulp /.pytest_cache diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index c808ae76fd..e325c7ba35 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -143,6 +143,8 @@ else fi fi export PULP_FIXTURES_URL="http://pulp-fixtures:8080" +# some pulp-cli tests use the api root envvar +export PULP_API_ROOT="$(EDITOR=cat pulp config edit 2>/dev/null | awk -F'"' '/api_root/{print $2; exit}')" pushd ../pulp-cli pip install -r test_requirements.txt pytest -v tests -m "pulpcore" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 40f5d17e7e..d5e9633557 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -134,7 +134,7 @@ jobs: if: always() run: | echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" - http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true + pulp status || true docker images || true docker ps -a || true docker logs pulp || true diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 0000000000..de0b82e7a3 --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,10 @@ +[allowlist] + description = "Allow specific test-keys." + paths = [ + ] + regexes = [ + '''AKIAIT2Z5TDYPX3ARJBA''', + '''qR\+vjWPU50fCqQuUWbj9Fain/j2pV\+ZtBCiDiieS''', + '''fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS''', + '''Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw''', + ] diff --git a/MANIFEST.in b/MANIFEST.in index 2004bd201b..727a78e2f8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -14,3 +14,5 @@ include pulpcore/app/templates/rest_framework/api.html include manage.py include test_requirements.txt exclude releasing.md + +exclude .gitleaks.toml \ No newline at end of file diff --git a/template_config.yml b/template_config.yml index f585cbb5f9..b2bd17e4ec 100644 --- a/template_config.yml +++ b/template_config.yml @@ -6,7 +6,6 @@ # After editing this file please always reapply the plugin template before committing any changes. --- -api_root: "/pulp/" black: true check_commit_message: true check_gettext: true @@ -50,6 +49,7 @@ pulp_settings: - "/tmp" allowed_import_paths: - "/tmp" + api_root: "/pulp/" orphan_protection_time: 0 pulp_settings_azure: AZURE_ACCOUNT_KEY: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" @@ -74,6 +74,7 @@ pulp_settings_s3: AWS_STORAGE_BUCKET_NAME: "pulp3" DEFAULT_FILE_STORAGE: "storages.backends.s3boto3.S3Boto3Storage" MEDIA_ROOT: "" + api_root: "/rerouted/djnd/" domain_enabled: true hide_guarded_distributions: true pydocstyle: true