tests

.ci/container_setup.d/README

@@ -0,0 +1,7 @@
+Files in this directory of the form '<number>-<name>.sh' are executed in alphabetical order.
+They can assume to be provided with the following environmnent variables:
+* PULP_CLI_CONFIG a path to a config file for the ci container
+* CONTAINER_RUNTIME the command for interacting with containers
+* BASE_PATH the directory the 'run_container.sh' script lives in
+
+Also a running container named 'pulp-ephemeral'.

.ci/gen_certs.py

@@ -0,0 +1,62 @@
+# /// script
+# requires-python = ">=3.10"
+# dependencies = [
+#     "trustme>=1.2.1,<1.3.0",
+# ]
+# ///
+
+import argparse
+import os
+import sys
+
+import trustme
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(prog="gen_certs")
+    parser.add_argument(
+        "-d",
+        "--dir",
+        default=os.getcwd(),
+        help="Directory where certificates and keys are written to. Defaults to cwd.",
+    )
+
+    args = parser.parse_args(sys.argv[1:])
+    cert_dir = args.dir
+
+    if not os.path.isdir(cert_dir):
+        raise ValueError(f"--dir={cert_dir} is not a directory")
+
+    key_type = trustme.KeyType["ECDSA"]
+
+    # Generate the CA certificate
+    ca = trustme.CA(key_type=key_type)
+    # Write the certificate the client should trust
+    ca_cert_path = os.path.join(cert_dir, "ca.pem")
+    ca.cert_pem.write_to_path(path=ca_cert_path)
+
+    # Generate the server certificate
+    server_cert = ca.issue_cert("localhost", "127.0.0.1", "::1", key_type=key_type)
+    # Write the certificate and private key the server should use
+    server_key_path = os.path.join(cert_dir, "server.key")
+    server_cert_path = os.path.join(cert_dir, "server.pem")
+    server_cert.private_key_pem.write_to_path(path=server_key_path)
+    with open(server_cert_path, mode="w") as f:
+        f.truncate()
+    for blob in server_cert.cert_chain_pems:
+        blob.write_to_path(path=server_cert_path, append=True)
+
+    # Generate the client certificate
+    client_cert = ca.issue_cert("admin@example.com", common_name="admin", key_type=key_type)
+    # Write the certificate and private key the client should use
+    client_key_path = os.path.join(cert_dir, "client.key")
+    client_cert_path = os.path.join(cert_dir, "client.pem")
+    client_cert.private_key_pem.write_to_path(path=client_key_path)
+    with open(client_cert_path, mode="w") as f:
+        f.truncate()
+    for blob in client_cert.cert_chain_pems:
+        blob.write_to_path(path=client_cert_path, append=True)
+
+
+if __name__ == "__main__":
+    main()

.ci/nginx.conf.j2

@@ -0,0 +1,167 @@
+# Copy from pulp-oci-images.
+# Ideally we can get it upstream again.
+#
+# The "nginx" package on fedora creates this user and group.
+user nginx nginx;
+# Gunicorn docs suggest this value.
+worker_processes 1;
+daemon off;
+events {
+    worker_connections 1024;  # increase if you have lots of clients
+    accept_mutex off;  # set to 'on' if nginx worker_processes > 1
+}
+
+http {
+    include mime.types;
+    # fallback in case we can't determine a type
+    default_type application/octet-stream;
+    sendfile on;
+
+    # If left at the default of 1024, nginx emits a warning about being unable
+    # to build optimal hash types.
+    types_hash_max_size 4096;
+
+    {%- if https | default(false) %}
+    map $ssl_client_s_dn $ssl_client_s_dn_cn {
+        default "";
+        ~CN=(?<CN>[^,]+) $CN;
+    }
+    {%- endif %}
+
+    upstream pulp-content {
+         server 127.0.0.1:24816;
+    }
+
+    upstream pulp-api {
+         server 127.0.0.1:24817;
+    }
+
+    server {
+        # Gunicorn docs suggest the use of the "deferred" directive on Linux.
+        {% if https | default(false) -%}
+        listen 443 default_server deferred ssl;
+
+        ssl_certificate /etc/pulp/certs/pulp_webserver.crt;
+        ssl_certificate_key /etc/pulp/certs/pulp_webserver.key;
+        ssl_session_cache shared:SSL:50m;
+        ssl_session_timeout 1d;
+        ssl_session_tickets off;
+
+        # intermediate configuration
+        ssl_protocols TLSv1.2;
+        ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256';
+        ssl_prefer_server_ciphers on;
+
+        # HSTS (ngx_http_headers_module is required) (15768000 seconds = 6 months)
+        add_header Strict-Transport-Security max-age=15768000;
+
+        # Configure client cert authentication
+        ssl_client_certificate /etc/pulp/certs/ca.pem;
+        ssl_verify_client optional;
+        {%- else -%}
+        listen 80 default_server deferred;
+        {%- endif %}
+        server_name $hostname;
+
+        # The default client_max_body_size is 1m. Clients uploading
+        # files larger than this will need to chunk said files.
+        client_max_body_size 10m;
+
+        # Gunicorn docs suggest this value.
+        keepalive_timeout 5;
+
+        location {{ content_path }} {
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header Host $http_host;
+            # we don't want nginx trying to do something clever with
+            # redirects, we set the Host: header above already.
+            proxy_redirect off;
+            proxy_pass http://pulp-content;
+        }
+
+        location {{ api_root }}api/v3/ {
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header Host $http_host;
+            {%- if https | default(false) %}
+            proxy_set_header Remoteuser $ssl_client_s_dn_cn;
+            {%- endif %}
+            # we don't want nginx trying to do something clever with
+            # redirects, we set the Host: header above already.
+            proxy_redirect off;
+            proxy_pass http://pulp-api;
+            client_max_body_size 0;
+        }
+
+        {%- if domain_enabled | default(false) %}
+        location ~ {{ api_root }}.+/api/v3/ {
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header Host $http_host;
+            # we don't want nginx trying to do something clever with
+            # redirects, we set the Host: header above already.
+            proxy_redirect off;
+            proxy_pass http://pulp-api;
+            client_max_body_size 0;
+        }
+        {%- endif %}
+
+        location /auth/login/ {
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header Host $http_host;
+            # we don't want nginx trying to do something clever with
+            # redirects, we set the Host: header above already.
+            proxy_redirect off;
+            proxy_pass http://pulp-api;
+        }
+
+        include pulp/*.conf;
+
+        location / {
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header Host $http_host;
+            # we don't want nginx trying to do something clever with
+            # redirects, we set the Host: header above already.
+            proxy_redirect off;
+            proxy_pass http://pulp-api;
+            # most pulp static files are served through whitenoise
+            # http://whitenoise.evans.io/en/stable/
+        }
+
+        {%- if https | default(false) %}
+        # ACME http-01 tokens, i.e, for Let's Encrypt
+        location /.well-known/ {
+            try_files $uri $uri/ =404;
+        }
+        {%- endif %}
+        {% if https | default(false) -%}
+        location /oauth2token/ {
+            auth_basic "Tokens, Tokens, Tokens";
+            auth_basic_user_file /etc/pulp/certs/oauth2passwd;
+            if ($request_method !~ POST) {
+                # This still triggers earlier than the auth_basic in the outer block.
+                return 403;
+            }
+            try_files /dev/null @oauth2token;
+        }
+        # Nginx "return" kicks in before basic_auth, so we must use it in a separate block.
+        # https://stackoverflow.com/questions/67975464/why-doesnt-basic-auth-work-with-a-simple-nginx-return-statement
+        location @oauth2token {
+            default_type application/json;
+            charset utf-8;
+
+            return 200 '{"access_token": "DEADBEEF", "token_type": "bearer", "expires_in": 30}';
+        }
+        {%- endif %}
+    }
+    {%- if https | default(false) %}
+    server {
+        listen 80 default_server;
+        server_name _;
+        return 301 https://$host$request_uri;
+    }
+    {%- endif %}
+}

.ci/run_container.sh

@@ -0,0 +1,126 @@
+#!/bin/sh
+
+set -eu
+
+BASEPATH="$(dirname "$(readlink -f "$0")")"
+export BASEPATH
+
+if [ -z "${CONTAINER_RUNTIME:+x}" ]
+then
+  if ls /usr/bin/podman
+  then
+    CONTAINER_RUNTIME=podman
+  else
+    CONTAINER_RUNTIME=docker
+  fi
+fi
+export CONTAINER_RUNTIME
+
+PULP_CLI_TEST_TMPDIR="$(mktemp -d)"
+export PULP_CLI_TEST_TMPDIR
+
+cleanup () {
+  "${CONTAINER_RUNTIME}" stop pulp-ephemeral && true
+  rm -rf "${PULP_CLI_TEST_TMPDIR}"
+}
+
+trap cleanup EXIT
+trap cleanup INT
+
+if [ -z "${KEEP_CONTAINER:+x}" ]
+then
+  RM="yes"
+else
+  RM=""
+fi
+
+IMAGE_TAG="${IMAGE_TAG:-latest}"
+FROM_TAG="${FROM_TAG:-latest}"
+
+if [ "${CONTAINER_FILE:+x}" ]
+then
+  IMAGE_TAG="ephemeral-build"
+  "${CONTAINER_RUNTIME}" build --file "${BASEPATH}/assets/${CONTAINER_FILE}" --build-arg FROM_TAG="${FROM_TAG}" --tag ghcr.io/pulp/pulp:"${IMAGE_TAG}" .
+fi
+
+if [ "$(getenforce)" = "Enforcing" ]; then
+    SELINUX="yes"
+else
+    SELINUX=""
+fi;
+
+mkdir -p "${PULP_CLI_TEST_TMPDIR}/settings/certs"
+cp "${BASEPATH}/settings/settings.py" "${PULP_CLI_TEST_TMPDIR}/settings/settings.py"
+echo "service_acct:$(openssl passwd secret)" > "${PULP_CLI_TEST_TMPDIR}/settings/certs/oauth2passwd"
+
+if [ -z "${PULP_HTTPS:+x}" ]
+then
+  PROTOCOL="http"
+  PORT="80"
+  PULP_CONTENT_ORIGIN="http://localhost:8080/"
+else
+  PROTOCOL="https"
+  PORT="443"
+  PULP_CONTENT_ORIGIN="https://localhost:8080/"
+  python3 "${BASEPATH}/gen_certs.py" -d "${PULP_CLI_TEST_TMPDIR}/settings/certs"
+  export PULP_CA_BUNDLE="${PULP_CLI_TEST_TMPDIR}/settings/certs/ca.pem"
+  ln -fs server.pem "${PULP_CLI_TEST_TMPDIR}/settings/certs/pulp_webserver.crt"
+  ln -fs server.key "${PULP_CLI_TEST_TMPDIR}/settings/certs/pulp_webserver.key"
+fi
+export PULP_CONTENT_ORIGIN
+
+"${CONTAINER_RUNTIME}" \
+  run ${RM:+--rm} \
+  --env S6_KEEP_ENV=1 \
+  ${OAS_VERSION:+--env PULP_SPECTACULAR_SETTINGS__OAS_VERSION="${OAS_VERSION}"} \
+  ${PULP_HTTPS:+--env PULP_HTTPS} \
+  ${PULP_OAUTH2:+--env PULP_OAUTH2} \
+  ${PULP_API_ROOT:+--env PULP_API_ROOT} \
+  ${PULP_DOMAIN_ENABLED:+--env PULP_DOMAIN_ENABLED} \
+  ${PULP_ENABLED_PLUGINS:+--env PULP_ENABLED_PLUGINS} \
+  --env PULP_CONTENT_ORIGIN \
+  --detach \
+  --name "pulp-ephemeral" \
+  --volume "${PULP_CLI_TEST_TMPDIR}/settings:/etc/pulp${SELINUX:+:Z}" \
+  --volume "${BASEPATH}/nginx.conf.j2:/nginx/nginx.conf.j2${SELINUX:+:Z}" \
+  --network bridge \
+  --publish "8080:${PORT}" \
+  "ghcr.io/pulp/pulp:${IMAGE_TAG}"
+
+echo "Wait for pulp to start."
+for counter in $(seq 40 -1 0)
+do
+  if [ "$counter" = "0" ]
+  then
+    echo "FAIL."
+    "${CONTAINER_RUNTIME}" images
+    "${CONTAINER_RUNTIME}" ps -a
+    "${CONTAINER_RUNTIME}" logs "pulp-ephemeral"
+    exit 1
+  fi
+
+  sleep 3
+  if curl --insecure --fail "${PROTOCOL}://localhost:8080${PULP_API_ROOT:-/pulp/}api/v3/status/" > /dev/null 2>&1
+  then
+    echo "SUCCESS."
+    break
+  fi
+  echo "."
+done
+
+# Set admin password
+"${CONTAINER_RUNTIME}" exec "pulp-ephemeral" pulpcore-manager reset-admin-password --password password
+
+# Create pulp config
+PULP_CLI_CONFIG="${PULP_CLI_TEST_TMPDIR}/settings/certs/cli.toml"
+export PULP_CLI_CONFIG
+pulp config create --overwrite --location "${PULP_CLI_CONFIG}" --base-url "${PROTOCOL}://localhost:8080" ${PULP_API_ROOT:+--api-root "${PULP_API_ROOT}"} --username "admin" --password "password"
+# show pulpcore/plugin versions we're using
+pulp --config "${PULP_CLI_CONFIG}" --refresh-api status
+
+if [ -d "${BASEPATH}/container_setup.d/" ]
+then
+  run-parts --exit-on-error --regex '^[0-9]+-[-_[:alnum:]]*\.sh$' "${BASEPATH}/container_setup.d/"
+fi
+
+PULP_LOGGING="${CONTAINER_RUNTIME}" "$@"