From 6f319a5ba5bac53524aa5513971459d93171618b Mon Sep 17 00:00:00 2001 From: Vitaly Korolev Date: Thu, 26 Feb 2026 09:31:23 -0800 Subject: [PATCH 1/2] Update the build pipeline to handle ARM builds. Adding new Docker image types: ubi9-arm and ubi9-rootless-arm. Restructure the pipeline to run tests on external ARM agent. --- .github/copilot-instructions.md | 229 ++++++++ .github/workflows/pr-workflow.yaml | 16 + Jenkinsfile | 532 +++++++++++++++--- LICENSE.txt | 2 +- Makefile | 62 +- NOTICE.txt | 20 +- README.md | 55 +- docker-compose/marklogic-multi-node.yaml | 1 + docker-compose/marklogic-single-node.yaml | 1 + dockerFiles/marklogic-deps-ubi9-arm:base | 29 + dockerFiles/marklogic-deps-ubi9:base | 10 +- dockerFiles/marklogic-deps-ubi:base | 21 +- .../marklogic-server-ubi-rootless:base | 21 +- dockerFiles/marklogic-server-ubi9-arm:base | 151 +++++ dockerFiles/marklogic-server-ubi:base | 10 +- src/scripts/start-marklogic-rootless.sh | 111 ++-- src/scripts/start-marklogic.sh | 121 ++-- test/README.md | 10 +- test/compose-test-1.yaml | 1 + test/compose-test-10.yaml | 1 + test/compose-test-11.yaml | 1 + test/compose-test-12.yaml | 1 + test/compose-test-13.yaml | 1 + test/compose-test-14.yaml | 1 + test/compose-test-15.yaml | 1 + test/compose-test-16.yaml | 315 +++++++++++ test/compose-test-2.yaml | 1 + test/compose-test-3.yaml | 1 + test/compose-test-4.yaml | 1 + test/compose-test-5.yaml | 1 + test/compose-test-6.yaml | 1 + test/compose-test-7.yaml | 1 + test/compose-test-8.yaml | 1 + test/compose-test-9.yaml | 1 + test/docker-tests.robot | 312 ++++++++-- test/keywords.resource | 350 +++++++++++- 36 files changed, 2128 insertions(+), 266 deletions(-) create mode 100644 .github/copilot-instructions.md create mode 100644 .github/workflows/pr-workflow.yaml create mode 100644 dockerFiles/marklogic-deps-ubi9-arm:base create mode 100644 dockerFiles/marklogic-server-ubi9-arm:base create mode 100644 test/compose-test-16.yaml diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 00000000..fd67aabd --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,229 @@ +# MarkLogic Docker Build System (Copilot + Contributor Guidance) + +This file is optimized for day-to-day contributor workflow and for GitHub Copilot context. +If you are using Copilot/AI to modify this repo, follow the **How Copilot Should Work** rules first. + +## How Copilot Should Work + +- Prefer the `Makefile` targets over ad-hoc commands (build/test/lint/scan). +- Keep changes minimal and aligned with existing patterns; avoid refactors unless requested. +- Treat these as **sources of truth**: + - Build behavior: `Makefile` and `dockerFiles/*` + - Runtime behavior: `src/scripts/start-marklogic*.sh` + - Test expectations: `test/docker-tests.robot`, `test/structure-test.yaml`, `test/keywords.resource` + - User documentation: `README.md`, `docker-compose/*.yaml` +- When changing behavior (env vars, logs, endpoints, defaults), update the relevant tests and docs in the same PR. +- Do not introduce new build systems, new base images, or extra “nice-to-have” tooling without an explicit request. +- Security rules: + - Never print secrets/credentials to logs. + - Prefer Docker secrets (`/run/secrets/*`) over env vars for credentials. + - Avoid adding packages unless required; vulnerability surface is tightly managed. + +## Change Checklist (Common Work) + +- **Startup scripts (`src/scripts/*.sh`)** + - Preserve existing log phrasing unless you also update Robot tests that match logs. + - Preserve the root vs rootless behavioral differences (sudo usage, config write mode, converter install). +- **Dockerfile templates (`dockerFiles/*`)** + - Keep the multi-stage + flattened final stage pattern (`COPY --from=builder / /`). + - Keep ownership/permissions correct for rootless (`marklogic_user:users`, UID 1000). + - If you add/remove files, update `test/structure-test.yaml` accordingly. +- **Env vars / secrets** + - Keep naming consistent across `README.md`, `docker-compose/*.yaml`, Dockerfiles, and tests. + - Canonical secret targets: `mldb_admin_username`, `mldb_admin_password`, `mldb_wallet_password`. +- **Tests** + - Add/adjust Robot assertions when behavior or logs change. + - Use the `long_running` tag for slow/integration-heavy tests. + +## Local Development Notes + +- Use `make lint` to run ShellCheck + Hadolint. +- Use `make test` for `structure-test` + Robot tests. +- This repo builds images for `linux/amd64` by default. +- macOS note: `make structure-test` uses GNU-style `sed -i` (GNU sed syntax); on macOS you may need GNU sed (`gsed`) or run the build/test in a Linux container/VM. + +## Project Overview + +This repository builds and maintains Docker images for **MarkLogic Server**, a multi-model NoSQL database. The project supports multiple base images (UBI8/UBI9) with both root and rootless variants, includes security hardening via OpenSCAP, and supports FIPS-enabled configurations. + +**Key directories:** +- `dockerFiles/` - Dockerfile templates for different image variants +- `src/scripts/` - Container initialization scripts (`start-marklogic.sh`, `start-marklogic-rootless.sh`) +- `test/` - Robot Framework test suite for container validation +- `docker-compose/` - Example cluster configurations + +## Build Architecture + +### Multi-Stage Build Process + +Images are built in two stages to reduce final image size: +1. **Builder stage**: Installs MarkLogic RPM, creates system user, adds TINI init system +2. **Final stage**: Copies from builder, flattens layers, removes unnecessary packages + +**Image variants** (controlled by `docker_image_type` parameter): +- `ubi` / `ubi9` - Root images on RedHat Universal Base Image +- `ubi-rootless` / `ubi9-rootless` - Hardened rootless images (user `marklogic_user:1000`) + +### Build Commands + +Use the `Makefile` for all build operations: + +```bash +# Build image (specify RPM package and image type) +make build docker_image_type=ubi9-rootless package=MarkLogic-11.3.nightly-rhel9.x86_64.rpm dockerTag=my-tag + +# Run structure tests +make structure-test docker_image_type=ubi9-rootless dockerTag=my-tag + +# Run Robot Framework tests +make docker-tests docker_image_type=ubi9-rootless dockerTag=my-tag + +# Run specific tests only +make docker-tests DOCKER_TEST_LIST="Smoke Test,Initialized MarkLogic container" + +# Security scanning with Grype +make scan docker_image_type=ubi9-rootless dockerTag=my-tag + +# SCAP hardening validation +make scap-scan docker_image_type=ubi9-rootless dockerTag=my-tag + +# Lint Dockerfiles and shell scripts +make lint +``` + +**Important:** Rootless images automatically apply OpenSCAP CIS hardening scripts during build. The build downloads `scap-security-guide-${open_scap_version}.zip` and extracts the appropriate remediation script for the OS version. + +## Container Initialization Logic + +The entrypoint scripts (`start-marklogic.sh` for root, `start-marklogic-rootless.sh` for rootless) handle: + +1. **Configuration management**: Writes environment variables to `/etc/marklogic.conf` +2. **Credential extraction**: Reads admin credentials from Docker secrets or env vars +3. **Server initialization**: Calls MarkLogic REST APIs to initialize security database +4. **Cluster joining**: Uses bootstrap host to join existing clusters (HTTP or HTTPS) +5. **Health checks**: Polls `/7997/LATEST/healthcheck` endpoint until ready + +### Key Environment Variables + +| Variable | Purpose | Notes | +|----------|---------|-------| +| `MARKLOGIC_INIT` | Initialize server with admin credentials | Must be `true` for automated setup | +| `MARKLOGIC_JOIN_CLUSTER` | Join existing cluster via bootstrap host | Requires `MARKLOGIC_BOOTSTRAP_HOST` | +| `MARKLOGIC_BOOTSTRAP_HOST` | Hostname of cluster bootstrap node | Defaults to `bootstrap` | +| `MARKLOGIC_JOIN_TLS_ENABLED` | Use HTTPS for cluster join | Requires `MARKLOGIC_JOIN_CACERT_FILE` secret | +| `OVERWRITE_ML_CONF` | Rewrite `/etc/marklogic.conf` | Always `true` for rootless images | +| `INSTALL_CONVERTERS` | Install MarkLogic Converters package | Uses `/converters.rpm` | + +**Secrets precedence**: Docker secrets (files in `/run/secrets/`) are preferred over environment variables for credentials. + +## Testing Strategy + +### Robot Framework Tests (`test/docker-tests.robot`) + +Tests use Robot Framework with Docker and HTTP libraries. Each test case creates containers, validates behavior, and tears down. + +**Test execution patterns:** +- Tests tagged `long_running` are excluded by default (use `DOCKER_TEST_LIST` to include) +- All tests create uniquely named containers based on test case name (spaces removed) +- Verification uses Docker logs pattern matching and HTTP endpoint checks + +**Common test patterns:** +```robotframework +Create container with -e MARKLOGIC_INIT=true -e MARKLOGIC_ADMIN_USERNAME=admin +Docker log should contain *MARKLOGIC_INIT is true, initializing* +Verify response for authenticated request with 8001 *No license key* +[Teardown] Delete container +``` + +### Structure Tests + +Container Structure Tests validate: +- File existence and permissions +- Metadata labels (version, build branch) +- Command availability +- Environment variables + +Template: `test/structure-test.yaml` (placeholders replaced during `make structure-test`) + +## Clustering Patterns + +**Bootstrap node** initialization: +```yaml +environment: + - MARKLOGIC_INIT=true + - MARKLOGIC_ADMIN_USERNAME_FILE=mldb_admin_username +``` + +**Additional nodes** joining cluster: +```yaml +environment: + - MARKLOGIC_INIT=true + - MARKLOGIC_ADMIN_USERNAME_FILE=mldb_admin_username + - MARKLOGIC_JOIN_CLUSTER=true + - MARKLOGIC_BOOTSTRAP_HOST=bootstrap_3n + - MARKLOGIC_GROUP=dnode # Optional: join specific group +``` + +**TLS-enabled joining** (requires CA certificate as secret): +```yaml +environment: + - MARKLOGIC_JOIN_TLS_ENABLED=true + - MARKLOGIC_JOIN_CACERT_FILE=certificate.cer +secrets: + - source: certificate.cer + target: certificate.cer +``` + +## Critical Implementation Details + +### Rootless vs Root Differences + +| Aspect | Root Image | Rootless Image | +|--------|-----------|----------------| +| User | `marklogic_user` (UID 1000) | Same | +| PID file | `/var/run/MarkLogic.pid` | `/home/marklogic_user/MarkLogic.pid` | +| Config overwrite | Controlled by `OVERWRITE_ML_CONF` | Always appends to config | +| Hardening | None | OpenSCAP CIS remediation applied | +| Privileges | Requires `sudo` for service start | Uses `start-marklogic.sh` directly | + +### Startup Script Retry Logic + +- `N_RETRY=15` attempts with `RETRY_INTERVAL=10` seconds for critical operations +- `CURL_TIMEOUT=300` seconds for individual HTTP requests +- **Non-idempotent endpoint**: `/admin/v1/instance-admin` called exactly once (no retries) +- `restart_check()` function polls `/admin/v1/timestamp` to detect server restarts + +### Dockerfile Conventions + +- Base images always use `ARG BASE_IMAGE` with defaults +- Multi-stage builds flatten layers using `COPY --from=builder / /` +- Security hardening: Removes packages with known vulnerabilities in final stage +- TINI init system (`/tini`) serves as PID 1 to handle zombie processes +- Volume mounted at `/var/opt/MarkLogic` for persistent data + +## Common Pitfalls + +1. **Rejoining clusters**: Nodes that previously left a cluster may fail to rejoin (known limitation) +2. **Leave button**: Admin UI "leave" may not work; use Management API instead +3. **Timezone**: Containers default to UTC unless `TZ` environment variable is set +4. **HugePages**: Container allocates up to 3/8 of memory limit as HugePages by default (override with `ML_HUGEPAGES_TOTAL`) +5. **Upgrade process**: Must update file ownership to `1000:100` (`marklogic_user:users`) when upgrading to rootless images + +## CI/CD Pipeline (Jenkinsfile) + +The pipeline supports: +- Pull request validation (draft checks, review state validation) +- Scheduled vulnerability scans (emails to security team) +- Multi-architecture builds (currently `linux/amd64` only) +- Jira ticket extraction from branch names (pattern: `MLE-\d{3,6}`) +- Image publishing to Artifactory and Azure Container Registry + +**Pipeline stages:** Checkout → Lint → Build → Structure Test → Docker Tests → Scan → Publish + +## Contributing Notes + +- PRs are used for inspiration but not merged directly (see `CONTRIBUTING.md`) +- Always create an issue before starting significant work +- Tests must be added/updated for new features +- Linting must pass: `hadolint` for Dockerfiles, `shellcheck` for scripts +- Security scan reports reviewed before merging diff --git a/.github/workflows/pr-workflow.yaml b/.github/workflows/pr-workflow.yaml new file mode 100644 index 00000000..f2a31ab9 --- /dev/null +++ b/.github/workflows/pr-workflow.yaml @@ -0,0 +1,16 @@ +name: 🏷️ JIRA ID Validator + +on: + # Using pull_request_target instead of pull_request to handle PRs from forks + pull_request_target: + types: [opened, edited, reopened, synchronize] + # No branch filtering - will run on all PRs + +jobs: + jira-pr-check: + name: 🏷️ Validate JIRA ticket ID + # Use the reusable workflow from the central repository + uses: marklogic/pr-workflows/.github/workflows/jira-id-check.yml@main + with: + # Pass the PR title from the event context + pr-title: ${{ github.event.pull_request.title }} diff --git a/Jenkinsfile b/Jenkinsfile index d539cf51..8cc7ca86 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,5 +1,7 @@ -/* groovylint-disable CompileStatic, LineLength, VariableTypeRequired */ +// Copyright © 2018-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. // This Jenkinsfile defines internal MarkLogic build pipeline. +// The pipeline builds, tests, scans, and optionally publishes MarkLogic Docker images. +// It can be triggered manually, by pull requests, or on a schedule. //Shared library definitions: https://github.com/marklogic/MarkLogic-Build-Libs/tree/1.0-declarative/vars @Library('shared-libraries@1.0-declarative') @@ -8,21 +10,67 @@ import groovy.json.JsonSlurperClassic // email list for scheduled builds (includes security vulnerability) emailList = 'vitaly.korolev@progress.com, Barkha.Choithani@progress.com, Sumanth.Ravipati@progress.com, Peng.Zhou@progress.com, romain.winieski@progress.com' // email list for security vulnerabilities only -emailSecList = 'Rangan.Doreswamy@progress.com, Mahalakshmi.Srinivasan@progress.com' +emailSecList = 'Mahalakshmi.Srinivasan@progress.com' gitCredID = 'marklogic-builder-github' dockerRegistry = 'ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com' +pdcSbRegistry = 'sandboxpdc.azurecr.io' +pdcDevRegistry = 'marklogicclouddev.azurecr.io' JIRA_ID_PATTERN = /(?i)(MLE)-\d{3,6}/ JIRA_ID = '' LINT_OUTPUT = '' SCAN_OUTPUT = '' IMAGE_SIZE = 0 RPMversion = '' +GRAVITON3_IMAGE_ARCHIVE = 'marklogic-image.tar' +builtImage = '' +publishImage = '' +latestTag = '' +upgradeDockerImage = '' // Define local funtions + +/** + * Determines if the current build is for an ARM image type. + * ARM workers (e.g., Graviton3) are only available for MarkLogic 11. + * @return true if dockerImageType contains 'arm', false otherwise. + */ +@NonCPS +def isArmImage() { + return params.dockerImageType.toLowerCase().contains('arm') +} + +/** + * Performs pre-build checks: + * - Initializes parameters as environment variables. + * - Extracts Jira ID from branch name or PR title. + * - Checks if the PR is a draft or has requested changes (for PR builds). + * - Validates ARM image types are only used with MarkLogic 11. + */ void preBuildCheck() { - // Initialize parameters as env variables as workaround for https://issues.jenkins-ci.org/browse/JENKINS-41929 + // Initialize parameters as env variables (workaround for https://issues.jenkins-ci.org/browse/JENKINS-41929) evaluate """${ def script = ''; params.each { k, v -> script += "env.${k} = '''${v}'''\n" }; return script}""" + // Validate ARM images are only supported for MarkLogic 11 + if (env.dockerImageType.contains('arm') && env.marklogicVersion != '11') { + error "ARM images (${env.dockerImageType}) are only supported for MarkLogic 11. Current version: ${env.marklogicVersion}" + } + + // If GRAVITON3_IP is not provided, try to retrieve it from Jenkins credentials + if (!params.GRAVITON3_IP) { + withCredentials([sshUserPrivateKey(credentialsId: 'KUBE_NINJAS_AWS_JENKINS', keyFileVariable: 'SSH_KEY_FILE', usernameVariable: 'GRAVITON_IP_FROM_SECRET')]) { + env.GRAVITON3_IP = env.GRAVITON_IP_FROM_SECRET + echo "GRAVITON3_IP retrieved from Jenkins credentials: ${env.GRAVITON3_IP}" + } + } else { + env.GRAVITON3_IP = params.GRAVITON3_IP + echo "Using provided GRAVITON3_IP parameter: ${env.GRAVITON3_IP}" + } + + // Validate that ARM builds have GRAVITON3_IP available + if (env.dockerImageType.contains('arm') && !env.GRAVITON3_IP) { + error "ARM image type ${env.dockerImageType} requires GRAVITON3_IP parameter or KUBE_NINJAS_AWS_JENKINS credentials to be configured. Please provide the Graviton3 instance IP or hostname." + } + JIRA_ID = extractJiraID() echo 'Jira ticket number: ' + JIRA_ID @@ -42,6 +90,10 @@ void preBuildCheck() { } } +/** + * Extracts a Jira ID (e.g., MLE-1234) from the PR title or branch name. + * @return The extracted Jira ID string, or an empty string if not found. + */ @NonCPS def extractJiraID() { // Extract Jira ID from one of the environment variables @@ -67,6 +119,11 @@ def extractJiraID() { } } +/** + * Checks if the current Pull Request is marked as a draft via GitHub API. + * Requires CHANGE_ID and GIT_URL environment variables. + * @return true if the PR is a draft, false otherwise. + */ def prDraftCheck() { withCredentials([usernameColonPassword(credentialsId: gitCredID, variable: 'Credentials')]) { PrObj = sh(returnStdout: true, script:''' @@ -77,6 +134,11 @@ def prDraftCheck() { return jsonObj.draft } +/** + * Gets the review state of the current Pull Request via GitHub API. + * Requires CHANGE_ID and GIT_URL environment variables. + * @return The review state string (e.g., 'APPROVED', 'CHANGES_REQUESTED'). + */ def getReviewState() { def reviewResponse def commitHash @@ -95,7 +157,12 @@ def getReviewState() { return reviewState } -void resultNotification(message) { +/** + * Sends an email notification with the build status and summary. + * Includes links to build URL, test reports, scan reports, and Jira ticket (if found). + * @param status The build status string (e.g., 'Success', 'Failure'). + */ +void resultNotification(status) { def author, authorEmail, emailList if (env.CHANGE_AUTHOR) { author = env.CHANGE_AUTHOR.toString().trim().toLowerCase() @@ -130,7 +197,7 @@ void resultNotification(message) { // If Jira ID is available, add comment to the ticket and add link to email. if (JIRA_ID) { def jira_link = "https://progresssoftware.atlassian.net/browse/${JIRA_ID}" - def comment = [ body: "Jenkins pipeline build result: ${message}" ] + def comment = [ body: "Jenkins pipeline build result: ${status}" ] jiraAddComment site: 'JIRA', input: comment, idOrKey: JIRA_ID, @@ -139,11 +206,20 @@ void resultNotification(message) { } mail to: "${emailList}", body: "${email_body}", - subject: "${message}: ${env.JOB_NAME} #${env.BUILD_NUMBER}", + subject: "🥷 ${status}: ${env.JOB_NAME} #${env.BUILD_NUMBER}", charset: 'UTF-8', mimeType: 'text/html' } +/** + * Copies the MarkLogic Server and Converters RPMs. + * Determines the correct RPM version/branch based on marklogicVersion parameter. + * Downloads nightly builds from Artifactory unless specific RPM URLs are provided via ML_RPM/ML_CONVERTERS parameters. + * Sets RPM, CONVERTERS, and marklogicVersion global variables. + */ void copyRPMs() { + // Determine architecture suffix based on image type + def archSuffix = dockerImageType.contains('arm') ? 'aarch64' : 'x86_64' + if (marklogicVersion == "10") { RPMsuffix = "-nightly" RPMbranch = "b10" @@ -160,54 +236,78 @@ void copyRPMs() { RPMversion = "11.3" } else if (marklogicVersion == "12") { - //if dockerImageType contains "ubi9" then use nightly-rhel9 suffix - if (dockerImageType.contains("ubi9")) { - RPMsuffix = ".nightly-rhel9" - } else { - RPMsuffix = ".nightly-rhel" - } + RPMsuffix = ".nightly-rhel" RPMbranch = "b12" - RPMversion = "12.0" + RPMversion = "12.1" } else { error "Invalid value in marklogicVersion parameter." } sh """ cd src + ARM_DATE=\$(TZ=America/Los_Angeles date +%Y%m%d) if [ -z ${env.ML_RPM} ]; then - wget --no-verbose https://bed-artifactory.bedford.progress.com:443/artifactory/ml-rpm-tierpoint/${RPMbranch}/server/MarkLogic-${RPMversion}${RPMsuffix}.x86_64.rpm + if [ "${archSuffix}" = "aarch64" ]; then + wget --no-verbose https://bed-artifactory.bedford.progress.com:443/artifactory/ml-rpm-dev-tierpoint/${RPMbranch}/server-arm/MarkLogic-${RPMversion}.\${ARM_DATE}-rhel9.aarch64.rpm + else + wget --no-verbose https://bed-artifactory.bedford.progress.com:443/artifactory/ml-rpm-tierpoint/${RPMbranch}/server/MarkLogic-${RPMversion}${RPMsuffix}.${archSuffix}.rpm + fi else - wget --no-verbose ${ML_RPM} + wget --no-verbose ${ML_RPM} fi - if [ -z ${env.ML_CONVERTERS}]; then - wget --no-verbose https://bed-artifactory.bedford.progress.com:443/artifactory/ml-rpm-tierpoint/${RPMbranch}/converters/MarkLogicConverters-${RPMversion}${RPMsuffix}.x86_64.rpm + if [ -z ${env.ML_CONVERTERS} ]; then + if [ "${archSuffix}" = "aarch64" ]; then + # ARM converters package not yet available in Artifactory - converters can be installed at runtime via INSTALL_CONVERTERS env var + # For now, create a placeholder to allow build to proceed + touch MarkLogicConverters-placeholder.rpm + else + wget --no-verbose https://bed-artifactory.bedford.progress.com:443/artifactory/ml-rpm-tierpoint/${RPMbranch}/converters/MarkLogicConverters-${RPMversion}${RPMsuffix}.${archSuffix}.rpm + fi else - wget --no-verbose ${ML_CONVERTERS} + wget --no-verbose ${ML_CONVERTERS} fi """ script { - // Get the RPM and Converters file names - RPM = sh(returnStdout: true, script: 'cd src;file MarkLogic-*.rpm | cut -d: -f1').trim() - CONVERTERS = sh(returnStdout: true, script: 'cd src;file MarkLogicConverters-*.rpm | cut -d: -f1').trim() - // Extract MarkLogic version from RPM file name - marklogicVersion = sh(returnStdout: true, script: "echo ${RPM}| awk -F \"MarkLogic-\" '{print \$2;}' | awk -F \".x86_64.rpm\" '{print \$1;}' | awk -F \"-rhel\" '{print \$1;}' ").trim() + // Get the RPM and Converters file names for the correct architecture (archSuffix already defined above) + RPM = sh(returnStdout: true, script: "cd src; ls -1 MarkLogic-*.${archSuffix}.rpm 2>/dev/null | head -1").trim() + CONVERTERS = sh(returnStdout: true, script: "cd src; ls -1 MarkLogicConverters-*.${archSuffix}.rpm 2>/dev/null || ls -1 MarkLogicConverters-*.rpm 2>/dev/null | head -1").trim() + // Extract MarkLogic version from RPM file name (handle both x86_64 and aarch64) + marklogicVersion = sh(returnStdout: true, script: "echo ${RPM} | awk -F 'MarkLogic-' '{print \$2;}' | awk -F '.x86_64.rpm' '{print \$1;}' | awk -F '.aarch64.rpm' '{print \$1;}' | awk -F '-rhel' '{print \$1;}'").trim() } } +/** + * Builds the Docker image using the 'make build' target. + * Sets various image tag variables (builtImage, publishImage, latestTag). + * Updates the Jenkins build display name. + */ void buildDockerImage() { builtImage="marklogic/marklogic-server-${dockerImageType}:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion}" publishImage="marklogic/marklogic-server-${dockerImageType}:${marklogicVersion}-${env.dockerImageType}" mlVerShort=marklogicVersion.split("\\.")[0] latestTag="marklogic/marklogic-server-${dockerImageType}:latest-${mlVerShort}" + // Use Los Angeles time (same as ARM_DATE in copyRPMs) to ensure consistency across UTC/PST boundaries + timeStamp = sh(returnStdout: true, script: "TZ=America/Los_Angeles date +%Y%m%d").trim() + timestamptedTag = builtImage.replace('nightly', timeStamp) sh "make build docker_image_type=${dockerImageType} dockerTag=${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} marklogicVersion=${marklogicVersion} dockerVersion=${env.dockerVersion} build_branch=${env.BRANCH_NAME} package=${RPM} converters=${CONVERTERS}" currentBuild.displayName = "#${BUILD_NUMBER}: ${marklogicVersion}-${env.dockerImageType} (${env.dockerVersion})" + echo "Built image: ${builtImage}" } +/** + * Pulls the Docker image required for upgrade testing. + * Uses the 'upgradeDockerImage' parameter or defaults to a corresponding 'ubi' image. + * Skips the pull if the target image is 'ubi-rootless' and DOCKER_TESTS is false. + */ void pullUpgradeDockerImage() { if (dockerImageType == "ubi-rootless" && params.DOCKER_TESTS != "true") { sh """ echo 'dockerImageType is set to ubi-rootless, skipping this stage and Docker upgrade test.' """ + } else if (isArmImage()) { + sh """ + echo 'ARM image type detected. Skipping upgrade test (no previous ARM images available for upgrade testing).' + """ } else { if (upgradeDockerImage != "" ) { sh """ @@ -224,18 +324,35 @@ void pullUpgradeDockerImage() { } } +/** + * Runs container structure tests using the 'make structure-test' target. + */ void structureTests() { sh """ - #install container-structure-test 1.16.0 binary - curl -s -LO https://storage.googleapis.com/container-structure-test/v1.16.0/container-structure-test-linux-amd64 && chmod +x container-structure-test-linux-amd64 && mv container-structure-test-linux-amd64 container-structure-test + #install container-structure-test 1.16.0 binary (detect architecture) + ARCH=\$(uname -m) + if [ "\$ARCH" = "aarch64" ]; then + PLATFORM="arm64" + else + PLATFORM="amd64" + fi + curl -s -LO https://storage.googleapis.com/container-structure-test/v1.16.0/container-structure-test-linux-\${PLATFORM} && chmod +x container-structure-test-linux-\${PLATFORM} && mv container-structure-test-linux-\${PLATFORM} container-structure-test make structure-test current_image=marklogic/marklogic-server-${dockerImageType}:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} marklogicVersion=${marklogicVersion} dockerVersion=${env.dockerVersion} build_branch=${env.BRANCH_NAME} docker_image_type=${env.dockerImageType} Jenkins=true """ } +/** + * Runs Docker functional tests using the 'make docker-tests' target. + */ void dockerTests() { - sh "make docker-tests current_image=marklogic/marklogic-server-${dockerImageType}:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} upgrade_image=${upgradeDockerImage} marklogicVersion=${marklogicVersion} build_branch=${env.BRANCH_NAME} dockerVersion=${env.dockerVersion} docker_image_type=${dockerImageType}" + sh "make docker-tests current_image=marklogic/marklogic-server-${dockerImageType}:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} upgrade_image=${upgradeDockerImage} marklogicVersion=${marklogicVersion} build_branch=${env.BRANCH_NAME} dockerVersion=${env.dockerVersion} docker_image_type=${dockerImageType} DOCKER_TEST_LIST=\"${params.DOCKER_TEST_LIST}\"" } +/** + * Lints the Dockerfile and startup scripts using hadolint and shellcheck via 'make lint'. + * Captures the lint output in the LINT_OUTPUT variable. + * Records the built image size in the IMAGE_SIZE variable. + */ void lint() { IMAGE_SIZE = sh(returnStdout: true, script: "docker images marklogic/marklogic-server-${dockerImageType}:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} --format '{{.Repository}}:{{.Tag}}\t{{.Size}}'") @@ -251,6 +368,12 @@ void lint() { """ } +/** + * Scans the built Docker image for vulnerabilities using Grype via 'make scan'. + * Captures the scan summary in the SCAN_OUTPUT variable. + * Archives the full JSON scan report. + * Sends an email notification if critical or high severity vulnerabilities are found. + */ void vulnerabilityScan() { sh """ make scan current_image=marklogic/marklogic-server-${dockerImageType}:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} docker_image_type=${dockerImageType} Jenkins=true @@ -263,41 +386,85 @@ void vulnerabilityScan() { archiveArtifacts artifacts: 'scan/*', onlyIfSuccessful: true } +/** + * Publishes the built Docker image to the internal Artifactory registry. + * Also publishes ML12 images to private Azure ACR repositories (PDC). + * Tags the image with multiple tags (version-specific, branch-specific, latest). + * Requires Artifactory and Azure ACR credentials. + */ void publishToInternalRegistry() { + // Use the discovered image tag if available (handles date/time mismatches across day boundaries) + def imageToPublish = env.IMAGE_TO_PUBLISH ?: builtImage + echo "Publishing image: ${imageToPublish}" + withCredentials([usernamePassword(credentialsId: 'builder-credentials-artifactory', passwordVariable: 'docker_password', usernameVariable: 'docker_user')]) { sh """ docker logout ${dockerRegistry} echo "${docker_password}" | docker login --username ${docker_user} --password-stdin ${dockerRegistry} - docker tag ${builtImage} ${dockerRegistry}/${builtImage} - docker tag ${builtImage} ${dockerRegistry}/${publishImage} - docker tag ${builtImage} ${dockerRegistry}/${latestTag} - docker push ${dockerRegistry}/${builtImage} + docker tag ${imageToPublish} ${dockerRegistry}/${publishImage} + docker tag ${imageToPublish} ${dockerRegistry}/${latestTag} docker push ${dockerRegistry}/${publishImage} docker push ${dockerRegistry}/${latestTag} """ } // Publish to private ECR repository that is used by the performance team. (only ML11) - if ( params.marklogicVersion == "11" ) { - withCredentials( [[ - $class: 'AmazonWebServicesCredentialsBinding', - credentialsId: "aws-engineering-ct-ecr", - accessKeyVariable: 'AWS_ACCESS_KEY_ID', - secretKeyVariable: 'AWS_SECRET_ACCESS_KEY' - ]]) { - sh """ - aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 713759029616.dkr.ecr.us-west-2.amazonaws.com - docker tag ${builtImage} 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} - docker tag ${builtImage} 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${marklogicVersion}-${env.dockerImageType} - docker push 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} - docker push 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${marklogicVersion}-${env.dockerImageType} - """ - } + // (disabled since it's not needed) + // if ( params.marklogicVersion == "11" ) { + // withCredentials( [[ + // $class: 'AmazonWebServicesCredentialsBinding', + // credentialsId: "aws-engineering-ct-ecr", + // accessKeyVariable: 'AWS_ACCESS_KEY_ID', + // secretKeyVariable: 'AWS_SECRET_ACCESS_KEY' + // ]]) { + // sh """ + // aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 713759029616.dkr.ecr.us-west-2.amazonaws.com + // docker tag ${builtImage} 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} + // docker tag ${builtImage} 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${marklogicVersion}-${env.dockerImageType} + // docker push 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} + // docker push 713759029616.dkr.ecr.us-west-2.amazonaws.com/ml-docker-nightly:${marklogicVersion}-${env.dockerImageType} + // """ + // } + // } + + // Publish to private ACR repositories that are used by PDC. (only ML12) + if ( params.marklogicVersion == "12" ) { + // Publish to Sandbox PDC registry + withCredentials([usernamePassword(credentialsId: 'PDC_SANDBOX_USER', passwordVariable: 'docker_password', usernameVariable: 'docker_user')]) { + sh """ + echo "${docker_password}" | docker login --username ${docker_user} --password-stdin ${pdcSbRegistry} + docker tag ${imageToPublish} ${pdcSbRegistry}/ml-docker-nightly:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} + docker tag ${imageToPublish} ${pdcSbRegistry}/ml-docker-nightly:${marklogicVersion}-${env.dockerImageType} + docker push ${pdcSbRegistry}/ml-docker-nightly:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} + docker push ${pdcSbRegistry}/ml-docker-nightly:${marklogicVersion}-${env.dockerImageType} + """ + } + // Publish to Dev PDC registry + withCredentials([usernamePassword(credentialsId: 'pdc-azure-cr', passwordVariable: 'docker_password', usernameVariable: 'docker_user')]) { + sh """ + echo "${docker_password}" | docker login --username ${docker_user} --password-stdin ${pdcDevRegistry} + docker tag ${imageToPublish} ${pdcDevRegistry}/marklogicdb-custom:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} + docker tag ${imageToPublish} ${pdcDevRegistry}/marklogicdb-custom:${marklogicVersion}-${env.dockerImageType} + docker push ${pdcDevRegistry}/marklogicdb-custom:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} + docker push ${pdcDevRegistry}/marklogicdb-custom:${marklogicVersion}-${env.dockerImageType} + """ + } } currentBuild.description = "Published" } +/** + * Triggers a BlackDuck scan job for the published image. + * Runs asynchronously (wait: false). + */ +void scanWithBlackDuck() { + build job: 'securityscans/Blackduck/KubeNinjas/docker', wait: false, parameters: [ string(name: 'BRANCH', value: "${env.BRANCH_NAME}"), string(name: 'CONTAINER_IMAGES', value: "${dockerRegistry}/${publishImage}"), string(name: 'ML_VER', value: "${params.marklogicVersion}"), string(name: 'DOCKER_TYPE', value: "${params.dockerImageType}") ] +} +/** + * Publishes JUnit XML test results from structure and Docker tests. + * Publishes HTML reports for Docker tests and SCAP scans if they were executed. + */ void publishTestResults() { junit allowEmptyResults:true, testResults: '**/test_results/docker-tests.xml,**/container-structure-test.xml' if (params.DOCKER_TESTS) { @@ -321,6 +488,10 @@ void publishTestResults() { } } +/** + * Runs an OpenSCAP compliance scan on the rootless image using 'make scap-scan'. + * Generates an HTML report. + */ void scapScan() { sh """ make scap-scan current_image=marklogic/marklogic-server-${dockerImageType}:${marklogicVersion}-${env.dockerImageType}-${env.dockerVersion} @@ -328,25 +499,36 @@ void scapScan() { } pipeline { - agent { - label { - label 'cld-docker' - } - } + agent none options { checkoutToSubdirectory '.' buildDiscarder logRotator(artifactDaysToKeepStr: '7', artifactNumToKeepStr: '', daysToKeepStr: '30', numToKeepStr: '') skipStagesAfterUnstable() } triggers { - parameterizedCron( env.BRANCH_NAME == 'develop' ? '''00 02 * * * % marklogicVersion=10;dockerImageType=ubi - 00 02 * * * % marklogicVersion=10;dockerImageType=ubi-rootless;SCAP_SCAN=true - 00 02 * * * % marklogicVersion=11;dockerImageType=ubi - 30 02 * * * % marklogicVersion=11;dockerImageType=ubi-rootless;SCAP_SCAN=true - 30 02 * * * % marklogicVersion=12;dockerImageType=ubi - 30 02 * * * % marklogicVersion=12;dockerImageType=ubi-rootless;SCAP_SCAN=true - 00 03 * * * % marklogicVersion=11;dockerImageType=ubi9 - 00 03 * * * % marklogicVersion=11;dockerImageType=ubi9-rootless;SCAP_SCAN=true''' : '') + // Trigger nightly builds on the develop branch for every supported version of MarkLogic + // and for every supported image type. + // Include SCAP scan for rootless images + parameterizedCron( + env.BRANCH_NAME == 'develop' ? ''' + 03 04 * * * % marklogicVersion=11;dockerImageType=ubi9-arm + 00 04 * * * % marklogicVersion=11;dockerImageType=ubi9-arm-rootless;SCAP_SCAN=true + 00 04 * * * % marklogicVersion=10;dockerImageType=ubi + 00 04 * * * % marklogicVersion=10;dockerImageType=ubi-rootless;SCAP_SCAN=true + 00 03 * * * % marklogicVersion=11;dockerImageType=ubi + 00 03 * * * % marklogicVersion=11;dockerImageType=ubi-rootless;SCAP_SCAN=true + 00 03 * * * % marklogicVersion=11;dockerImageType=ubi9 + 00 03 * * * % marklogicVersion=11;dockerImageType=ubi9-rootless;SCAP_SCAN=true + 00 02 * * * % marklogicVersion=12;dockerImageType=ubi + 00 02 * * * % marklogicVersion=12;dockerImageType=ubi-rootless;SCAP_SCAN=true + 00 02 * * * % marklogicVersion=12;dockerImageType=ubi9 + 00 02 * * * % marklogicVersion=12;dockerImageType=ubi9-rootless;SCAP_SCAN=true + 00 05 * * 7 % marklogicVersion=10;dockerImageType=ubi;DOCKER_TEST_LIST=Initialized MarkLogic container with latency + 30 05 * * 7 % marklogicVersion=11;dockerImageType=ubi;DOCKER_TEST_LIST=Initialized MarkLogic container with latency + 00 06 * * 7 % marklogicVersion=12;dockerImageType=ubi;DOCKER_TEST_LIST=Initialized MarkLogic container with latency''' : + env.BRANCH_NAME == 'Docker-ARM-support' ? ''' + 03 04 * * * % marklogicVersion=11;dockerImageType=ubi9-arm + 00 04 * * * % marklogicVersion=11;dockerImageType=ubi9-arm-rootless;SCAP_SCAN=true''' : '') } environment { QA_LICENSE_KEY = credentials('QA_LICENSE_KEY') @@ -354,92 +536,268 @@ pipeline { parameters { string(name: 'emailList', defaultValue: emailList, description: 'List of email for build notification', trim: true) - string(name: 'dockerVersion', defaultValue: '2.1.0', description: 'ML Docker version. This version along with ML rpm package version will be the image tag as {ML_Version}_{dockerVersion}', trim: true) - choice(name: 'dockerImageType', choices: 'ubi-rootless\nubi\nubi9-rootless\nubi9', description: 'Platform type for Docker image. Will be made part of the docker image tag') + string(name: 'dockerVersion', defaultValue: '2.2.3', description: 'ML Docker version. This version along with ML rpm package version will be the image tag as {ML_Version}_{dockerVersion}', trim: true) + choice(name: 'dockerImageType', choices: 'ubi-rootless\nubi\nubi9-rootless\nubi9\nubi9-arm\nubi9-rootless-arm', description: 'Platform type for Docker image. Will be made part of the docker image tag') string(name: 'upgradeDockerImage', defaultValue: '', description: 'Docker image for testing upgrades. Defaults to ubi image if left blank.\n Currently upgrading to ubi-rotless is not supported hence the test is skipped when ubi-rootless image is provided.', trim: true) - choice(name: 'marklogicVersion', choices: '11\n12\n10', description: 'MarkLogic Server Branch. used to pick appropriate rpm') + choice(name: 'marklogicVersion', choices: '12\n11\n10', description: 'MarkLogic Server Branch. used to pick appropriate rpm') string(name: 'ML_RPM', defaultValue: '', description: 'URL for RPM to be used for Image creation. \n If left blank nightly ML rpm will be used.\n Please provide Jenkins accessible path e.g. /project/engineering or /project/qa', trim: true) string(name: 'ML_CONVERTERS', defaultValue: '', description: 'URL for the converters RPM to be included in the image creation \n If left blank the nightly ML Converters Package will be used.', trim: true) booleanParam(name: 'PUBLISH_IMAGE', defaultValue: false, description: 'Publish image to internal registry') booleanParam(name: 'TEST_STRUCTURE', defaultValue: true, description: 'Run container structure tests') booleanParam(name: 'DOCKER_TESTS', defaultValue: true, description: 'Run docker tests') + string(name: 'DOCKER_TEST_LIST', defaultValue: '', description: 'Comma separated list of test names to run (e.g Test one, Test two). Leave empty to run all tests.', trim: true) booleanParam(name: 'SCAP_SCAN', defaultValue: false, description: 'Run Open SCAP scan on the image.') + string(name: 'GRAVITON3_IP', defaultValue: '', description: '[ARM only] Public IP or hostname of Graviton3 instance. Will pull from secrets by default.', trim: true) } stages { + // Stage: Perform initial checks (PR status, Jira ID) stage('Pre-Build-Check') { + agent { node { label 'cld-docker' } } steps { preBuildCheck() } } + // Stage: Download MarkLogic Server and Converters RPMs (ARM builds on x86) stage('Copy-RPMs') { + agent { node { label 'cld-docker' } } steps { copyRPMs() } } + // Stage: Build the Docker image + // Save image archive to workspace and stash for cross-agent stages. stage('Build-Image') { + agent { node { label 'cld-docker' } } steps { buildDockerImage() + script { + // Always save image for cases where agents might differ + sh """ + echo "Saving ${builtImage} to ${WORKSPACE}/${GRAVITON3_IMAGE_ARCHIVE}..." + docker image save ${builtImage} -o ${WORKSPACE}/${GRAVITON3_IMAGE_ARCHIVE} + ls -lh ${WORKSPACE}/${GRAVITON3_IMAGE_ARCHIVE} + """ + stash name: 'built-image-archive', includes: "${GRAVITON3_IMAGE_ARCHIVE}", allowEmpty: false + } } } + // Stage: Pull the base image needed for upgrade testing stage('Pull-Upgrade-Image') { + agent { node { label 'cld-docker' } } steps { pullUpgradeDockerImage() } } + // Stage: Lint Dockerfile and startup scripts (x86 only) stage('Lint') { + agent { node { label 'cld-docker' } } steps { lint() } } + // Stage: Scan the image for vulnerabilities (x86 only) stage('Scan') { + agent { node { label 'cld-docker' } } steps { - vulnerabilityScan() + echo 'Skipping vulnerability scan due to compatibility issues.' + // vulnerabilityScan() } } + // Stage: Run OpenSCAP compliance scan (conditional) stage('SCAP-Scan') { + agent { + node { + label isArmImage() ? 'cld-docker-graviton' : 'cld-docker' + } + } when { + beforeAgent true expression { return params.SCAP_SCAN } } steps { + script { + unstash 'built-image-archive' + // Load image from tar if not already available (applies to all build types) + def imageSource = "${WORKSPACE}/${GRAVITON3_IMAGE_ARCHIVE}" + sh """ + if ! docker image inspect ${builtImage} &>/dev/null; then + echo "Loading image from ${imageSource} for SCAP scan..." + docker image load -i ${imageSource} + else + echo "Image ${builtImage} already available locally" + fi + """ + } scapScan() + stash name: 'scap-results', includes: 'scap/**', allowEmpty: true + } + } + + // Stage: Load image from tar archive (ARM builds only) + stage('Load-Image') { + agent { label 'cld-docker-graviton' } + when { + beforeAgent true + expression { return isArmImage() && env.GRAVITON3_IP } + } + steps { + script { + unstash 'built-image-archive' + sh """ + if ! docker image inspect ${builtImage} &>/dev/null; then + echo "Loading image from ${WORKSPACE}/${GRAVITON3_IMAGE_ARCHIVE}..." + docker image load -i ${WORKSPACE}/${GRAVITON3_IMAGE_ARCHIVE} + else + echo "Image ${builtImage} already loaded (likely by SCAP-Scan stage)" + fi + docker images | head -5 + """ + } } } + // Stage: Run container structure tests (conditional) stage('Structure-Tests') { + agent { + node { + label isArmImage() ? 'cld-docker-graviton' : 'cld-docker' + } + } when { + beforeAgent true expression { return params.TEST_STRUCTURE } } steps { + script { + unstash 'built-image-archive' + // Load image from tar if not already available (applies to all build types) + def imageSource = "${WORKSPACE}/${GRAVITON3_IMAGE_ARCHIVE}" + sh """ + if ! docker image inspect ${builtImage} &>/dev/null; then + echo "Loading image from ${imageSource} for Structure-Tests..." + docker image load -i ${imageSource} + else + echo "Image ${builtImage} already available locally" + fi + """ + } structureTests() + stash name: 'structure-test-results', includes: 'container-structure-test.xml', allowEmpty: true } } + // Stage: Run Docker functional tests (conditional) stage('Docker-Run-Tests') { + agent { + node { + label isArmImage() ? 'cld-docker-graviton' : 'cld-docker' + } + } when { + beforeAgent true expression { return params.DOCKER_TESTS } } steps { + script { + unstash 'built-image-archive' + // Load image from tar if not already available (applies to all build types) + def imageSource = "${WORKSPACE}/${GRAVITON3_IMAGE_ARCHIVE}" + sh """ + if ! docker image inspect ${builtImage} &>/dev/null; then + echo "Loading image from ${imageSource} for Docker-Run-Tests..." + docker image load -i ${imageSource} + else + echo "Image ${builtImage} already available locally" + fi + """ + } dockerTests() + stash name: 'docker-test-results', includes: 'test/test_results/**', allowEmpty: true } } + // Stage: Publish image to internal registries (conditional) stage('Publish-Image') { + agent { node { label 'cld-docker' } } when { + beforeAgent true anyOf { branch 'develop' expression { return params.PUBLISH_IMAGE } } } steps { + script { + unstash 'built-image-archive' + // Load image from tar if not already available (applies to all build types) + sh """ + if ! docker image inspect ${builtImage} &>/dev/null; then + echo "Image not found locally, loading from ${WORKSPACE}/${GRAVITON3_IMAGE_ARCHIVE}..." + docker image load -i ${WORKSPACE}/${GRAVITON3_IMAGE_ARCHIVE} + else + echo "Image ${builtImage} already available locally" + fi + """ + + // If builtImage doesn't exist, find the loaded image by repo pattern + def actualImage = sh( + returnStdout: true, + script: """docker images --format 'table {{.Repository}}:{{.Tag}}' | grep "marklogic/marklogic-server-${dockerImageType}:" | head -1""" + ).trim() + + if (!actualImage) { + actualImage = builtImage + echo "Using builtImage tag: ${actualImage}" + } else { + echo "Found loaded image: ${actualImage}" + } + + // Store for use in publishToInternalRegistry + env.IMAGE_TO_PUBLISH = actualImage + } publishToInternalRegistry() - build job: 'MarkLogic-Docker-Kubernetes/docker/docker-nightly-builds-qa', wait: false, parameters: [string(name: 'dockerImageType', value: "${dockerImageType}"), string(name: 'marklogicVersion', value: "${RPMversion}")] + // Trigger downstream QA image build job + build job: 'KubeNinjas/docker/docker-nightly-builds-qa', wait: false, parameters: [string(name: 'dockerImageType', value: "${dockerImageType}"), string(name: 'marklogicVersion', value: "${RPMversion}")] + } + } + + // Stage: Trigger BlackDuck security scan (conditional) + stage('BlackDuck-Scan') { + agent { node { label 'cld-docker' } } + when { + anyOf { + branch 'develop' + expression { return params.PUBLISH_IMAGE } + } + } + steps { + scanWithBlackDuck() + } + } + + // Stage: Cleanup ARM agent (ARM builds only) + stage('Cleanup-ARM') { + agent { label 'cld-docker-graviton' } + when { + beforeAgent true + expression { return isArmImage() && env.GRAVITON3_IP } + } + steps { + sh ''' + echo "Cleaning up ARM agent..." + # Stop all running containers + docker stop $(docker ps -a -q) || true + # Docker cleanup + docker system prune --force --all --volumes + docker system df + ''' } } @@ -447,23 +805,47 @@ pipeline { post { always { - sh ''' - cd src - rm -rf *.rpm NOTICE.txt - docker stop $(docker ps -a -q) || true - docker system prune --force --all --volumes - docker system df - ''' - publishTestResults() + node('cld-docker') { + // Clean up the workspace and Docker resources + sh """ + # Remove any stale test artifacts before unstash + rm -rf test/test_results scap container-structure-test.xml + # Remove ARM image tar archive + rm -f ${WORKSPACE}/${GRAVITON3_IMAGE_ARCHIVE} + # Remove ARM image if it was built + if [ -n "${builtImage}" ]; then + docker rmi ${builtImage} || true + fi + # Clean up RPMs + if [ -d src ]; then + cd src + rm -rf *.rpm NOTICE.txt + cd .. + fi + # Docker cleanup applies to both agents + docker stop \$(docker ps -a -q) || true + docker system prune --force --all --volumes + docker system df + """ + script { + try { unstash 'structure-test-results' } catch (e) { echo 'No structure test results to unstash.' } + try { unstash 'docker-test-results' } catch (e) { echo 'No docker test results to unstash.' } + try { unstash 'scap-results' } catch (e) { echo 'No SCAP results to unstash.' } + } + publishTestResults() + } } success { - resultNotification('BUILD SUCCESS ✅') + resultNotification('✅ Success') } failure { - resultNotification('BUILD ERROR ❌') + resultNotification('❌ Failure') } unstable { - resultNotification('BUILD UNSTABLE ❌') + resultNotification('⚠️ Unstable') } - } + aborted { + resultNotification('🚫 Aborted') + } + } } diff --git a/LICENSE.txt b/LICENSE.txt index 3bb2b516..319849de 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright © 2023 MarkLogic Corporation. +Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at diff --git a/Makefile b/Makefile index 07e260aa..47a4a4cd 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,26 @@ +# Copyright © 2018-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. dockerTag?=internal package?=MarkLogic.rpm repo_dir=marklogic -docker_build_options=--compress --platform linux/amd64 +docker_build_options=--compress build_branch?=local docker_image_type?=ubi upgrade_docker_image_type?=ubi upgrade_image?=${repo_dir}/marklogic-server-${upgrade_docker_image_type}:${dockerTag} current_image?=${repo_dir}/marklogic-server-${docker_image_type}:${dockerTag} -open_scap_version?=0.1.74 +# Latest release tag can be found here: https://github.com/ComplianceAsCode/content/releases +open_scap_version?=0.1.79 + +#*************************************************************************** +# set docker platform based on the docker image type +#*************************************************************************** +ifeq ($(findstring arm,$(docker_image_type)),arm) + docker_build_options += --platform linux/arm64 + export DOCKER_PLATFORM=linux/arm64 +else + docker_build_options += --platform linux/amd64 + export DOCKER_PLATFORM=linux/amd64 +endif #*************************************************************************** # build docker image @@ -16,6 +29,13 @@ build: # NOTICE file need to be in the build context to be included in the built image cp NOTICE.txt src/NOTICE.txt +# Install ARM64 emulation support on Linux (assuming Jenkins environment which is not aarch64) +ifeq ($(findstring arm,$(docker_image_type)),arm) +ifeq ($(shell uname -s),Linux) + docker run --privileged --rm tonistiigi/binfmt --install arm64 +endif +endif + # rootless images use the same dependencies as ubi image so we copy the file ifeq ($(docker_image_type),ubi9) cp dockerFiles/marklogic-server-ubi\:base dockerFiles/marklogic-server-ubi9\:base @@ -25,10 +45,15 @@ ifeq ($(findstring rootless,$(docker_image_type)),rootless) cp dockerFiles/marklogic-deps-ubi9\:base dockerFiles/marklogic-deps-ubi9-rootless\:base cp dockerFiles/marklogic-server-ubi-rootless\:base dockerFiles/marklogic-server-ubi9-rootless\:base endif +# ubi9-rootless-arm needs deps from ubi9-arm and server template from ubi-rootless +ifeq ($(docker_image_type),ubi9-rootless-arm) + cp dockerFiles/marklogic-deps-ubi9-arm\:base dockerFiles/marklogic-deps-ubi9-rootless-arm\:base + cp dockerFiles/marklogic-server-ubi-rootless\:base dockerFiles/marklogic-server-ubi9-rootless-arm\:base +endif # retrieve and copy open scap hardening script ifeq ($(findstring rootless,$(docker_image_type)),rootless) - [ -f scap-security-guide-${open_scap_version}.zip ] || curl -Lo scap-security-guide-${open_scap_version}.zip https://github.com/ComplianceAsCode/content/releases/download/v${open_scap_version}/scap-security-guide-${open_scap_version}.zip + [ -f scap-security-guide-${open_scap_version}.zip ] || curl -Lso scap-security-guide-${open_scap_version}.zip https://github.com/ComplianceAsCode/content/releases/download/v${open_scap_version}/scap-security-guide-${open_scap_version}.zip #UBI9 needs a different version of the remediation script ifeq ($(findstring ubi9,$(docker_image_type)),ubi9) unzip -p scap-security-guide-${open_scap_version}.zip scap-security-guide-${open_scap_version}/bash/rhel9-script-cis.sh > src/rhel-script-cis.sh @@ -39,11 +64,11 @@ endif # build the image - cd src/; docker build ${docker_build_options} -t "${repo_dir}/marklogic-deps-${docker_image_type}:${dockerTag}" -f ../dockerFiles/marklogic-deps-${docker_image_type}:base . + cd src/; docker build ${docker_build_options} -t "${repo_dir}/marklogic-deps-${docker_image_type}:${dockerTag}" --build-arg ML_VERSION=${marklogicVersion} -f ../dockerFiles/marklogic-deps-${docker_image_type}:base . cd src/; docker build ${docker_build_options} -t "${repo_dir}/marklogic-server-${docker_image_type}:${dockerTag}" --build-arg BASE_IMAGE=${repo_dir}/marklogic-deps-${docker_image_type}:${dockerTag} --build-arg ML_RPM=${package} --build-arg ML_USER=marklogic_user --build-arg ML_DOCKER_VERSION=${dockerVersion} --build-arg ML_VERSION=${marklogicVersion} --build-arg ML_CONVERTERS=${converters} --build-arg BUILD_BRANCH=${build_branch} --build-arg ML_DOCKER_TYPE=${docker_image_type} -f ../dockerFiles/marklogic-server-${docker_image_type}:base . # remove temporary files - rm -f dockerFiles/marklogic-deps-ubi-rootless\:base dockerFiles/marklogic-deps-ubi9-rootless\:base dockerFiles/marklogic-server-ubi9-rootless\:base dockerFiles/marklogic-server-ubi9\:base src/NOTICE.txt src/rhel-script-cis.sh + rm -f dockerFiles/marklogic-deps-ubi-rootless\:base dockerFiles/marklogic-deps-ubi9-rootless\:base dockerFiles/marklogic-server-ubi9-rootless\:base dockerFiles/marklogic-server-ubi9\:base dockerFiles/marklogic-deps-ubi9-rootless-arm\:base dockerFiles/marklogic-server-ubi9-rootless-arm\:base src/NOTICE.txt src/rhel-script-cis.sh #*************************************************************************** # strcture test docker images @@ -70,7 +95,20 @@ docker-tests: python3 -m venv python_env; \ source ./python_env/bin/activate; \ pip3 install -r requirements.txt; \ - robot -x docker-tests.xml --outputdir test_results --randomize all --variable TEST_IMAGE:${current_image} --variable UPGRADE_TEST_IMAGE:${upgrade_image} --variable MARKLOGIC_VERSION:${marklogicVersion} --variable BUILD_BRANCH:${build_branch} --variable MARKLOGIC_DOCKER_VERSION:${dockerVersion} --variable IMAGE_TYPE:${docker_image_type} --maxerrorlines 9999 ./docker-tests.robot; \ + TEST_ARGS=""; \ + if [ -n "$(DOCKER_TEST_LIST)" ]; then \ + echo "$(DOCKER_TEST_LIST)" | sed 's/,/\n/g' | while IFS= read -r ITEM; do \ + ITEM=$$(echo "$$ITEM" | sed -e 's/^ *//' -e 's/ *$$//' -e 's/^"//' -e 's/"$$//'); \ + [ -n "$$ITEM" ] && echo "--test \"$$ITEM\""; \ + done | tr '\n' ' ' > /tmp/test_args; \ + TEST_ARGS=$$(cat /tmp/test_args); \ + rm -f /tmp/test_args; \ + echo "Running selected tests: $(DOCKER_TEST_LIST)"; \ + else \ + TEST_ARGS="--exclude long_running"; \ + echo "Running all tests except those tagged with 'long_running'"; \ + fi; \ + eval "robot --consolewidth 120 -x docker-tests.xml --outputdir test_results --randomize all --variable TEST_IMAGE:${current_image} --variable UPGRADE_TEST_IMAGE:${upgrade_image} --variable MARKLOGIC_VERSION:${marklogicVersion} --variable BUILD_BRANCH:${build_branch} --variable MARKLOGIC_DOCKER_VERSION:${dockerVersion} --variable IMAGE_TYPE:${docker_image_type} --maxerrorlines 9999 $$TEST_ARGS ./docker-tests.robot"; \ deactivate; \ rm -rf python_env @@ -99,7 +137,7 @@ lint: .PHONY: scan scan: ifeq ($(Jenkins),true) - docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -v ${PWD}/scan:/scan anchore/grype:latest --output json --file /scan/report-${docker_image_type}.json ${current_image} + docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -v ${PWD}/scan:/scan anchore/grype:latest --output json --file scan/report-${docker_image_type}.json ${current_image} sudo chown -R builder.ml-eng scan echo -e "Grype scan summary\n------------------" > scan/report-${docker_image_type}.txt jq '.matches[].vulnerability.severity' scan/report-${docker_image_type}.json | sort | uniq -c >> scan/report-${docker_image_type}.txt @@ -118,15 +156,21 @@ endif # security scan docker images #*************************************************************************** scap-scan: + # Clean up any existing scap-scan container from previous runs + docker rm -f scap-scan 2>/dev/null || true mkdir -p scap - [ -f scap-security-guide-${open_scap_version}.zip ] || curl -Lo scap-security-guide-${open_scap_version}.zip https://github.com/ComplianceAsCode/content/releases/download/v${open_scap_version}/scap-security-guide-${open_scap_version}.zip + [ -f scap-security-guide-${open_scap_version}.zip ] || curl -Lso scap-security-guide-${open_scap_version}.zip https://github.com/ComplianceAsCode/content/releases/download/v${open_scap_version}/scap-security-guide-${open_scap_version}.zip #UBI9 needs a different version of the evaluation profile ifeq ($(findstring ubi9,$(current_image)),ubi9) unzip -p scap-security-guide-${open_scap_version}.zip scap-security-guide-${open_scap_version}/ssg-rhel9-ds.xml > scap/ssg-rhel-ds.xml else unzip -p scap-security-guide-${open_scap_version}.zip scap-security-guide-${open_scap_version}/ssg-rhel8-ds.xml > scap/ssg-rhel-ds.xml endif - docker run -itd --name scap-scan -v $(PWD)/scap:/scap ${current_image} + docker run -itd --name scap-scan --entrypoint /bin/bash -v $(PWD)/scap:/scap ${current_image} -c "sleep infinity" + # Wait a moment for container to be fully up + sleep 2 + # Verify container is running + docker ps | grep scap-scan || (docker logs scap-scan; exit 1) docker exec -u root scap-scan /bin/bash -c "microdnf update -y; microdnf install -y openscap-scanner" # ensure the file is owned by root in order to avoid permission issues docker exec -u root scap-scan /bin/bash -c "chown root:root /scap/ssg-rhel-ds.xml" diff --git a/NOTICE.txt b/NOTICE.txt index 87f98e1a..ede40c0f 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1,6 +1,6 @@ -MarkLogic® Docker Container Image v2.1.1 +MarkLogic® Docker Container Image v2 -Copyright © 2022-2024 MarkLogic Corporation. MarkLogic and MarkLogic logo are trademarks or registered trademarks of MarkLogic Corporation in the United States and other countries. All other trademarks are the property of their respective owners. +Copyright © 2018-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. This project is licensed under the Apache License, Version 2.0 (the "License"); you may not use this project except in compliance with the License. You may obtain a copy of the License at @@ -12,7 +12,7 @@ To the extent required by the applicable open-source license, a complete machine MarkLogic® - Docker Container Image. If Licensee selects the option to download the Product in the Docker container image, then the Product will be provided in a Docker container image that in a separate layer also contains third-party software, RedHat UBI Docker Base Image 8 and its components and RedHat UBI Docker Base Image 9 and its components (collectively, “RedHat UBI”), licensed under the RedHat EULA found here (https://www.redhat.com/licenses/EULA_Red_Hat_Universal_Base_Image_English_20190422.pdf) and , together with the third-party licenses applicable to each component. -Notwithstanding anything to the contrary in the MarkLogic end user license agreement, MarkLogic makes no representations or warranties and assumes no indemnification obligations in regard to RedHat UBI, its operation, or its security. User modifications to the version of RedHat UBI provided by MarkLogic in the Docker Image, while permitted under The RedHat EULA, may result in errors or instability in performance of the Product, which are not covered by MarkLogic under warranty or maintenance terms. The source code for RedHat UBI Docker Base Image 8 may be obtained at http://iue.progress.com/3dpartysoftwares/Pages/default.aspx, and source code for any of its third-party components covered by a license requiring that source code be made available may be obtained at https://catalog.redhat.com/software/containers/ubi8/5c647760bed8bd28d0e38f9f?architecture=amd64&image=6643ab9ff6bc4ca6c09fb093&container-tabs=packages (for UBI Docker Base Image 8) and https://catalog.redhat.com/software/containers/ubi9/ubi/615bcf606feffc5384e8452e?container-tabs=packages (for UBI Docker Base Image 9). +Notwithstanding anything to the contrary in the Progress end user license agreement, Progress makes no representations or warranties and assumes no indemnification obligations in regard to RedHat UBI, its operation, or its security. User modifications to the version of RedHat UBI provided by Progress in the Docker Image, while permitted under The RedHat EULA, may result in errors or instability in performance of the Product, which are not covered by Progress under warranty or maintenance terms. The source code for RedHat UBI Docker Base Image 8 may be obtained at http://iue.progress.com/3dpartysoftwares/Pages/default.aspx, and source code for any of its third-party components covered by a license requiring that source code be made available may be obtained at https://catalog.redhat.com/software/containers/ubi8/5c647760bed8bd28d0e38f9f?architecture=amd64&image=6643ab9ff6bc4ca6c09fb093&container-tabs=packages (for UBI Docker Base Image 8) and https://catalog.redhat.com/software/containers/ubi9/ubi/615bcf606feffc5384e8452e?container-tabs=packages (for UBI Docker Base Image 9). Licensee is responsible for obtaining, at its own expense, any required licenses from Docker to deploy the Docker container image that contains the Product and RedHat UBI and any such deployment of the Product must comply with the terms and conditions of this Agreement. @@ -21,11 +21,11 @@ Licensee is responsible for obtaining, at its own expense, any required licenses Third Party Components RedHat UBI Docker Base Image 8 (Commercial) -RedHat UBI Docker Base Image 9 (Commercial) -robotframework 7.0 (Apache-2.0) - robotframework-requests 0.9.7 (MIT) - test (MIT) - Tini 0.19.0 (MIT) + RedHat UBI Docker Base Image 9 (Commercial) + robotframework 7.0 (Apache-2.0) + robotframework-requests 0.9.7 (MIT) + test (MIT) + Tini 0.19.0 (MIT) Common Licenses @@ -35,7 +35,7 @@ Apache License, Version 2.0, January 2004 (Apache-2.0) Third-Party Components -The following is a list of third-party components used by MarkLogic Docker Container Image v2.1.0 (last updated October 18, 2024): +The following is a list of third-party components used by MarkLogic Docker Container Image v2 (last updated October 18, 2024): RedHat UBI Docker Base Image 8 https://catalog.redhat.com/software/containers/ubi8/5c647760bed8bd28d0e38f9f?architecture=amd64&image=6643ab9ff6bc4ca6c09fb093&container-tabs=gti @@ -85,6 +85,7 @@ Permission is hereby granted, free of charge, to any person obtaining a copy of The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +------------------------------------------------------------------------- Common License Appendix @@ -139,3 +140,4 @@ You may add Your own copyright statement to Your modifications and may provide a 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS + diff --git a/README.md b/README.md index f7b7396b..fed6fc19 100644 --- a/README.md +++ b/README.md @@ -46,10 +46,21 @@ Docker images are maintained by MarkLogic. Send feedback to the MarkLogic Docker Supported Docker architectures: x86_64 -Base OS: UBI8 and UBI9 with rootless variants. +Base OS: UBI8 and UBI9 with `rootless` variants. Published image artifact details: https://github.com/marklogic/marklogic-docker, https://hub.docker.com/r/progressofficial/marklogic-db +## Docker image hardening + +Docker images with `rootless` variants are hardened using Openscap (). + +Scoring : 96.67% +See [Known Issues and Limitations](#known-issues-and-limitations) + +## FIPS Enabled + +Only Docker images under Base OS UBI8 with `rootless` variants are FIPS enabled following RedHat () + # MarkLogic [MarkLogic Server](http://www.marklogic.com/) is a multi-model database that has both NoSQL and trusted enterprise data management capabilities. It is the most secure multi-model database, and it’s deployable in any environment. @@ -170,8 +181,8 @@ MarkLogic Server Docker containers are configured using a set of environment var | MARKLOGIC_GROUP | dnode | no | n/a | will join the host to the given MarkLogic group | | LICENSE_KEY | license key | no | n/a | set MarkLogic license key | | LICENSEE | licensee information | no | n/a | set MarkLogic licensee information | -|INSTALL_CONVERTERS | true | no | false | Installs converters for the client if they are not already installed | -|OVERWRITE_ML_CONF | true | no | false | Deletes and rewrites `/etc/marklogic.conf` with the passed in env variables if set to true | +| INSTALL_CONVERTERS | true | no | false | Installs converters for the client if they are not already installed | +| OVERWRITE_ML_CONF | true | no | true | Deletes and rewrites `/etc/marklogic.conf` with the passed in env variables if set to true. Always true in rootless image. | Note: MARKLOGIC_JOIN_TLS_ENABLED and MARKLOGIC_JOIN_CACERT_FILE should be used only for nodes joining the cluster. These two parameters will be ignored for bootstrap host configurations. @@ -853,6 +864,14 @@ The /space mounted on the Docker volume can now be used as backup directory for # Debugging +## Platform warnings on Apple Silicon + +When running the MarkLogic Docker image on Apple Silicon, you may see the following warning message: +`WARNING: The requested image's platform (linux/amd64) does not match the detected host platform (linux/arm64/v8) and no specific platform was requested` + +Add the `--platform linux/amd64` flag to the `docker run` command to avoid this warning message. + + ## View MarkLogic Server Startup Status To check the MarkLogic Server startup status, run the below command to tail the MarkLogic log file ``` @@ -901,7 +920,7 @@ $ docker exec -it f484a784d998 /bin/bash 4. To verify that MarkLogic is running, use this command: ``` -$ sudo service MarkLogic status +$ service MarkLogic status ``` Example output: @@ -915,7 +934,7 @@ MarkLogic (pid 34) is running... For example, you can list the 8001 error logs, and view them with a single command: ``` -$ sudo cd /var/opt/MarkLogic/Logs && ls && vi ./8001_ErrorLog.txt +$ cd /var/opt/MarkLogic/Logs && ls && cat ./8001_ErrorLog.txt ``` 6. To exit the container when you are through debugging, use the exit command: @@ -1044,13 +1063,31 @@ Where is calculated as described in the [Configuring HugePages](https://github.c # Known Issues and Limitations -1. The image must be run in privileged mode. If the image isn't run as privileged, the calls that use `sudo` in the startup script will fail due to lack of required permissions as the image will not be able to create a user with the required permissions. To run in non-privileged mode, use one of the “rootless” image options. +1. The root image must be run in privileged mode. If the image isn't run as privileged, the calls that use `sudo` in the startup script will fail due to lack of required permissions as the image will not be able to create a user with the required permissions. To run in non-privileged mode, use one of the “rootless” image options. 2. Using the "leave" button in the Admin interface to remove a node from a cluster may not succeed, depending on your network configuration. Use the Management API to remove a node from a cluster. See: [https://docs.marklogic.com/REST/DELETE/admin/v1/host-config](https://docs.marklogic.com/REST/DELETE/admin/v1/host-config). 3. Rejoining a node to a cluster, that had previously left that cluster, may not succeed. 4. MarkLogic Server will default to the UTC timezone. 5. The latest released version of RedHat UBI images have known security vulnerabilities. - - CVE-2024-6602, CVE-2024-34397, CVE-2024-2236, CVE-2023-7207, CVE-2023-51764, CVE-2023-37920, CVE-2023-32636, CVE-2023-29499, CVE-2023-2650, CVE-2022-4899, CVE-2021-42694, CVE-2021-3997, CVE-2020-35512, CVE-2020-15945, CVE-2019-9937, CVE-2019-9936, CVE-2019-9705, CVE-2019-19244, CVE-2019-17543, CVE-2019-12904, CVE-2019-12900, CVE-2018-20839, CVE-2024-6602, CVE-2024-6119, CVE-2024-26462, CVE-2024-2236, CVE-2023-7207, CVE-2023-37920, CVE-2023-2953, CVE-2022-4899, CVE-2021-3997, CVE-2024-10041 - -These libraries are included in the RedHat UBI base images but, to-date, no fixes have been made available. Even though these libraries may be present in the base image that is used by MarkLogic Server, they are not used by MarkLogic Server itself, hence there is no impact or mitigation required. + - curl (CVE-2016-5420, CVE-2016-5419, CVE-2016-5421, CVE-2017-3604, CVE-2016-3418, CVE-2017-3605, CVE-2016-0694, CVE-2017-3607, CVE-2017-3608, CVE-2017-3606, CVE-2016-0689, CVE-2017-3609, CVE-2016-0692, CVE-2016-0682, CVE-2016-5420, CVE-2016-5419, CVE-2016-5421, CVE-2023-28322) + - elfutils (CVE-2017-3610, CVE-2017-3611, CVE-2017-3612, CVE-2017-3613, CVE-2017-3614, CVE-2017-3615) + - gawk (CVE-2017-3616) + - gdb (CVE-2017-3617) + - glib/glibc (CVE-2016-5420, CVE-2016-5421, CVE-2016-5419, CVE-2016-5419, CVE-2016-5420, CVE-2016-5421, CVE-2019-12450, CVE-2020-6096) + - libcap (CVE-2023-2603) + - libdb-utils (CVE-2016-0682, CVE-2016-0689, CVE-2016-0692, CVE-2016-0694, CVE-2016-3418, CVE-2017-3604, CVE-2017-3605, CVE-2017-3606, CVE-2017-3607, CVE-2017-3608, CVE-2017-3609, CVE-2017-3610, CVE-2017-3611, CVE-2017-3612, CVE-2017-3613, CVE-2017-3614, CVE-2017-3615, CVE-2017-3616, CVE-2017-3617, CVE-2015-2583, CVE-2015-2626, CVE-2015-2640, CVE-2015-2654, CVE-2015-2656, CVE-2015-4754, CVE-2015-2624, CVE-2015-4784, CVE-2015-4787, CVE-2015-4789, CVE-2015-4785, CVE-2015-4786, CVE-2015-4783, CVE-2015-4764, CVE-2015-4780, CVE-2015-4790, CVE-2015-4776, CVE-2015-4775, CVE-2015-4778, CVE-2015-4777, CVE-2015-4782, CVE-2015-4781, CVE-2015-4774) + - libcroco (CVE-2017-8871) + - libksba (CVE-2022-3515, CVE-2022-47629) + - libssh (CVE-2023-6004) + - libxml2 (CVE-2022-23308) + - nspr (CVE-2016-1951) + - pam (CVE-2022-28321) + - systemd (CVE-2020-13776) + +These packages are included in the RedHat UBI base images but, to-date, no fixes have been made available. Even though these libraries may be present in the base image that is used by MarkLogic Server, they are not used by MarkLogic Server itself, hence there is no impact or mitigation required. 6. As part of the hardening process, the following packages are removed from the image: `vim-minimal`, `cups-client`, `cups-libs`, `tar`, `python3-pip-wheel`, `platform-python`, `python3-libs`, `platform-python-setuptools`, `avahi-libs`, `binutils`, `expat`, `libarchive`, `python3`, `python3-libs`, `python-unversioned-command`. These packages are not required for the operation of MarkLogic Server and are removed to reduce the attack surface of the image. If you require any of these packages, you can install them in your own Dockerfile. + +7. The scoring of the hardening process is 96.67% that because `authselect is not used but files from the 'pam' package have been altered, so the authselect configuration won't be forced.` + +It is a medium severity and not applicable in container environment there is not authentication required when login into a container. +8. The cryptographic modules of RHEL 9 are not yet certified for the FIPS 140-3 requirements. diff --git a/docker-compose/marklogic-multi-node.yaml b/docker-compose/marklogic-multi-node.yaml index a38571af..4722510c 100644 --- a/docker-compose/marklogic-multi-node.yaml +++ b/docker-compose/marklogic-multi-node.yaml @@ -1,3 +1,4 @@ +# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. #Docker compose file sample to setup a three node cluster version: '3.6' services: diff --git a/docker-compose/marklogic-single-node.yaml b/docker-compose/marklogic-single-node.yaml index 03a711b9..9e5166b0 100644 --- a/docker-compose/marklogic-single-node.yaml +++ b/docker-compose/marklogic-single-node.yaml @@ -1,3 +1,4 @@ +# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. #Docker compose file sample to setup single node cluster version: '3.6' services: diff --git a/dockerFiles/marklogic-deps-ubi9-arm:base b/dockerFiles/marklogic-deps-ubi9-arm:base new file mode 100644 index 00000000..fe93ab01 --- /dev/null +++ b/dockerFiles/marklogic-deps-ubi9-arm:base @@ -0,0 +1,29 @@ +############################################################### +# +# Copyright © 2018-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. +# +############################################################### + +FROM registry.access.redhat.com/ubi9/ubi-minimal:9.7-1771346502 +LABEL "com.marklogic.maintainer"="docker@marklogic.com" + +############################################################### +# install libnsl rpm package +############################################################### + +RUN microdnf -y update \ + && rpm -i https://download.rockylinux.org/pub/rocky/9/BaseOS/aarch64/os/Packages/l/libnsl-2.34-231.el9_7.10.aarch64.rpm + +############################################################### +# install networking, base deps and tzdata for timezone +############################################################### +# hadolint ignore=DL3006 +RUN echo "NETWORKING=yes" > /etc/sysconfig/network \ + && microdnf -y install --setopt install_weak_deps=0 gdb nss libtool-ltdl cpio tzdata util-linux hostname \ + && microdnf clean all + + +############################################################### +# Enable FIPS Mode +############################################################### +RUN update-crypto-policies --set FIPS \ No newline at end of file diff --git a/dockerFiles/marklogic-deps-ubi9:base b/dockerFiles/marklogic-deps-ubi9:base index 56dca487..5fe0de0a 100644 --- a/dockerFiles/marklogic-deps-ubi9:base +++ b/dockerFiles/marklogic-deps-ubi9:base @@ -1,10 +1,10 @@ ############################################################### # -# Copyright (c) 2023 MarkLogic Corporation +# Copyright © 2018-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. # ############################################################### -FROM registry.access.redhat.com/ubi9/ubi-minimal:9.5-1733767867 +FROM registry.access.redhat.com/ubi9/ubi-minimal:9.7-1771346502 LABEL "com.marklogic.maintainer"="docker@marklogic.com" ############################################################### @@ -12,16 +12,14 @@ LABEL "com.marklogic.maintainer"="docker@marklogic.com" ############################################################### RUN microdnf -y update \ - && curl -Lso libnsl.rpm https://bed-artifactory.bedford.progress.com:443/artifactory/ml-rpm-release-tierpoint/devdependencies/libnsl-2.34-125.el9_5.1.x86_64.rpm \ - && rpm -i libnsl.rpm \ - && rm -f libnsl.rpm + && rpm -i https://repo.almalinux.org/almalinux/9/BaseOS/x86_64/os/Packages/libnsl-2.34-231.el9_7.10.x86_64.rpm ############################################################### # install networking, base deps and tzdata for timezone ############################################################### # hadolint ignore=DL3006 RUN echo "NETWORKING=yes" > /etc/sysconfig/network \ - && microdnf -y install --setopt install_weak_deps=0 gdb nss libtool-ltdl cpio tzdata util-linux \ + && microdnf -y install --setopt install_weak_deps=0 gdb nss libtool-ltdl cpio tzdata util-linux hostname \ && microdnf clean all diff --git a/dockerFiles/marklogic-deps-ubi:base b/dockerFiles/marklogic-deps-ubi:base index 27b48476..fe415096 100644 --- a/dockerFiles/marklogic-deps-ubi:base +++ b/dockerFiles/marklogic-deps-ubi:base @@ -1,29 +1,38 @@ ############################################################### # -# Copyright (c) 2023 MarkLogic Corporation +# Copyright © 2018-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. # ############################################################### -FROM registry.access.redhat.com/ubi8/ubi-minimal:8.10-1130 +FROM registry.access.redhat.com/ubi8/ubi-minimal:8.10-1771947229 LABEL "com.marklogic.maintainer"="docker@marklogic.com" +# MarkLogic version passed from build to enable conditional deps +ARG ML_VERSION + ############################################################### # install libnsl rpm package ############################################################### RUN microdnf -y update \ - && curl -Lso libnsl.rpm https://bed-artifactory.bedford.progress.com:443/artifactory/ml-rpm-release-tierpoint/devdependencies/libnsl-2.28-251.el8_10.5.x86_64.rpm \ - && rpm -i libnsl.rpm \ - && rm -f libnsl.rpm + && rpm -i https://repo.almalinux.org/almalinux/8/BaseOS/x86_64/os/Packages/libnsl-2.28-251.el8_10.27.x86_64.rpm ############################################################### # install networking, base deps and tzdata for timezone ############################################################### # hadolint ignore=DL3006 RUN echo "NETWORKING=yes" > /etc/sysconfig/network \ - && microdnf -y install --setopt install_weak_deps=0 gdb redhat-lsb-core initscripts tzdata glibc libstdc++.i686 \ + && microdnf -y install --setopt install_weak_deps=0 gdb redhat-lsb-core initscripts tzdata glibc libstdc++ hostname \ && microdnf clean all +############################################################### +# MarkLogic 10 requires 32-bit libstdc++ runtime +############################################################### +RUN if [ -n "$ML_VERSION" ] && [ "${ML_VERSION%%.*}" = "10" ]; then \ + microdnf -y install --setopt install_weak_deps=0 libstdc++.i686 && \ + microdnf clean all; \ + fi + ############################################################### # Enable FIPS Mode diff --git a/dockerFiles/marklogic-server-ubi-rootless:base b/dockerFiles/marklogic-server-ubi-rootless:base index 7706a5e3..3892d6df 100644 --- a/dockerFiles/marklogic-server-ubi-rootless:base +++ b/dockerFiles/marklogic-server-ubi-rootless:base @@ -1,6 +1,6 @@ ############################################################### # -# Copyright (c) 2023 MarkLogic Corporation +# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. # ############################################################### @@ -59,9 +59,15 @@ RUN touch /etc/marklogic.conf \ # Add TINI to serve as PID 1 process ############################################################### ENV TINI_VERSION=v0.19.0 -ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini -RUN chown ${ML_USER}:users /tini \ - && chmod +x /tini +ARG ML_DOCKER_TYPE=ubi +RUN if [ "${ML_DOCKER_TYPE}" = "ubi9-rootless-arm" ]; then \ + TINI_BIN="tini-arm64"; \ + else \ + TINI_BIN="tini"; \ + fi && \ + curl -fsSL https://github.com/krallin/tini/releases/download/${TINI_VERSION}/${TINI_BIN} -o /tini && \ + chown ${ML_USER}:users /tini && \ + chmod +x /tini ############################################################### # second stage for flattening layers @@ -71,7 +77,7 @@ FROM ${BASE_IMAGE} COPY --from=builder / / ARG ML_USER="marklogic_user" -ARG ML_VERSION=10-internal +ARG ML_VERSION=11-internal ARG ML_DOCKER_VERSION=local ARG BUILD_BRANCH=local ARG ML_DOCKER_TYPE=ubi @@ -118,7 +124,6 @@ ENV MARKLOGIC_INSTALL_DIR=/opt/MarkLogic \ MARKLOGIC_WALLET_PASSWORD_FILE=mldb_wallet_password \ BUILD_BRANCH=${BUILD_BRANCH} \ MARKLOGIC_JOIN_TLS_ENABLED=false \ - OVERWRITE_ML_CONF=true \ MARKLOGIC_EC2_HOST=0 \ TZ=UTC @@ -150,12 +155,12 @@ RUN touch /.dockerenv \ ############################################################### WORKDIR / COPY ${ML_CONVERTERS} /tmp/converters.rpm -RUN chown ${ML_USER}:users /tmp/converters.rpm +RUN if [ -s /tmp/converters.rpm ]; then chown ${ML_USER}:users /tmp/converters.rpm; else rm -f /tmp/converters.rpm; fi ############################################################### # Remove optional packages that have known vulnerabilities ############################################################### -RUN for package in vim-minimal cups-client cups-libs tar python3-pip-wheel platform-python python3-libs platform-python-setuptools avahi-libs binutils expat libarchive python3 python3-libs python-unversioned-command; \ +RUN for package in vim-minimal cups-client cups-libs tar python3-pip-wheel platform-python python3-libs platform-python-setuptools avahi-libs binutils expat libarchive python3 python3-libs python-unversioned-command binutils-gold; \ do rpm -e --nodeps $package || true; \ done; diff --git a/dockerFiles/marklogic-server-ubi9-arm:base b/dockerFiles/marklogic-server-ubi9-arm:base new file mode 100644 index 00000000..ad81e263 --- /dev/null +++ b/dockerFiles/marklogic-server-ubi9-arm:base @@ -0,0 +1,151 @@ +############################################################### +# +# Copyright © 2018-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. +# +############################################################### + +ARG BASE_IMAGE=marklogic-ubi/marklogic-deps-ubi9-arm:11-internal +FROM ${BASE_IMAGE} AS builder + +############################################################### +# set build args +############################################################### + +ARG ML_RPM=marklogic.rpm +ARG ML_USER="marklogic_user" +ARG ML_VERSION=11-internal +ARG ML_CONVERTERS=marklogic.converters +#################################################### +# inject init, start and clustering scripts +############################################################### + +COPY scripts/start-marklogic.sh /usr/local/bin/start-marklogic.sh + +############################################################### +# install MarkLogic server, sudo, and remove mlcmd packages +############################################################### +COPY ${ML_RPM} /tmp/marklogic-server.rpm +RUN rpm -i /tmp/marklogic-server.rpm \ + && rm /tmp/marklogic-server.rpm \ + && microdnf -y install --setopt install_weak_deps=0 sudo \ + && microdnf -y clean all \ + && rm -rf ./opt/MarkLogic/mlcmd/lib/* \ + && rm -rf ./opt/MarkLogic/mlcmd/ext/* + +############################################################### +# Add TINI to serve as PID 1 process +############################################################### +ENV TINI_VERSION=v0.19.0 +ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-arm64 /tini +RUN chmod +x /tini + +############################################################### +# Copy converters package +############################################################### +WORKDIR / +COPY ${ML_CONVERTERS} converters.rpm +############################################################### +# create system user +############################################################### + +RUN adduser --gid users --uid 1000 ${ML_USER} \ + && echo ${ML_USER}" ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers + +############################################################### +# second stage for flattening layers +############################################################### +FROM ${BASE_IMAGE} + +COPY --from=builder / / + +ARG ML_USER="marklogic_user" +ARG ML_VERSION=11-internal +ARG ML_DOCKER_VERSION=local +ARG BUILD_BRANCH=local +ARG ML_DOCKER_TYPE=ubi +############################################################### +# define docker labels +############################################################### + +LABEL "com.marklogic.maintainer"="docker@marklogic.com" +LABEL "com.marklogic.name"="MarkLogic Server ${ML_VERSION}" +LABEL "com.marklogic.docker-version"="${ML_DOCKER_VERSION}" +LABEL "com.marklogic.release-version"="${ML_VERSION}" +LABEL "com.marklogic.build-branch"="${BUILD_BRANCH}" +LABEL "com.marklogic"="MarkLogic" +LABEL "com.marklogic.release-type"="production" +LABEL "com.marklogic.license"="MarkLogic EULA" +LABEL "com.marklogic.license.description"="By subscribing to this product, you agree to the terms and conditions outlined in MarkLogic's End User License Agreement (EULA) here https://developer.marklogic.com/eula " +LABEL "com.marklogic.license.url"="https://developer.marklogic.com/eula" +LABEL "com.marklogic.description"="MarkLogic is the only Enterprise NoSQL database. It is a new generation database built with a flexible data model to store, manage, and search JSON, XML, RDF, and more - without sacrificing enterprise features such as ACID transactions, certified security, backup, and recovery. With these capabilities, MarkLogic is ideally suited for making heterogeneous data integration simpler and faster, and for delivering dynamic content at massive scale. The current release of the MarkLogic Server Developer Docker image includes all features and is limited to developer use." +LABEL docker.cmd="docker run -it -p 7997-8010:7997-8010 -e MARKLOGIC_INIT=true -e MARKLOGIC_ADMIN_USERNAME= -e MARKLOGIC_ADMIN_PASSWORD= --mount src=MarkLogic,dst=/var/opt/MarkLogic progressofficial/marklogic-db:${ML_VERSION}" + +############################################################### +# copy notice file +############################################################### +COPY --chown=${ML_USER}:users NOTICE.txt /home/${ML_USER}/NOTICE.txt + +############################################################### +# set env vars +############################################################### + +ENV MARKLOGIC_INSTALL_DIR=/opt/MarkLogic \ + MARKLOGIC_DATA_DIR=/var/opt/MarkLogic \ + MARKLOGIC_USER=${ML_USER} \ + MARKLOGIC_PID_FILE=/var/run/MarkLogic.pid \ + MARKLOGIC_UMASK=022 \ + LD_LIBRARY_PATH=/lib64:$LD_LIBRARY_PATH:/opt/MarkLogic/lib \ + MARKLOGIC_VERSION="${ML_VERSION}" \ + MARKLOGIC_DOCKER_VERSION="${ML_DOCKER_VERSION}" \ + MARKLOGIC_IMAGE_TYPE="$ML_DOCKER_TYPE" \ + MARKLOGIC_BOOTSTRAP_HOST=bootstrap \ + MARKLOGIC_ADMIN_USERNAME_FILE=mldb_admin_user \ + MARKLOGIC_ADMIN_PASSWORD_FILE=mldb_password_user \ + MARKLOGIC_WALLET_PASSWORD_FILE=mldb_wallet_password \ + BUILD_BRANCH=${BUILD_BRANCH} \ + MARKLOGIC_JOIN_TLS_ENABLED=false \ + OVERWRITE_ML_CONF=true \ + MARKLOGIC_EC2_HOST=0 + +################################################################ +# Set Timezone +################################################################ + +RUN microdnf -y reinstall tzdata + +############################################################### +# Remove optional packages that have known vulnerabilities +############################################################### +RUN for package in vim-minimal cups-client cups-libs tar python3-pip-wheel platform-python python3-libs platform-python-setuptools avahi-libs binutils expat libarchive python3 python3-libs python-unversioned-command binutils-gold; \ + do rpm -e --nodeps $package || true; \ + done; + +############################################################### +# expose MarkLogic server ports +############################################################### + +EXPOSE 25 7997-8010 + +############################################################### +# set system user +############################################################### + +USER ${ML_USER} + +#################################################### +# Set Linux Language Settings +############################################################### + +ENV LANG=en_US.UTF-8 +ENV LC_ALL=C.UTF-8 + +############################################################### +# define volume for persistent MarkLogic server data +############################################################### + +VOLUME /var/opt/MarkLogic + +############################################################### +# set entrypoint +############################################################### +ENTRYPOINT ["/tini", "--", "/usr/local/bin/start-marklogic.sh"] diff --git a/dockerFiles/marklogic-server-ubi:base b/dockerFiles/marklogic-server-ubi:base index b7f7acff..7fa8ca2a 100644 --- a/dockerFiles/marklogic-server-ubi:base +++ b/dockerFiles/marklogic-server-ubi:base @@ -1,6 +1,6 @@ ############################################################### # -# Copyright (c) 2023 MarkLogic Corporation +# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. # ############################################################### @@ -59,7 +59,7 @@ FROM ${BASE_IMAGE} COPY --from=builder / / ARG ML_USER="marklogic_user" -ARG ML_VERSION=10-internal +ARG ML_VERSION=11-internal ARG ML_DOCKER_VERSION=local ARG BUILD_BRANCH=local ARG ML_DOCKER_TYPE=ubi @@ -103,7 +103,9 @@ ENV MARKLOGIC_INSTALL_DIR=/opt/MarkLogic \ MARKLOGIC_ADMIN_PASSWORD_FILE=mldb_password_user \ MARKLOGIC_WALLET_PASSWORD_FILE=mldb_wallet_password \ BUILD_BRANCH=${BUILD_BRANCH} \ - MARKLOGIC_JOIN_TLS_ENABLED=false + MARKLOGIC_JOIN_TLS_ENABLED=false \ + OVERWRITE_ML_CONF=true \ + MARKLOGIC_EC2_HOST=0 ################################################################ # Set Timezone @@ -114,7 +116,7 @@ RUN microdnf -y reinstall tzdata ############################################################### # Remove optional packages that have known vulnerabilities ############################################################### -RUN for package in vim-minimal cups-client cups-libs tar python3-pip-wheel platform-python python3-libs platform-python-setuptools avahi-libs binutils expat libarchive python3 python3-libs python-unversioned-command; \ +RUN for package in vim-minimal cups-client cups-libs tar python3-pip-wheel platform-python python3-libs platform-python-setuptools avahi-libs binutils expat libarchive python3 python3-libs python-unversioned-command binutils-gold; \ do rpm -e --nodeps $package || true; \ done; diff --git a/src/scripts/start-marklogic-rootless.sh b/src/scripts/start-marklogic-rootless.sh index b58edddc..6b4c3d33 100755 --- a/src/scripts/start-marklogic-rootless.sh +++ b/src/scripts/start-marklogic-rootless.sh @@ -1,7 +1,7 @@ #! /bin/bash ############################################################### # -# Copyright 2023 MarkLogic Corporation. All Rights Reserved. +# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. # ############################################################### # Initialise and start MarkLogic server @@ -32,11 +32,6 @@ log () { echo "${TIMESTAMP} ${LOG_LEVEL}: $*" } -############################################################### -# removing MarkLogic ready file and create it when 8001 is accessible on node -############################################################### -rm -f /var/opt/MarkLogic/ready - ############################################################### # Prepare script ############################################################### @@ -54,7 +49,7 @@ done HOST_FQDN="${HOSTNAME}" if [[ -n "${MARKLOGIC_FQDN_SUFFIX}" ]]; then HOST_FQDN="$(hostname).${MARKLOGIC_FQDN_SUFFIX}" - echo "export MARKLOGIC_HOSTNAME=\"${HOST_FQDN}\"" | tee /etc/marklogic.conf + echo "export MARKLOGIC_HOSTNAME=\"${HOST_FQDN}\"" | tee -a /etc/marklogic.conf fi ################################################################ @@ -62,15 +57,28 @@ fi ################################################################ # If an ENV value exists in a list, append it to the /etc/marklogic.conf file - - [[ "${MARKLOGIC_PID_FILE}" ]] && echo "export MARKLOGIC_PID_FILE=$MARKLOGIC_PID_FILE" >>/etc/marklogic.conf - [[ "${MARKLOGIC_UMASK}" ]] && echo "export MARKLOGIC_UMASK=$MARKLOGIC_UMASK" >>/etc/marklogic.conf - [[ "${TZ}" ]] && echo "export TZ=$TZ" >>/etc/marklogic.conf - [[ "${ML_HUGEPAGES_TOTAL}" ]] && echo "export ML_HUGEPAGES_TOTAL=$ML_HUGEPAGES_TOTAL" >>/etc/marklogic.conf - [[ "${MARKLOGIC_DISABLE_JVM}" ]] && echo "export MARKLOGIC_DISABLE_JVM=$MARKLOGIC_DISABLE_JVM" >>/etc/marklogic.conf - [[ "${MARKLOGIC_USER}" ]] && echo "export MARKLOGIC_USER=$MARKLOGIC_USER" >>/etc/marklogic.conf - [[ "${JAVA_HOME}" ]] && echo "export JAVA_HOME=$JAVA_HOME" >>/etc/marklogic.conf - [[ "${CLASSPATH}" ]] && echo "export CLASSPATH=$CLASSPATH" >>/etc/marklogic.conf +info "/etc/marklogic.conf will be appended with provided environment variables." + +# List of environment variables to append to /etc/marklogic.conf if set +ENV_VARS=( + "MARKLOGIC_PID_FILE" + "MARKLOGIC_UMASK" + "TZ" + "ML_HUGEPAGES_TOTAL" + "MARKLOGIC_DISABLE_JVM" + "MARKLOGIC_USER" + "JAVA_HOME" + "CLASSPATH" + "MARKLOGIC_EC2_HOST" +) + +for var in "${ENV_VARS[@]}"; do + value="${!var}" + if [[ -n "$value" ]]; then + echo "export $var=$value" >> /etc/marklogic.conf + info "Appended $var to /etc/marklogic.conf" + fi +done ################################################################ # Install Converters if required @@ -92,10 +100,12 @@ else error "INSTALL_CONVERTERS must be true or false." exit fi - -# Values taken directy from documentation: https://docs.marklogic.com/guide/admin-api/cluster#id_10889 -N_RETRY=5 +# N_RETRY: Number of retries for failed operations (default: 5) +N_RETRY=${N_RETRY:-15} +# RETRY_INTERVAL: Interval in seconds between retries (default: 10) RETRY_INTERVAL=10 +# CURL_TIMEOUT: Timeout in seconds for curl commands (default: 300) +CURL_TIMEOUT=${CURL_TIMEOUT:-300} ################################################################ # restart_check(hostname, baseline_timestamp) @@ -111,11 +121,11 @@ RETRY_INTERVAL=10 function restart_check { info "Waiting for MarkLogic to restart." local retry_count LAST_START - LAST_START=$(curl -s --anyauth --user "${ML_ADMIN_USERNAME}":"${ML_ADMIN_PASSWORD}" "http://$1:8001/admin/v1/timestamp") + LAST_START=$(curl -s -m "${CURL_TIMEOUT}" --anyauth --user "${ML_ADMIN_USERNAME}":"${ML_ADMIN_PASSWORD}" "http://$1:8001/admin/v1/timestamp") for ((retry_count = 0; retry_count < N_RETRY; retry_count = retry_count + 1)); do - if [ "$2" == "${LAST_START}" ] || [ -z "${LAST_START}" ]; then + if [[ "$2" == "${LAST_START}" ]] || [[ -z "${LAST_START}" ]]; then sleep ${RETRY_INTERVAL} - LAST_START=$(curl -s --anyauth --user "${ML_ADMIN_USERNAME}":"${ML_ADMIN_PASSWORD}" "http://$1:8001/admin/v1/timestamp") + LAST_START=$(curl -s -m "${CURL_TIMEOUT}" --anyauth --user "${ML_ADMIN_USERNAME}":"${ML_ADMIN_PASSWORD}" "http://$1:8001/admin/v1/timestamp") else info "MarkLogic has restarted." return 0 @@ -172,9 +182,9 @@ function validate_cert { local cacertfile=$1 local return_code local curl_output - curl_output=$(curl -s -S -L --cacert "${cacertfile}" --ssl "${ML_BOOTSTRAP_PROTOCOL}"://"${MARKLOGIC_BOOTSTRAP_HOST}":8001 --anyauth --user "${ML_ADMIN_USERNAME}":"${ML_ADMIN_PASSWORD}") + curl_output=$(curl -s -S -L -m "${CURL_TIMEOUT}" --cacert "${cacertfile}" --ssl "${ML_BOOTSTRAP_PROTOCOL}"://"${MARKLOGIC_BOOTSTRAP_HOST}":8001 --anyauth --user "${ML_ADMIN_USERNAME}":"${ML_ADMIN_PASSWORD}") return_code=$? - if [ $return_code -ne 0 ]; then + if [[ $return_code != 0 ]]; then info "$curl_output" error "MARKLOGIC_JOIN_CACERT_FILE is not valid, please check above error for details. Node shutting down." exit fi @@ -187,6 +197,8 @@ function validate_cert { # Use RETRY_INTERVAL to tune the test length. # Validate that response code is the same as expected response # code or exit with an error. +# For instance-admin endpoint, implement special retry logic with exponential backoff +# due to network latency issues. # # $1 : Flag indicating if the script should exit if the given response code is not received ("true" to exit, "false" to return the response code") # $2 : The target url to test against @@ -200,9 +212,26 @@ function curl_retry_validate { local expected_response_code=$1; shift local curl_options=("$@") + # Special case: instance-admin must only be invoked once (non-idempotent) + if [[ "${endpoint}" == *"/admin/v1/instance-admin"* ]]; then + response=$(curl -s -m "${CURL_TIMEOUT}" -w '%{http_code}' "${curl_options[@]}" "$endpoint") + response_code=$(tail -n1 <<< "$response") + response_content=$(sed '$ d' <<< "$response") + + if [[ ${response_code} -eq ${expected_response_code} ]]; then + return "${response_code}" + fi + + echo "${response_content}" > start-marklogic_curl_retry_validate.log + if [[ "${return_error}" == "false" ]] ; then + return "${response_code}" + fi + [[ -f "start-marklogic_curl_retry_validate.log" ]] && cat start-marklogic_curl_retry_validate.log + error "Expected response code ${expected_response_code}, got ${response_code} from ${endpoint}." exit + fi + for ((retry_count = 0; retry_count < N_RETRY; retry_count = retry_count + 1)); do - - response=$(curl -s -m 30 -w '%{http_code}' "${curl_options[@]}" "$endpoint") + response=$(curl -s -m "${CURL_TIMEOUT}" -w '%{http_code}' "${curl_options[@]}" "$endpoint") response_code=$(tail -n1 <<< "$response") response_content=$(sed '$ d' <<< "$response") @@ -214,10 +243,10 @@ function curl_retry_validate { sleep ${RETRY_INTERVAL} done - if [[ "${return_error}" = "false" ]] ; then + if [[ "${return_error}" == "false" ]] ; then return "${response_code}" fi - [ -f "start-marklogic_curl_retry_validate.log" ] && cat start-marklogic_curl_retry_validate.log + [[ -f "start-marklogic_curl_retry_validate.log" ]] && cat start-marklogic_curl_retry_validate.log error "Expected response code ${expected_response_code}, got ${response_code} from ${endpoint}." exit } @@ -367,7 +396,7 @@ elif [[ "${MARKLOGIC_INIT}" == "true" ]]; then fi info "Initializing MarkLogic on ${HOSTNAME}" - TIMESTAMP=$(curl --anyauth -m 30 -s --retry 5 \ + TIMESTAMP=$(curl --anyauth -m "${CURL_TIMEOUT}" -s --retry 5 \ -i -X POST -H "Content-type:application/json" \ -d "${LICENSE_PAYLOAD}" \ http://"${HOSTNAME}":8001/admin/v1/init | @@ -470,31 +499,29 @@ fi # use latest health check only for version 11 and up if [[ "${MARKLOGIC_VERSION}" =~ "10" ]] || [[ "${MARKLOGIC_VERSION}" =~ "9" ]]; then HEALTH_CHECK="7997" -else +else HEALTH_CHECK="7997/LATEST/healthcheck" + OLD_HEALTH_CHECK="7997" fi ML_HOST_PROTOCOL=$(get_host_protocol "localhost" "7997") while true do HOST_RESP_CODE=$(curl "${ML_HOST_PROTOCOL}"://"${HOSTNAME}":"${HEALTH_CHECK}" -X GET -o host_health.xml -s -w "%{http_code}\n" --cacert "${ML_CACERT_FILE}") - if [[ "${MARKLOGIC_INIT}" == "true" ]] && [ "${HOST_RESP_CODE}" -eq 200 ]; then - touch /var/opt/MarkLogic/ready - info "Cluster config complete, marking this container as ready." - break - elif [[ "${MARKLOGIC_INIT}" != "true" ]]; then - touch /var/opt/MarkLogic/ready - info "Cluster config complete, marking this container as ready." - rm -f host_health.xml - break - elif [[ -f /var/opt/MarkLogic/DOCKER_INIT ]] && [ "${HOST_RESP_CODE}" -eq 200 ]; then - touch /var/opt/MarkLogic/ready + if [[ "${HOST_RESP_CODE}" == "200" ]] || [[ "${MARKLOGIC_INIT}" != "true" ]]; then info "Cluster config complete, marking this container as ready." break + elif [[ "${HOST_RESP_CODE}" == "404" ]]; then + # check old healthcheck in case of upgrade + HOST_RESP_CODE=$(curl "${ML_HOST_PROTOCOL}"://"${HOSTNAME}":"${OLD_HEALTH_CHECK}" -X GET -o host_health.xml -s -w "%{http_code}\n" --cacert "${ML_CACERT_FILE}") + if [[ "${HOST_RESP_CODE}" == "200" ]]; then + info "Cluster config complete, marking this container as ready." + break + fi else info "MarkLogic not ready yet, retrying." - sleep 5 fi + sleep 5 done ################################################################ diff --git a/src/scripts/start-marklogic.sh b/src/scripts/start-marklogic.sh index caf2dbd2..68679a44 100755 --- a/src/scripts/start-marklogic.sh +++ b/src/scripts/start-marklogic.sh @@ -1,7 +1,7 @@ #! /bin/bash ############################################################### # -# Copyright 2023 MarkLogic Corporation. All Rights Reserved. +# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. # ############################################################### # Initialise and start MarkLogic server @@ -32,11 +32,6 @@ log () { echo "${TIMESTAMP} ${LOG_LEVEL}: $*" } -############################################################### -# removing MarkLogic ready file and create it when 8001 is accessible on node -############################################################### -sudo rm -f /var/opt/MarkLogic/ready - ############################################################### # Prepare script ############################################################### @@ -54,7 +49,8 @@ done HOST_FQDN="${HOSTNAME}" if [[ -n "${MARKLOGIC_FQDN_SUFFIX}" ]]; then HOST_FQDN="$(hostname).${MARKLOGIC_FQDN_SUFFIX}" - echo "export MARKLOGIC_HOSTNAME=\"${HOST_FQDN}\"" | sudo tee /etc/marklogic.conf + echo "export MARKLOGIC_HOSTNAME=\"${HOST_FQDN}\"" | sudo tee -a /etc/marklogic.conf + sudo chmod 777 /etc/marklogic.conf fi ################################################################ @@ -67,21 +63,24 @@ if [[ "${OVERWRITE_ML_CONF}" == "true" ]]; then rm -f /etc/marklogic.conf sudo touch /etc/marklogic.conf && sudo chmod 777 /etc/marklogic.conf - [[ "${MARKLOGIC_PID_FILE}" ]] && echo "export MARKLOGIC_PID_FILE=$MARKLOGIC_PID_FILE" >>/etc/marklogic.conf - [[ "${MARKLOGIC_UMASK}" ]] && echo "export MARKLOGIC_UMASK=$MARKLOGIC_UMASK" >>/etc/marklogic.conf - [[ "${TZ}" ]] && echo "export TZ=$TZ " >>/etc/marklogic.conf - [[ "${MARKLOGIC_ADMIN_USERNAME}" ]] && echo "export MARKLOGIC_ADMIN_USERNAME=$MARKLOGIC_ADMIN_USERNAME" >>/etc/marklogic.conf - [[ "${MARKLOGIC_ADMIN_PASSWORD}" ]] && echo "export MARKLOGIC_ADMIN_PASSWORD=$MARKLOGIC_ADMIN_PASSWORD" >>/etc/marklogic.conf - [[ "${MARKLOGIC_WALLET_PASSWORD}" ]] && echo "export MARKLOGIC_WALLET_PASSWORD=$MARKLOGIC_WALLET_PASSWORD" >>/etc/marklogic.conf - [[ "${REALM}" ]] && echo "export REALM=$REALM" >>/etc/marklogic.conf - [[ "${MARKLOGIC_LICENSEE}" ]] && echo "export MARKLOGIC_LICENSEE=$MARKLOGIC_LICENSEE" >>/etc/marklogic.conf - [[ "${MARKLOGIC_LICENSE_KEY}" ]] && echo "export MARKLOGIC_LICENSE_KEY=$MARKLOGIC_LICENSE_KEY" >>/etc/marklogic.conf - [[ "${MARKLOGIC_GROUP}" ]] && echo "export MARKLOGIC_GROUP=$MARKLOGIC_GROUP" >>/etc/marklogic.conf - [[ "${ML_HUGEPAGES_TOTAL}" ]] && echo "export ML_HUGEPAGES_TOTAL=$ML_HUGEPAGES_TOTAL" >>/etc/marklogic.conf - [[ "${MARKLOGIC_DISABLE_JVM}" ]] && echo "export MARKLOGIC_DISABLE_JVM=$MARKLOGIC_DISABLE_JVM" >>/etc/marklogic.conf - [[ "${MARKLOGIC_USER}" ]] && echo "export MARKLOGIC_USER=$MARKLOGIC_USER" >>/etc/marklogic.conf - [[ "${JAVA_HOME}" ]] && echo "export JAVA_HOME=$JAVA_HOME" >>/etc/marklogic.conf - [[ "${CLASSPATH}" ]] && echo "export CLASSPATH=$CLASSPATH" >>/etc/marklogic.conf + ENV_VARS=( + "MARKLOGIC_PID_FILE" + "MARKLOGIC_UMASK" + "TZ" + "ML_HUGEPAGES_TOTAL" + "MARKLOGIC_DISABLE_JVM" + "MARKLOGIC_USER" + "JAVA_HOME" + "CLASSPATH" + "MARKLOGIC_EC2_HOST" + ) + for var in "${ENV_VARS[@]}"; do + value="${!var}" + if [[ -n "$value" ]]; then + echo "export $var=$value" >> /etc/marklogic.conf + info "Appended $var to /etc/marklogic.conf" + fi + done sudo chmod 400 /etc/marklogic.conf @@ -111,15 +110,18 @@ fi ################################################################ # Setup timezone ################################################################ -if [ -n "${TZ}" ]; then +if [[ -n "${TZ}" ]]; then info "TZ is defined, setting timezone to ${TZ}." sudo ln -snf "/usr/share/zoneinfo/${TZ}" /etc/localtime echo "${TZ}" | sudo tee /etc/timezone fi -# Values taken directy from documentation: https://docs.marklogic.com/guide/admin-api/cluster#id_10889 -N_RETRY=5 +# N_RETRY: Number of retries for failed operations (default: 5) +N_RETRY=${N_RETRY:-15} +# RETRY_INTERVAL: Interval in seconds between retries (default: 10) RETRY_INTERVAL=10 +# CURL_TIMEOUT: Timeout in seconds for curl commands (default: 300) +CURL_TIMEOUT=${CURL_TIMEOUT:-300} ################################################################ # restart_check(hostname, baseline_timestamp) @@ -135,15 +137,15 @@ RETRY_INTERVAL=10 function restart_check { info "Waiting for MarkLogic to restart." local retry_count LAST_START - LAST_START=$(curl -s --anyauth --user "${ML_ADMIN_USERNAME}":"${ML_ADMIN_PASSWORD}" "http://$1:8001/admin/v1/timestamp") + LAST_START=$(curl -s -m "${CURL_TIMEOUT}" --anyauth --user "${ML_ADMIN_USERNAME}":"${ML_ADMIN_PASSWORD}" "http://$1:8001/admin/v1/timestamp") for ((retry_count = 0; retry_count < N_RETRY; retry_count = retry_count + 1)); do - if [ "$2" == "${LAST_START}" ] || [ -z "${LAST_START}" ]; then + if [[ "$2" == "${LAST_START}" ]] || [[ -z "${LAST_START}" ]]; then sleep ${RETRY_INTERVAL} - LAST_START=$(curl -s --anyauth --user "${ML_ADMIN_USERNAME}":"${ML_ADMIN_PASSWORD}" "http://$1:8001/admin/v1/timestamp") + LAST_START=$(curl -s -m "${CURL_TIMEOUT}" --anyauth --user "${ML_ADMIN_USERNAME}":"${ML_ADMIN_PASSWORD}" "http://$1:8001/admin/v1/timestamp") else info "MarkLogic has restarted." - return 0 - fi + return 0 + fi done error "Failed to restart $1" exit } @@ -196,9 +198,9 @@ function validate_cert { local cacertfile=$1 local return_code local curl_output - curl_output=$(curl -s -S -L --cacert "${cacertfile}" --ssl "${ML_BOOTSTRAP_PROTOCOL}"://"${MARKLOGIC_BOOTSTRAP_HOST}":8001 --anyauth --user "${ML_ADMIN_USERNAME}":"${ML_ADMIN_PASSWORD}") + curl_output=$(curl -s -S -L -m "${CURL_TIMEOUT}" --cacert "${cacertfile}" --ssl "${ML_BOOTSTRAP_PROTOCOL}"://"${MARKLOGIC_BOOTSTRAP_HOST}":8001 --anyauth --user "${ML_ADMIN_USERNAME}":"${ML_ADMIN_PASSWORD}") return_code=$? - if [ $return_code -ne 0 ]; then + if [[ $return_code != "0" ]]; then info "$curl_output" error "MARKLOGIC_JOIN_CACERT_FILE is not valid, please check above error for details. Node shutting down." exit fi @@ -211,6 +213,8 @@ function validate_cert { # Use RETRY_INTERVAL to tune the test length. # Validate that response code is the same as expected response # code or exit with an error. +# NOTE: /admin/v1/instance-admin is NOT idempotent; call it exactly once +# and do not retry. # # $1 : Flag indicating if the script should exit if the given response code is not received ("true" to exit, "false" to return the response code") # $2 : The target url to test against @@ -224,9 +228,26 @@ function curl_retry_validate { local expected_response_code=$1; shift local curl_options=("$@") + # Special case: instance-admin must only be invoked once (non-idempotent) + if [[ "${endpoint}" == *"/admin/v1/instance-admin"* ]]; then + response=$(curl -s -m "${CURL_TIMEOUT}" -w '%{http_code}' "${curl_options[@]}" "$endpoint") + response_code=$(tail -n1 <<< "$response") + response_content=$(sed '$ d' <<< "$response") + + if [[ ${response_code} -eq ${expected_response_code} ]]; then + return "${response_code}" + fi + + echo "${response_content}" > start-marklogic_curl_retry_validate.log + if [[ "${return_error}" == "false" ]] ; then + return "${response_code}" + fi + [[ -f "start-marklogic_curl_retry_validate.log" ]] && cat start-marklogic_curl_retry_validate.log + error "Expected response code ${expected_response_code}, got ${response_code} from ${endpoint}." exit + fi + for ((retry_count = 0; retry_count < N_RETRY; retry_count = retry_count + 1)); do - - response=$(curl -s -m 30 -w '%{http_code}' "${curl_options[@]}" "$endpoint") + response=$(curl -s -m "${CURL_TIMEOUT}" -w '%{http_code}' "${curl_options[@]}" "$endpoint") response_code=$(tail -n1 <<< "$response") response_content=$(sed '$ d' <<< "$response") @@ -238,10 +259,10 @@ function curl_retry_validate { sleep ${RETRY_INTERVAL} done - if [[ "${return_error}" = "false" ]] ; then + if [[ "${return_error}" == "false" ]] ; then return "${response_code}" fi - [ -f "start-marklogic_curl_retry_validate.log" ] && cat start-marklogic_curl_retry_validate.log + [[ -f "start-marklogic_curl_retry_validate.log" ]] && cat start-marklogic_curl_retry_validate.log error "Expected response code ${expected_response_code}, got ${response_code} from ${endpoint}." exit } @@ -391,7 +412,7 @@ elif [[ "${MARKLOGIC_INIT}" == "true" ]]; then fi info "Initializing MarkLogic on ${HOSTNAME}" - TIMESTAMP=$(curl --anyauth -m 30 -s --retry 5 \ + TIMESTAMP=$(curl --anyauth -m "${CURL_TIMEOUT}" -s --retry 5 \ -i -X POST -H "Content-type:application/json" \ -d "${LICENSE_PAYLOAD}" \ http://"${HOSTNAME}":8001/admin/v1/init | @@ -486,7 +507,7 @@ elif [[ "${MARKLOGIC_JOIN_CLUSTER}" == "true" ]]; then elif [[ -z "${MARKLOGIC_JOIN_CLUSTER}" ]] || [[ "${MARKLOGIC_JOIN_CLUSTER}" == "false" ]]; then info "MARKLOGIC_JOIN_CLUSTER is false or not defined, not joining cluster." else - error "MARKLOGIC_JOIN_CLUSTER must be true or false." exit + error "MARKLOGIC_JOIN_CLUSTER must be true or false." exit fi ################################################################ @@ -496,31 +517,29 @@ fi # use latest health check only for version 11 and up if [[ "${MARKLOGIC_VERSION}" =~ "10" ]] || [[ "${MARKLOGIC_VERSION}" =~ "9" ]]; then HEALTH_CHECK="7997" -else +else HEALTH_CHECK="7997/LATEST/healthcheck" + OLD_HEALTH_CHECK="7997" fi ML_HOST_PROTOCOL=$(get_host_protocol "localhost" "7997") while true do HOST_RESP_CODE=$(curl "${ML_HOST_PROTOCOL}"://"${HOSTNAME}":"${HEALTH_CHECK}" -X GET -o host_health.xml -s -w "%{http_code}\n" --cacert "${ML_CACERT_FILE}") - if [[ "${MARKLOGIC_INIT}" == "true" ]] && [ "${HOST_RESP_CODE}" -eq 200 ]; then - sudo touch /var/opt/MarkLogic/ready - info "Cluster config complete, marking this container as ready." - break - elif [[ "${MARKLOGIC_INIT}" != "true" ]]; then - sudo touch /var/opt/MarkLogic/ready - info "Cluster config complete, marking this container as ready." - rm -f host_health.xml - break - elif [[ -f /var/opt/MarkLogic/DOCKER_INIT ]] && [ "${HOST_RESP_CODE}" -eq 200 ]; then - sudo touch /var/opt/MarkLogic/ready + if [[ "${HOST_RESP_CODE}" == "200" ]] || [[ "${MARKLOGIC_INIT}" != "true" ]]; then info "Cluster config complete, marking this container as ready." break + elif [[ "${HOST_RESP_CODE}" == "404" ]]; then + # check old healthcheck in case of upgrade + HOST_RESP_CODE=$(curl "${ML_HOST_PROTOCOL}"://"${HOSTNAME}":"${OLD_HEALTH_CHECK}" -X GET -o host_health.xml -s -w "%{http_code}\n" --cacert "${ML_CACERT_FILE}") + if [[ "${HOST_RESP_CODE}" == "200" ]]; then + info "Cluster config complete, marking this container as ready." + break + fi else info "MarkLogic not ready yet, retrying." - sleep 5 fi + sleep 5 done ################################################################ diff --git a/test/README.md b/test/README.md index 958de74d..9cc70752 100644 --- a/test/README.md +++ b/test/README.md @@ -13,6 +13,7 @@ If you'd like to change the image being tested change the variables in the makef ## Docker Image Tests Docker image tests are implemented with Robot framework. The framework requires Python 3.6+ and pip. Framework requirements are listed in requirements file and can be installed with the following commands: +`cd test` `python3 -m venv python_env` `source ./python_env/bin/activate` `pip3 install -r requirements.txt` @@ -26,7 +27,7 @@ For additional installation instruction see https://robotframework.org/robotfram In order to run Docker tests, you need to have MarkLogic Docker image available in your environment. The image can be passed as a parameter or set with DOCKER_TEST_IMAGE environment variable. In order to execute a single test case with a published Docker image, use -`robot --variable TEST_IMAGE:progressofficial/marklogic-db --test "Initialized MarkLogic container" ./docker-tests.robot` +`robot --test "Smoke Test" ./docker-tests.robot` In order to run all tests you can use make from root folder with `make docker-tests test_image=progressofficial/marklogic-db` @@ -36,9 +37,14 @@ or by running Robot in test directly with *Note* QA_LICENSE_KEY environment variable needs to be set to a valid license key for license test to pass. -If UBI rootless image is used, specify image type for individual tests: +Some tests expect certain variable to have valid values. For example, if UBI rootless image is used, specify image type for individual tests: `robot --variable TEST_IMAGE:marklogic/marklogic-server-ubi-rootless:internal --variable IMAGE_TYPE:ubi-rootless --test 'Initialized MarkLogic container with config overrides' docker-tests.robot` +### Resource Requirements for Dynamic Host Tests +The "Dynamic Host Cluster Test" and "Dynamic Host Cluster Concurrency Join Test" require significant system resources as they create 13 containers (3 cluster nodes + 10 dynamic hosts) simultaneously. + +**Note:** These tests may fail with "Connection refused" errors if insufficient resources are allocated. Increase Docker's CPU and memory limits in Docker Desktop/Rancher Desktop preferences if you encounter initialization failures. + For a quick start guide for Robot framework see https://robotframework.org/#getting-started Full user guide is available at https://robotframework.org/robotframework/#user-guide diff --git a/test/compose-test-1.yaml b/test/compose-test-1.yaml index 1715d7cd..8a5a9459 100644 --- a/test/compose-test-1.yaml +++ b/test/compose-test-1.yaml @@ -1,3 +1,4 @@ +# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. version: '3.6' services: bootstrap_3n: diff --git a/test/compose-test-10.yaml b/test/compose-test-10.yaml index cc086567..99bf90aa 100644 --- a/test/compose-test-10.yaml +++ b/test/compose-test-10.yaml @@ -1,3 +1,4 @@ +# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. version: '3.6' services: node2: diff --git a/test/compose-test-11.yaml b/test/compose-test-11.yaml index 830e1367..46897cfb 100644 --- a/test/compose-test-11.yaml +++ b/test/compose-test-11.yaml @@ -1,3 +1,4 @@ +# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. version: '3.6' services: node3: diff --git a/test/compose-test-12.yaml b/test/compose-test-12.yaml index 1715d7cd..8a5a9459 100644 --- a/test/compose-test-12.yaml +++ b/test/compose-test-12.yaml @@ -1,3 +1,4 @@ +# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. version: '3.6' services: bootstrap_3n: diff --git a/test/compose-test-13.yaml b/test/compose-test-13.yaml index 81455f14..493060f4 100644 --- a/test/compose-test-13.yaml +++ b/test/compose-test-13.yaml @@ -1,3 +1,4 @@ +# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. version: '3.6' services: node2: diff --git a/test/compose-test-14.yaml b/test/compose-test-14.yaml index 1715d7cd..8a5a9459 100644 --- a/test/compose-test-14.yaml +++ b/test/compose-test-14.yaml @@ -1,3 +1,4 @@ +# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. version: '3.6' services: bootstrap_3n: diff --git a/test/compose-test-15.yaml b/test/compose-test-15.yaml index 81455f14..493060f4 100644 --- a/test/compose-test-15.yaml +++ b/test/compose-test-15.yaml @@ -1,3 +1,4 @@ +# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. version: '3.6' services: node2: diff --git a/test/compose-test-16.yaml b/test/compose-test-16.yaml new file mode 100644 index 00000000..4728689c --- /dev/null +++ b/test/compose-test-16.yaml @@ -0,0 +1,315 @@ +# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. +#Docker compose file sample to setup a three node cluster +version: '3.6' +services: + bootstrap_3n: + image: progressofficial/marklogic-db + container_name: bootstrap_3n + hostname: bootstrap_3n + dns_search: "" + environment: + - MARKLOGIC_INIT=true + - MARKLOGIC_ADMIN_USERNAME_FILE=mldb_admin_username + - MARKLOGIC_ADMIN_PASSWORD_FILE=mldb_admin_password + - TZ=Europe/Prague + volumes: + - MarkLogic_3n_vol1:/var/opt/MarkLogic + secrets: + - mldb_admin_password + - mldb_admin_username + ports: + - 7100-7110:8000-8010 + - 7197:7997 + networks: + - external_net + node2: + image: progressofficial/marklogic-db + container_name: node2 + hostname: node2 + dns_search: "" + environment: + - MARKLOGIC_INIT=true + - MARKLOGIC_ADMIN_USERNAME_FILE=mldb_admin_username + - MARKLOGIC_ADMIN_PASSWORD_FILE=mldb_admin_password + - MARKLOGIC_JOIN_CLUSTER=true + - MARKLOGIC_BOOTSTRAP_HOST=bootstrap_3n + - TZ=Europe/Prague + volumes: + - MarkLogic_3n_vol2:/var/opt/MarkLogic + secrets: + - mldb_admin_password + - mldb_admin_username + ports: + - 7200-7210:8000-8010 + - 7297:7997 + depends_on: + - bootstrap_3n + networks: + - external_net + node3: + image: progressofficial/marklogic-db + container_name: node3 + hostname: node3 + dns_search: "" + environment: + - MARKLOGIC_INIT=true + - MARKLOGIC_ADMIN_USERNAME_FILE=mldb_admin_username + - MARKLOGIC_ADMIN_PASSWORD_FILE=mldb_admin_password + - MARKLOGIC_JOIN_CLUSTER=true + - MARKLOGIC_BOOTSTRAP_HOST=bootstrap_3n + - TZ=Europe/Prague + volumes: + - MarkLogic_3n_vol3:/var/opt/MarkLogic + secrets: + - mldb_admin_password + - mldb_admin_username + ports: + - 7300-7310:8000-8010 + - 7397:7997 + depends_on: + - bootstrap_3n + networks: + - external_net + dynamic0: + image: progressofficial/marklogic-db + container_name: dynamic0 + hostname: dynamic0 + dns_search: "" + environment: + - MARKLOGIC_INIT=false + - MARKLOGIC_ADMIN_USERNAME_FILE=mldb_admin_username + - MARKLOGIC_ADMIN_PASSWORD_FILE=mldb_admin_password + # No MARKLOGIC_JOIN_CLUSTER flag - this keeps it standalone + - TZ=Europe/Prague + volumes: + - MarkLogic_dynamic0_vol:/var/opt/MarkLogic + secrets: + - mldb_admin_password + - mldb_admin_username + ports: + - 7400-7410:8000-8010 + - 7497:7997 + networks: + - external_net + dynamic1: + image: progressofficial/marklogic-db + container_name: dynamic1 + hostname: dynamic1 + dns_search: "" + environment: + - MARKLOGIC_INIT=false + - MARKLOGIC_ADMIN_USERNAME_FILE=mldb_admin_username + - MARKLOGIC_ADMIN_PASSWORD_FILE=mldb_admin_password + - TZ=Europe/Prague + volumes: + - MarkLogic_dynamic1_vol:/var/opt/MarkLogic + secrets: + - mldb_admin_password + - mldb_admin_username + ports: + - 7500-7510:8000-8010 + - 7597:7997 + networks: + - external_net + dynamic2: + image: progressofficial/marklogic-db + container_name: dynamic2 + hostname: dynamic2 + dns_search: "" + environment: + - MARKLOGIC_INIT=false + - MARKLOGIC_ADMIN_USERNAME_FILE=mldb_admin_username + - MARKLOGIC_ADMIN_PASSWORD_FILE=mldb_admin_password + - TZ=Europe/Prague + volumes: + - MarkLogic_dynamic2_vol:/var/opt/MarkLogic + secrets: + - mldb_admin_password + - mldb_admin_username + ports: + - 7600-7610:8000-8010 + - 7697:7997 + networks: + - external_net + dynamic3: + image: progressofficial/marklogic-db + container_name: dynamic3 + hostname: dynamic3 + dns_search: "" + environment: + - MARKLOGIC_INIT=false + - MARKLOGIC_ADMIN_USERNAME_FILE=mldb_admin_username + - MARKLOGIC_ADMIN_PASSWORD_FILE=mldb_admin_password + - TZ=Europe/Prague + volumes: + - MarkLogic_dynamic3_vol:/var/opt/MarkLogic + secrets: + - mldb_admin_password + - mldb_admin_username + ports: + - 7700-7710:8000-8010 + - 7797:7997 + networks: + - external_net + dynamic4: + image: progressofficial/marklogic-db + container_name: dynamic4 + hostname: dynamic4 + dns_search: "" + environment: + - MARKLOGIC_INIT=false + - MARKLOGIC_ADMIN_USERNAME_FILE=mldb_admin_username + - MARKLOGIC_ADMIN_PASSWORD_FILE=mldb_admin_password + - TZ=Europe/Prague + volumes: + - MarkLogic_dynamic4_vol:/var/opt/MarkLogic + secrets: + - mldb_admin_password + - mldb_admin_username + ports: + - 7800-7810:8000-8010 + - 7897:7997 + networks: + - external_net + dynamic5: + image: progressofficial/marklogic-db + container_name: dynamic5 + hostname: dynamic5 + dns_search: "" + environment: + - MARKLOGIC_INIT=false + - MARKLOGIC_ADMIN_USERNAME_FILE=mldb_admin_username + - MARKLOGIC_ADMIN_PASSWORD_FILE=mldb_admin_password + - TZ=Europe/Prague + volumes: + - MarkLogic_dynamic5_vol:/var/opt/MarkLogic + secrets: + - mldb_admin_password + - mldb_admin_username + ports: + - 7900-7910:8000-8010 + - 7997:7997 + networks: + - external_net + dynamic6: + image: progressofficial/marklogic-db + container_name: dynamic6 + hostname: dynamic6 + dns_search: "" + environment: + - MARKLOGIC_INIT=false + - MARKLOGIC_ADMIN_USERNAME_FILE=mldb_admin_username + - MARKLOGIC_ADMIN_PASSWORD_FILE=mldb_admin_password + - TZ=Europe/Prague + volumes: + - MarkLogic_dynamic6_vol:/var/opt/MarkLogic + secrets: + - mldb_admin_password + - mldb_admin_username + ports: + - 8000-8010:8000-8010 + - 8097:7997 + networks: + - external_net + dynamic7: + image: progressofficial/marklogic-db + container_name: dynamic7 + hostname: dynamic7 + dns_search: "" + environment: + - MARKLOGIC_INIT=false + - MARKLOGIC_ADMIN_USERNAME_FILE=mldb_admin_username + - MARKLOGIC_ADMIN_PASSWORD_FILE=mldb_admin_password + - TZ=Europe/Prague + volumes: + - MarkLogic_dynamic7_vol:/var/opt/MarkLogic + secrets: + - mldb_admin_password + - mldb_admin_username + ports: + - 8100-8110:8000-8010 + - 8197:7997 + networks: + - external_net + dynamic8: + image: progressofficial/marklogic-db + container_name: dynamic8 + hostname: dynamic8 + dns_search: "" + environment: + - MARKLOGIC_INIT=false + - MARKLOGIC_ADMIN_USERNAME_FILE=mldb_admin_username + - MARKLOGIC_ADMIN_PASSWORD_FILE=mldb_admin_password + - TZ=Europe/Prague + volumes: + - MarkLogic_dynamic8_vol:/var/opt/MarkLogic + secrets: + - mldb_admin_password + - mldb_admin_username + ports: + - 8200-8210:8000-8010 + - 8297:7997 + networks: + - external_net + dynamic9: + image: progressofficial/marklogic-db + container_name: dynamic9 + hostname: dynamic9 + dns_search: "" + environment: + - MARKLOGIC_INIT=false + - MARKLOGIC_ADMIN_USERNAME_FILE=mldb_admin_username + - MARKLOGIC_ADMIN_PASSWORD_FILE=mldb_admin_password + - TZ=Europe/Prague + volumes: + - MarkLogic_dynamic9_vol:/var/opt/MarkLogic + secrets: + - mldb_admin_password + - mldb_admin_username + ports: + - 8300-8310:8000-8010 + - 8397:7997 + networks: + - external_net + dynamic10: + image: progressofficial/marklogic-db + container_name: dynamic10 + hostname: dynamic10 + dns_search: "" + environment: + - MARKLOGIC_INIT=false + - MARKLOGIC_ADMIN_USERNAME_FILE=mldb_admin_username + - MARKLOGIC_ADMIN_PASSWORD_FILE=mldb_admin_password + - TZ=Europe/Prague + volumes: + - MarkLogic_dynamic10_vol:/var/opt/MarkLogic + secrets: + - mldb_admin_password + - mldb_admin_username + ports: + - 8400-8410:8000-8010 + - 8497:7997 + networks: + - external_net +secrets: + mldb_admin_password: + file: ./mldb_admin_password.txt + mldb_admin_username: + file: ./mldb_admin_username.txt +networks: + external_net: {} +volumes: + MarkLogic_3n_vol1: + MarkLogic_3n_vol2: + MarkLogic_3n_vol3: + MarkLogic_dynamic0_vol: + MarkLogic_dynamic1_vol: + MarkLogic_dynamic2_vol: + MarkLogic_dynamic3_vol: + MarkLogic_dynamic4_vol: + MarkLogic_dynamic5_vol: + MarkLogic_dynamic6_vol: + MarkLogic_dynamic7_vol: + MarkLogic_dynamic8_vol: + MarkLogic_dynamic9_vol: + MarkLogic_dynamic10_vol: \ No newline at end of file diff --git a/test/compose-test-2.yaml b/test/compose-test-2.yaml index 81455f14..493060f4 100644 --- a/test/compose-test-2.yaml +++ b/test/compose-test-2.yaml @@ -1,3 +1,4 @@ +# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. version: '3.6' services: node2: diff --git a/test/compose-test-3.yaml b/test/compose-test-3.yaml index b13fb8e6..8af814a9 100644 --- a/test/compose-test-3.yaml +++ b/test/compose-test-3.yaml @@ -1,3 +1,4 @@ +# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. #Docker compose file sample to setup a three node cluster version: '3.6' services: diff --git a/test/compose-test-4.yaml b/test/compose-test-4.yaml index ba813159..0203c9a9 100644 --- a/test/compose-test-4.yaml +++ b/test/compose-test-4.yaml @@ -1,3 +1,4 @@ +# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. #Docker compose file sample to setup a three node cluster version: '3.6' services: diff --git a/test/compose-test-5.yaml b/test/compose-test-5.yaml index 3e78fba1..2df9c725 100644 --- a/test/compose-test-5.yaml +++ b/test/compose-test-5.yaml @@ -1,3 +1,4 @@ +# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. #Docker compose file sample to setup a three node cluster version: '3.6' services: diff --git a/test/compose-test-6.yaml b/test/compose-test-6.yaml index 11a1cfeb..08c16eae 100644 --- a/test/compose-test-6.yaml +++ b/test/compose-test-6.yaml @@ -1,3 +1,4 @@ +# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. #Docker compose file to setup the bootstrap node on cluster version: '3.6' services: diff --git a/test/compose-test-7.yaml b/test/compose-test-7.yaml index 10e4b581..aaf145ed 100644 --- a/test/compose-test-7.yaml +++ b/test/compose-test-7.yaml @@ -1,3 +1,4 @@ +# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. #Docker compose file to setup and join node2 on cluster using MARKLOGIC_GROUP param version: '3.6' services: diff --git a/test/compose-test-8.yaml b/test/compose-test-8.yaml index 12a4fe80..d444747d 100644 --- a/test/compose-test-8.yaml +++ b/test/compose-test-8.yaml @@ -1,3 +1,4 @@ +# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. #Docker compose file sample to setup a one node cluster version: '3.6' services: diff --git a/test/compose-test-9.yaml b/test/compose-test-9.yaml index ea0ac5a8..d3563f99 100644 --- a/test/compose-test-9.yaml +++ b/test/compose-test-9.yaml @@ -1,3 +1,4 @@ +# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. #Docker compose file sample to setup a two node cluster version: '3.6' services: diff --git a/test/docker-tests.robot b/test/docker-tests.robot index 57fcafe5..8a3f1c40 100644 --- a/test/docker-tests.robot +++ b/test/docker-tests.robot @@ -1,17 +1,34 @@ +# Copyright © 2018-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. *** Settings *** Resource keywords.resource Documentation Test all initialization options using Docker run and Docker Compose. ... Each test case creates and then tears down one or more Docker containers. ... Verification is done using REST calls to MarkLogic server and Docker logs. +Suite Setup Ensure Test Results Directory Exists *** Test Cases *** +Smoke Test + Create container with + Docker log should contain *MARKLOGIC_INIT is set to false or not defined, not initializing.* + [Teardown] Delete container + Uninitialized MarkLogic container Create container with -e MARKLOGIC_INIT=false + IF 'rootless' not in '${IMAGE_TYPE}' # ROOT image + Docker log should contain *OVERWRITE_ML_CONF is true, deleting existing /etc/marklogic.conf and overwriting with ENV variables.* + END + IF 'rootless' in '${IMAGE_TYPE}' # ROOTLESS image + Docker log should contain */etc/marklogic.conf will be appended with provided environment variables.* + END Docker log should contain *MARKLOGIC_JOIN_CLUSTER is false or not defined, not joining cluster.* Docker log should contain *MARKLOGIC_INIT is set to false or not defined, not initializing.* Docker log should contain *Starting container with MarkLogic Server.* Docker log should contain *| server ver: ${MARKLOGIC_VERSION} | scripts ver: ${MARKLOGIC_DOCKER_VERSION} | image type: ${IMAGE_TYPE} | branch: ${BUILD_BRANCH} |* + Docker log should contain *Appended MARKLOGIC_PID_FILE to /etc/marklogic.conf* + Docker log should contain *Appended MARKLOGIC_UMASK to /etc/marklogic.conf* + Docker log should contain *Appended MARKLOGIC_USER to /etc/marklogic.conf* + Docker log should contain *Appended MARKLOGIC_EC2_HOST to /etc/marklogic.conf* Verify response for unauthenticated request with 8000 *Forbidden* Verify response for unauthenticated request with 8001 *This server must now self-install the initial databases and application servers. Click OK to continue.* Verify response for unauthenticated request with 8002 *Forbidden* @@ -20,12 +37,23 @@ Uninitialized MarkLogic container Verify response for authenticated request with 8002 *Forbidden* [Teardown] Delete container -Uninitialized MarkLogic container no parameters +Uninitialized MarkLogic container with no parameters Create container with + IF 'rootless' not in '${IMAGE_TYPE}' # ROOT image + Docker log should contain *OVERWRITE_ML_CONF is true, deleting existing /etc/marklogic.conf and overwriting with ENV variables.* + END + IF 'rootless' in '${IMAGE_TYPE}' # ROOTLESS image + Docker log should contain */etc/marklogic.conf will be appended with provided environment variables.* + END Docker log should contain *MARKLOGIC_JOIN_CLUSTER is false or not defined, not joining cluster.* Docker log should contain *MARKLOGIC_INIT is set to false or not defined, not initializing.* Docker log should contain *Starting container with MarkLogic Server.* Docker log should contain *| server ver: ${MARKLOGIC_VERSION} | scripts ver: ${MARKLOGIC_DOCKER_VERSION} | image type: ${IMAGE_TYPE} | branch: ${BUILD_BRANCH} |* + Docker log should contain *Appended MARKLOGIC_PID_FILE to /etc/marklogic.conf* + Docker log should contain *Appended MARKLOGIC_UMASK to /etc/marklogic.conf* + Docker log should contain *Appended MARKLOGIC_USER to /etc/marklogic.conf* + Docker log should contain *Appended MARKLOGIC_EC2_HOST to /etc/marklogic.conf* + Verify That marklogic.conf contains MARKLOGIC_PID_FILE MARKLOGIC_UMASK MARKLOGIC_USER MARKLOGIC_EC2_HOST=0 Verify response for unauthenticated request with 8000 *Forbidden* Verify response for unauthenticated request with 8001 *This server must now self-install the initial databases and application servers. Click OK to continue.* Verify response for unauthenticated request with 8002 *Forbidden* @@ -36,12 +64,56 @@ Uninitialized MarkLogic container no parameters Initialized MarkLogic container Create container with -e MARKLOGIC_INIT=true - ... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} - ... -e MARKLOGIC_ADMIN_PASSWORD=${DEFAULT ADMIN PASS} + ... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} + ... -e MARKLOGIC_ADMIN_PASSWORD=${DEFAULT ADMIN PASS} + IF 'rootless' not in '${IMAGE_TYPE}' # ROOT image + Docker log should contain *OVERWRITE_ML_CONF is true, deleting existing /etc/marklogic.conf and overwriting with ENV variables.* + END + IF 'rootless' in '${IMAGE_TYPE}' # ROOTLESS image + Docker log should contain */etc/marklogic.conf will be appended with provided environment variables.* + END Docker log should contain *MARKLOGIC_JOIN_CLUSTER is false or not defined, not joining cluster.* Docker log should contain *MARKLOGIC_INIT is true, initializing the MarkLogic server.* Docker log should contain *Starting container with MarkLogic Server.* Docker log should contain *| server ver: ${MARKLOGIC_VERSION} | scripts ver: ${MARKLOGIC_DOCKER_VERSION} | image type: ${IMAGE_TYPE} | branch: ${BUILD_BRANCH} |* + Docker log should contain *Appended MARKLOGIC_PID_FILE to /etc/marklogic.conf* + Docker log should contain *Appended MARKLOGIC_UMASK to /etc/marklogic.conf* + Docker log should contain *Appended MARKLOGIC_USER to /etc/marklogic.conf* + Docker log should contain *Appended MARKLOGIC_EC2_HOST to /etc/marklogic.conf* + Verify That marklogic.conf contains MARKLOGIC_PID_FILE MARKLOGIC_UMASK MARKLOGIC_USER MARKLOGIC_EC2_HOST=0 + Verify response for unauthenticated request with 8000 *Unauthorized* + Verify response for unauthenticated request with 8001 *Unauthorized* + Verify response for unauthenticated request with 8002 *Unauthorized* + Verify response for authenticated request with 8000 *Query Console* + Verify response for authenticated request with 8001 *No license key has been entered* + Verify response for authenticated request with 8002 *Monitoring Dashboard* + [Teardown] Delete container + +Initialized MarkLogic container with latency + [Tags] long_running + [Documentation] This test verifies the initialization of the MarkLogic container with high latency. + ... Setup on a linux host can be done with the following commands: + ... sudo dnf install kernel-modules-extra + ... sudo modprobe sch_netem + Skip If '${IMAGE_TYPE}' != 'ubi' + Create container with latency -e MARKLOGIC_INIT=true + ... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} + ... -e MARKLOGIC_ADMIN_PASSWORD=${DEFAULT ADMIN PASS} + IF 'rootless' not in '${IMAGE_TYPE}' # ROOT image + Docker log should contain *OVERWRITE_ML_CONF is true, deleting existing /etc/marklogic.conf and overwriting with ENV variables.* + END + IF 'rootless' in '${IMAGE_TYPE}' # ROOTLESS image + Docker log should contain */etc/marklogic.conf will be appended with provided environment variables.* + END + Docker log should contain *MARKLOGIC_JOIN_CLUSTER is false or not defined, not joining cluster.* + Docker log should contain *MARKLOGIC_INIT is true, initializing the MarkLogic server.* + Docker log should contain *Starting container with MarkLogic Server.* + Docker log should contain *| server ver: ${MARKLOGIC_VERSION} | scripts ver: ${MARKLOGIC_DOCKER_VERSION} | image type: ${IMAGE_TYPE} | branch: ${BUILD_BRANCH} |* + Docker log should contain *Appended MARKLOGIC_PID_FILE to /etc/marklogic.conf* + Docker log should contain *Appended MARKLOGIC_UMASK to /etc/marklogic.conf* + Docker log should contain *Appended MARKLOGIC_USER to /etc/marklogic.conf* + Docker log should contain *Appended MARKLOGIC_EC2_HOST to /etc/marklogic.conf* + Verify That marklogic.conf contains MARKLOGIC_PID_FILE MARKLOGIC_UMASK MARKLOGIC_USER MARKLOGIC_EC2_HOST=0 Verify response for unauthenticated request with 8000 *Unauthorized* Verify response for unauthenticated request with 8001 *Unauthorized* Verify response for unauthenticated request with 8002 *Unauthorized* @@ -52,9 +124,10 @@ Initialized MarkLogic container Upgrade MarkLogic container Skip If 'rootless' in '${IMAGE_TYPE}' msg = Skipping Upgrade MarkLogic test for rootless image + Skip If 'arm' in '${IMAGE_TYPE}' msg = Skipping Upgrade MarkLogic test for ARM image Create test container with -e MARKLOGIC_INIT=true - ... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} - ... -e MARKLOGIC_ADMIN_PASSWORD=${DEFAULT ADMIN PASS} +... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} +... -e MARKLOGIC_ADMIN_PASSWORD=${DEFAULT ADMIN PASS} Docker log should contain *MARKLOGIC_JOIN_CLUSTER is false or not defined, not joining cluster.* Docker log should contain *MARKLOGIC_INIT is true, initializing the MarkLogic server.* Docker log should contain *Starting container with MarkLogic Server.* @@ -74,10 +147,62 @@ Upgrade MarkLogic container [Teardown] Run Keywords Delete container True ... AND Delete Volume +Upgrade MarkLogic container with init parameter + Skip If 'rootless' in '${IMAGE_TYPE}' msg = Skipping Upgrade MarkLogic test for rootless image + Skip If 'arm' in '${IMAGE_TYPE}' msg = Skipping Upgrade MarkLogic test for ARM image + Create test container with -e MARKLOGIC_INIT=true +... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} +... -e MARKLOGIC_ADMIN_PASSWORD=${DEFAULT ADMIN PASS} + Docker log should contain *MARKLOGIC_JOIN_CLUSTER is false or not defined, not joining cluster.* + Docker log should contain *MARKLOGIC_INIT is true, initializing the MarkLogic server.* + Docker log should contain *Starting container with MarkLogic Server.* + Docker log should contain *| server ver: ${MARKLOGIC_VERSION} | scripts ver: ${MARKLOGIC_DOCKER_VERSION} | image type: ${IMAGE_TYPE} | branch: ${BUILD_BRANCH} |* + Verify response for unauthenticated request with 8000 *Unauthorized* + Verify response for unauthenticated request with 8001 *Unauthorized* + Verify response for unauthenticated request with 8002 *Unauthorized* + Verify response for authenticated request with 8000 *Query Console* + Verify response for authenticated request with 8001 *No license key has been entered* + Verify response for authenticated request with 8002 *Monitoring Dashboard* + Stop container + Create upgrade container with -e MARKLOGIC_INIT=true + Docker log should contain *Cluster config complete, marking this container as ready.* True + Verify response for authenticated request with 8000 *Query Console* + Verify response for authenticated request with 8001 *No license key has been entered* + Verify response for authenticated request with 8002 *Monitoring Dashboard* + [Teardown] Run Keywords Delete container True + ... AND Delete Volume + +Upgrade MarkLogic container with init and credential parameters + Skip If 'rootless' in '${IMAGE_TYPE}' msg = Skipping Upgrade MarkLogic test for rootless image + Skip If 'arm' in '${IMAGE_TYPE}' msg = Skipping Upgrade MarkLogic test for ARM image + Create test container with -e MARKLOGIC_INIT=true +... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} +... -e MARKLOGIC_ADMIN_PASSWORD=${DEFAULT ADMIN PASS} + Docker log should contain *MARKLOGIC_JOIN_CLUSTER is false or not defined, not joining cluster.* + Docker log should contain *MARKLOGIC_INIT is true, initializing the MarkLogic server.* + Docker log should contain *Starting container with MarkLogic Server.* + Docker log should contain *| server ver: ${MARKLOGIC_VERSION} | scripts ver: ${MARKLOGIC_DOCKER_VERSION} | image type: ${IMAGE_TYPE} | branch: ${BUILD_BRANCH} |* + Verify response for unauthenticated request with 8000 *Unauthorized* + Verify response for unauthenticated request with 8001 *Unauthorized* + Verify response for unauthenticated request with 8002 *Unauthorized* + Verify response for authenticated request with 8000 *Query Console* + Verify response for authenticated request with 8001 *No license key has been entered* + Verify response for authenticated request with 8002 *Monitoring Dashboard* + Stop container + Create upgrade container with -e MARKLOGIC_INIT=true + ... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} + ... -e MARKLOGIC_ADMIN_PASSWORD=${DEFAULT ADMIN PASS} + Docker log should contain *Cluster config complete, marking this container as ready.* True + Verify response for authenticated request with 8000 *Query Console* + Verify response for authenticated request with 8001 *No license key has been entered* + Verify response for authenticated request with 8002 *Monitoring Dashboard* + [Teardown] Run Keywords Delete container True + ... AND Delete Volume + Initialized MarkLogic container with admin password containing special characters Create container with -e MARKLOGIC_INIT=true - ... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} - ... -e MARKLOGIC_ADMIN_PASSWORD=${SPEC CHARS ADMIN PASS} +... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} +... -e MARKLOGIC_ADMIN_PASSWORD=${SPEC CHARS ADMIN PASS} Docker log should contain *MARKLOGIC_JOIN_CLUSTER is false or not defined, not joining cluster.* Docker log should contain *MARKLOGIC_INIT is true, initializing the MarkLogic server.* Docker log should contain *| server ver: ${MARKLOGIC_VERSION} | scripts ver: ${MARKLOGIC_DOCKER_VERSION} | image type: ${IMAGE_TYPE} | branch: ${BUILD_BRANCH} |* @@ -91,10 +216,10 @@ Initialized MarkLogic container with admin password containing special character Initialized MarkLogic container with license key installed and MARKLOGIC_INIT set to TRUE Create container with -e MARKLOGIC_INIT=TRUE - ... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} - ... -e MARKLOGIC_ADMIN_PASSWORD=${DEFAULT ADMIN PASS} - ... -e LICENSEE=${LICENSEE} - ... -e LICENSE_KEY=${LICENSE KEY} + ... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} + ... -e MARKLOGIC_ADMIN_PASSWORD=${DEFAULT ADMIN PASS} + ... -e LICENSEE=${LICENSEE} + ... -e LICENSE_KEY=${LICENSE KEY} Docker log should contain *MARKLOGIC_JOIN_CLUSTER is false or not defined, not joining cluster.* Docker log should contain *MARKLOGIC_INIT is true, initializing the MarkLogic server.* Verify response for unauthenticated request with 8000 *Unauthorized* @@ -114,9 +239,9 @@ Initialized MarkLogic container without credentials Initialized MarkLogic container with invalid value for MARKLOGIC_JOIN_CLUSTER [Tags] negative Create failing container with -e MARKLOGIC_INIT=true - ... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} - ... -e MARKLOGIC_ADMIN_PASSWORD=${DEFAULT ADMIN PASS} - ... -e MARKLOGIC_JOIN_CLUSTER=invalid + ... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} + ... -e MARKLOGIC_ADMIN_PASSWORD=${DEFAULT ADMIN PASS} + ... -e MARKLOGIC_JOIN_CLUSTER=invalid Docker log should contain *MARKLOGIC_INIT is true, initializing the MarkLogic server.* Docker log should contain *Error: MARKLOGIC_JOIN_CLUSTER must be true or false.* [Teardown] Delete container @@ -124,30 +249,69 @@ Initialized MarkLogic container with invalid value for MARKLOGIC_JOIN_CLUSTER Invalid value for INIT [Tags] negative Create failing container with -e MARKLOGIC_INIT=invalid - ... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} - ... -e MARKLOGIC_ADMIN_PASSWORD=${DEFAULT ADMIN PASS} + ... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} + ... -e MARKLOGIC_ADMIN_PASSWORD=${DEFAULT ADMIN PASS} Docker log should contain *Error: MARKLOGIC_INIT must be true or false.* [Teardown] Delete container Invalid value for HOSTNAME [Tags] negative Create failing container with -e HOSTNAME=invalid_hostname - ... -e MARKLOGIC_INIT=true - ... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} - ... -e MARKLOGIC_ADMIN_PASSWORD=${DEFAULT ADMIN PASS} + ... -e MARKLOGIC_INIT=true + ... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} + ... -e MARKLOGIC_ADMIN_PASSWORD=${DEFAULT ADMIN PASS} Docker log should contain *Error: Failed to restart invalid_hostname* [Teardown] Delete container -Initialized MarkLogic container with config overrides +Initialized MarkLogic container without config overrides Create container with -e MARKLOGIC_INIT=true - ... -e OVERWRITE_ML_CONF=true - ... -e TZ=America/Los_Angeles - ... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} - ... -e MARKLOGIC_ADMIN_PASSWORD=${DEFAULT ADMIN PASS} + ... -e OVERWRITE_ML_CONF=false + ... -e TZ=America/Los_Angeles + ... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} + ... -e MARKLOGIC_ADMIN_PASSWORD=${DEFAULT ADMIN PASS} + # ROOT image IF 'rootless' not in '${IMAGE_TYPE}' + Docker log should contain *OVERWRITE_ML_CONF is false, not writing to /etc/marklogic.conf* + Docker log should contain *TZ is defined, setting timezone to America/Los_Angeles.* + Docker log should NOT contain *Appended MARKLOGIC_PID_FILE to /etc/marklogic.conf* + Docker log should NOT contain *Appended MARKLOGIC_UMASK to /etc/marklogic.conf* + Docker log should NOT contain *Appended MARKLOGIC_USER to /etc/marklogic.conf* + Docker log should NOT contain *Appended MARKLOGIC_EC2_HOST to /etc/marklogic.conf* + END + # ROOTLESS image doesn't support OVERWRITE_ML_CONF=false + IF 'rootless' in '${IMAGE_TYPE}' + Docker log should contain */etc/marklogic.conf will be appended with provided environment variables.* + Docker log should contain *Appended MARKLOGIC_PID_FILE to /etc/marklogic.conf* + Docker log should contain *Appended MARKLOGIC_UMASK to /etc/marklogic.conf* + Docker log should contain *Appended MARKLOGIC_USER to /etc/marklogic.conf* + Docker log should contain *Appended MARKLOGIC_EC2_HOST to /etc/marklogic.conf* + Verify That marklogic.conf contains MARKLOGIC_PID_FILE MARKLOGIC_UMASK MARKLOGIC_USER MARKLOGIC_EC2_HOST=0 TZ=America/Los_Angeles + END + Docker log should contain *INSTALL_CONVERTERS is false, not installing converters.* + Docker log should contain *MARKLOGIC_INIT is true, initializing the MarkLogic server.* + Verify response for unauthenticated request with 8000 *Unauthorized* + Verify response for unauthenticated request with 8001 *Unauthorized* + Verify response for unauthenticated request with 8002 *Unauthorized* + Verify response for authenticated request with 8000 *Query Console* + Verify response for authenticated request with 8001 *No license key has been entered* + Verify response for authenticated request with 8002 *Monitoring Dashboard* + Verify container timezone America/Los_Angeles + [Teardown] Delete container + +Initialized MarkLogic container with config overrides + Create container with -e MARKLOGIC_INIT=true + ... -e OVERWRITE_ML_CONF=true + ... -e TZ=America/Los_Angeles + ... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} + ... -e MARKLOGIC_ADMIN_PASSWORD=${DEFAULT ADMIN PASS} + IF 'rootless' not in '${IMAGE_TYPE}' # ROOT image Docker log should contain *OVERWRITE_ML_CONF is true, deleting existing /etc/marklogic.conf and overwriting with ENV variables.* Docker log should contain *TZ is defined, setting timezone to America/Los_Angeles.* END + IF 'rootless' in '${IMAGE_TYPE}' # ROOTLESS image + Docker log should contain */etc/marklogic.conf will be appended with provided environment variables.* + END + Verify That marklogic.conf contains TZ=America/Los_Angeles Docker log should contain *INSTALL_CONVERTERS is false, not installing converters.* Docker log should contain *MARKLOGIC_INIT is true, initializing the MarkLogic server.* Verify response for unauthenticated request with 8000 *Unauthorized* @@ -408,10 +572,10 @@ Two node compose with second node uninitialized Initialized MarkLogic Server with wallet password and realm Create container with -e MARKLOGIC_INIT=true - ... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} - ... -e MARKLOGIC_ADMIN_PASSWORD=${DEFAULT ADMIN PASS} - ... -e MARKLOGIC_WALLET_PASSWORD=test_wallet_pass - ... -e REALM=public + ... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} + ... -e MARKLOGIC_ADMIN_PASSWORD=${DEFAULT ADMIN PASS} + ... -e MARKLOGIC_WALLET_PASSWORD=test_wallet_pass + ... -e REALM=public Verify response for unauthenticated request with 8000 *Unauthorized* Verify response for unauthenticated request with 8001 *Unauthorized* Verify response for unauthenticated request with 8002 *Unauthorized* @@ -421,12 +585,96 @@ Initialized MarkLogic Server with wallet password and realm [Teardown] Delete container Initialized MarkLogic container with ML converters + Skip If 'arm' in '${IMAGE_TYPE}' msg = Skipping ML converters test for ARM image (converters not available) Create container with -e MARKLOGIC_INIT=true - ... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} - ... -e MARKLOGIC_ADMIN_PASSWORD=${DEFAULT ADMIN PASS} - ... -e INSTALL_CONVERTERS=true + ... -e MARKLOGIC_ADMIN_USERNAME=${DEFAULT ADMIN USER} + ... -e MARKLOGIC_ADMIN_PASSWORD=${DEFAULT ADMIN PASS} + ... -e INSTALL_CONVERTERS=true Docker log should contain *INSTALL_CONVERTERS is true, installing converters.* Docker log should contain *MARKLOGIC_INIT is true, initializing the MarkLogic server.* MarkLogic Error log should contain .*Info: MarkLogic Converters.*found Verify converter package installation - [Teardown] Delete container \ No newline at end of file + [Teardown] Delete container + +Dynamic Host Cluster Test + [Tags] dynamic-hosts + ${major_version}= Set Variable ${MARKLOGIC_VERSION.split('.')[0]} + Skip If '${major_version}' == '' or '${major_version}' == 'None' or int('${major_version}' or '0') < 12 msg=Dynamic Host Concurrency Test requires MarkLogic 12 or higher (current version: ${MARKLOGIC_VERSION}) + Start compose from compose-test-16.yaml + # give it some time to prepare the large cluster + Sleep 60s + ${group}= set Variable dynamic + Set up dynamic host group ${group} + Enable API token authentication on 7202 for group Default + Dynamic Host Join Successful on ${group} with 7401 + Dynamic Host Join Failure on dynamic with 7501 with wrong token + Dynamic Host Join Failure on dynamic with 7501 when feature disabled + Dynamic Host Join Failure on ${group} with 7501 when not using the Admin app server + Dynamic Host Remove Successful When Host is down + Dynamic Host Join Successful on ${group} with 7601 + Dynamic Host Remove Successful When All Node is up + Dynamic Host Added When Some Host is down 7701 + Dynamic Host Join Successful on dynamic with 7801 + Dynamic Host Returns All Id dynamic4 + Verify Full Cluster Restart Removes Dynamic Host Configuration dynamic + Enable dynamic host feature on 7102 for group Default + Dynamic Host Join Successful with d-node on Default with 7901 + Disable dynamic host feature on 7102 for group Default + Verify Dynamic Host Count on port 7102 for group Default equals 1 + Enable dynamic host feature on 7102 for group Default + Dynamic Host Join Fails When Token Expires ${group} + Dynamic Host Join Fails After Token Revoked ${group} + Verify Dynamic Host Can Execute Query Default 7902 + [Teardown] Delete compose from compose-test-16.yaml + +Dynamic Host Cluster Concurrecy Join Test + [Tags] dynamic-hosts + ${major_version}= Set Variable ${MARKLOGIC_VERSION.split('.')[0]} + Skip If '${major_version}' == '' or '${major_version}' == 'None' or int('${major_version}' or '0') < 12 msg=Dynamic Host Concurrency Test requires MarkLogic 12 or higher (current version: ${MARKLOGIC_VERSION}) + Start compose from compose-test-16.yaml + # give it some time to prepare the large cluster + Sleep 60s + ${group}= set Variable dynamic + Set up dynamic host group ${group} + Enable API token authentication on 7202 for group Default + Concurrent Dynamic Host Join Test + + [Teardown] Delete compose from compose-test-16.yaml + +Verify parameter overrides + Create container with -e OVERWRITE_ML_CONF=true + ... -e TZ=America/Los_Angeles + ... -e MARKLOGIC_PID_FILE=/tmp/MarkLogic.pid.test + ... -e MARKLOGIC_UMASK=022 + ... -e ML_HUGEPAGES_TOTAL=0 + ... -e MARKLOGIC_DISABLE_JVM=true + ... -e MARKLOGIC_USER=marklogic_user + ... -e JAVA_HOME=fakejava + ... -e CLASSPATH=fakeclasspath + ... -e MARKLOGIC_EC2_HOST=false + + IF 'rootless' not in '${IMAGE_TYPE}' + Docker log should contain *OVERWRITE_ML_CONF is true, deleting existing /etc/marklogic.conf and overwriting with ENV variables.* + Docker log should contain *TZ is defined, setting timezone to America/Los_Angeles.* + END + Verify That marklogic.conf contains TZ=America/Los_Angeles MARKLOGIC_PID_FILE=/tmp/MarkLogic.pid.test MARKLOGIC_UMASK=022 ML_HUGEPAGES_TOTAL=0 MARKLOGIC_DISABLE_JVM=true MARKLOGIC_USER=marklogic_user JAVA_HOME=fakejava CLASSPATH=fakeclasspath MARKLOGIC_EC2_HOST=false + [Teardown] Delete container + +Verify implicit parameter overrides + Create container with -e TZ=America/Los_Angeles + ... -e MARKLOGIC_PID_FILE=/tmp/MarkLogic.pid.test + ... -e MARKLOGIC_UMASK=022 + ... -e ML_HUGEPAGES_TOTAL=0 + ... -e MARKLOGIC_DISABLE_JVM=true + ... -e MARKLOGIC_USER=marklogic_user + ... -e JAVA_HOME=fakejava + ... -e CLASSPATH=fakeclasspath + ... -e MARKLOGIC_EC2_HOST=false + + IF 'rootless' not in '${IMAGE_TYPE}' + Docker log should contain *OVERWRITE_ML_CONF is true, deleting existing /etc/marklogic.conf and overwriting with ENV variables.* + Docker log should contain *TZ is defined, setting timezone to America/Los_Angeles.* + END + Verify That marklogic.conf contains TZ=America/Los_Angeles MARKLOGIC_PID_FILE=/tmp/MarkLogic.pid.test MARKLOGIC_UMASK=022 ML_HUGEPAGES_TOTAL=0 MARKLOGIC_DISABLE_JVM=true MARKLOGIC_USER=marklogic_user JAVA_HOME=fakejava CLASSPATH=fakeclasspath MARKLOGIC_EC2_HOST=false + [Teardown] Delete container + \ No newline at end of file diff --git a/test/keywords.resource b/test/keywords.resource index 2baf6a54..cb23823b 100644 --- a/test/keywords.resource +++ b/test/keywords.resource @@ -1,3 +1,4 @@ +# Copyright © 2018-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. *** Settings *** Library Process Library String @@ -7,20 +8,23 @@ Library Collections Library DateTime *** Variables *** -@{DOCKER DEFAULTS} -it -d -p 8000:8000 -p 8001:8001 -p 8002:8002 -p7997:7997 +${DOCKER_PLATFORM} %{DOCKER_PLATFORM=linux/amd64} +@{DOCKER DEFAULTS} -it -d -p 8000:8000 -p 8001:8001 -p 8002:8002 -p7997:7997 --platform ${DOCKER_PLATFORM} ${DEFAULT ADMIN USER} test_admin ${DEFAULT ADMIN PASS} test_admin_pass ${SPEC CHARS ADMIN PASS} Admin@2$s%^&*! -${TEST_IMAGE} %{DOCKER_TEST_IMAGE=marklogic/marklogic-server-ubi:internal} -${UPGRADE_TEST_IMAGE} marklogic/marklogic-server-ubi:internal -${DOCKER TIMEOUT} 240s +${TEST_IMAGE} %{DOCKER_TEST_IMAGE=progressofficial/marklogic-db:11.3.1-ubi-rootless-2.2.3} +${UPGRADE_TEST_IMAGE} progressofficial/marklogic-db:11.3.1-ubi-rootless-2.2.3 +${DOCKER TIMEOUT} 300s ${LICENSE KEY} %{QA_LICENSE_KEY=none} ${LICENSEE} MarkLogic - Version 9 QA Test License -${MARKLOGIC_VERSION} internal -${BUILD_BRANCH} local -${IMAGE_TYPE} ubi +${MARKLOGIC_VERSION} 11.3.1 +${BUILD_BRANCH} release_2.2.1 +${IMAGE_TYPE} ubi-rootless ${VOL_NAME} MarkLogic_vol_1 ${VOL_INFO} src=${VOL_NAME},dst=/var/opt/MarkLogic +${MARKLOGIC_DOCKER_VERSION} 2.2.3 +${TEST_RESULTS_DIR} test_results *** Keywords *** Create container with @@ -37,6 +41,24 @@ Create container with File Should Be Empty test_results/stderr-${container name}.txt Docker log should contain *Cluster config complete, marking this container as ready.* +Create container with latency + [Arguments] @{input parameters} + [Documentation] Use Docker run to create a single container with defined defaults and input parameters and simulated network latency. + ... Container is named based on test case so that the same name can be used in cleanup. + ... Also verifies that the container is up by checking Docker logs. + ${container name}= Remove spaces from ${TEST NAME} + Run Process docker run @{DOCKER DEFAULTS} @{input parameters} + ... --name ${container name} + ... --cap-add NET_ADMIN --entrypoint /bin/bash + ... ${TEST_IMAGE} + ... -c sudo microdnf -y install iproute iptables && (sudo yum install -y iproute-tc || sudo microdnf install -y iproute) && sudo tc qdisc add dev lo root netem delay 30000ms && sudo tc qdisc show dev lo && /tini -- /usr/local/bin/start-marklogic.sh + ... stderr=test_results/stderr-${container name}.txt + ... stdout=test_results/stdout-${container name}.txt + ... timeout=15000 + File Should Be Empty test_results/stderr-${container name}.txt + Wait Until Keyword Succeeds 5 minutes 10s Get container log message *root refcnt 2 limit 1000 delay 30s* + Wait Until Keyword Succeeds 30 minutes 30s Get container log message *Cluster config complete, marking this container as ready.* + Create failing container with [Arguments] @{input parameters} [Documentation] Creates Docker container without verifying its status. Used for negative tests. @@ -145,16 +167,25 @@ Docker log should contain [Documentation] Wrapper keyword that reties Get container log message, set second argument to True get logs of second container. Wait Until Keyword Succeeds ${DOCKER TIMEOUT} 10s Get container log message ${string pattern} ${second container} -Get container log message +Docker log should NOT contain [Arguments] ${string pattern} ${second container}=False - [Documentation] Get Docker logs for a current image and find a matching string based on input pattern, set second argument to True get logs of second container. + [Documentation] Wrapper keyword that reties Get container log message with inversion, set second argument to True get logs of second container. + Wait Until Keyword Succeeds ${DOCKER TIMEOUT} 10s Get container log message ${string pattern} ${second container} invert=True + +Get container log message + [Arguments] ${string pattern} ${second container}=False ${invert}=False + [Documentation] Get Docker logs for a current image and find a matching string based on input pattern, set second argument to True get logs of second container. If invert is True, asserts the pattern is NOT present. ${container name}= Remove spaces from ${TEST NAME} IF ${second container} == True ${container name}= Set Variable ${container name}-2 END Run Process docker logs ${container name} stdout=test_results/stdout-${container name}.txt timeout=${DOCKER TIMEOUT} ${docker log}= Get File test_results/stdout-${container name}.txt - Should Match ${docker log} ${string pattern} msg=Did not find ${string pattern} in ${container name} container log + IF ${invert} + Should Not Match ${docker log} ${string pattern} msg=Found unexpected pattern '${string pattern}' in ${container name} container log + ELSE + Should Match ${docker log} ${string pattern} msg=Did not find ${string pattern} in ${container name} container log + END MarkLogic Error log should contain [Arguments] ${string pattern} @@ -243,8 +274,6 @@ Check host count on port ${port} should be ${count} ${response}= GET On Session RestSession url=http://localhost:${port}/manage/v2/hosts?view=status&format=json ${host count}= Set Variable ${response.json()['host-status-list']['status-list-summary']['total-hosts']['value']} Should Be Equal ${host count} ${count} - #cluster hosts: - #${response.json()['host-status-list']['status-list-items']['status-list-item']} Host ${hostname} should be part of group ${group} [Documentation] Wrapper keyword that retries Check host ${hostname} should be in group ${group} @@ -333,3 +362,300 @@ Verify container timezone ${offset}= Get Regexp Matches ${response.text} -?PT[HM0-9]+ ${expected}= Convert timezone to MarkLogic offset ${timezone} Should Be Equal ${expected} ${offset[0]} + +Create dynamic host token for group ${group} on host ${host} and port ${port} using docker port ${dockerport} with duration ${duration} and comment ${comment} + [Documentation] This generate a dynamic host token that can be used for join purpose. + ${auth} ${headers}= Generate digest authorization for ${DEFAULT ADMIN USER} ${DEFAULT ADMIN PASS} + ${header}= Create Dictionary Content-type=application/json + ${port_num}= Convert To Integer ${port} + ${token_details}= Create Dictionary group=${group} host=${host} port=${port_num} duration=${duration} comment=${comment} + ${token_body}= Create Dictionary dynamic-host-token=${token_details} + ${response}= Create Digest Session RestSession url=http://localhost:${dockerport} headers=${headers} auth=${auth} + ${response}= POST On Session RestSession url=/manage/v2/clusters/Default/dynamic-host-token json=${token_body} headers=${header} + ${token_xml}= Set Variable ${response.text} + ${token_content}= Get Regexp Matches ${token_xml} ]*>(.*) 1 + ${token}= Set Variable ${token_content[0]} + RETURN ${token} + +Enable dynamic host feature on ${dockerport} for group ${group} + [Documentation] This enables the dynamic host feature for the specified group. + ${auth} ${headers}= Generate digest authorization for ${DEFAULT ADMIN USER} ${DEFAULT ADMIN PASS} + ${header}= Create Dictionary Content-type=application/json Accept=application/json + # Create JSON payload to enable dynamic hosts + ${json_body}= Create Dictionary group-name=${group} allow-dynamic-hosts=${TRUE} + ${response}= Create Digest Session RestSession url=http://localhost:${dockerport} headers=${headers} auth=${auth} + ${response}= PUT On Session RestSession url=/manage/v2/groups/${group}/properties?format=json json=${json_body} headers=${header} + RETURN ${response} + +Enable API token authentication on ${dockerport} for group ${group} + [Documentation] This enables the API token authentication for the specified group + ${auth} ${headers}= Generate digest authorization for ${DEFAULT ADMIN USER} ${DEFAULT ADMIN PASS} + ${header}= Create Dictionary Content-type=application/json Accept=application/json + ${json_body}= Create Dictionary API-token-authentication=${TRUE} + ${response}= Create Digest Session RestSession url=http://localhost:${dockerport} headers=${headers} auth=${auth} + ${response}= PUT On Session RestSession url=/manage/v2/servers/Admin/properties?group-id=${group} json=${json_body} headers=${header} + RETURN ${response} + +Disable dynamic host feature on ${dockerport} for group ${group} + [Documentation] This enables the dynamic host feature for the specified group. + ${auth} ${headers}= Generate digest authorization for ${DEFAULT ADMIN USER} ${DEFAULT ADMIN PASS} + ${header}= Create Dictionary Content-type=application/json Accept=application/json + # Create JSON payload to enable dynamic hosts + ${json_body}= Create Dictionary group-name=${group} allow-dynamic-hosts=${FALSE} + ${response}= Create Digest Session RestSession url=http://localhost:${dockerport} headers=${headers} auth=${auth} + ${response}= PUT On Session RestSession url=/manage/v2/groups/${group}/properties?format=json json=${json_body} headers=${header} + RETURN ${response} + +Init dynamic host ${dockerport} with token ${token} + [Documentation] This initializes the dynamic host with the specified token (no auth needed) + ${header}= Create Dictionary Content-type=application/xml + # Create XML payload with the token + ${xml_body}= Set Variable ${token} + + # Use standard session without authentication + Create Session RestSession url=http://localhost:${dockerport} + ${response}= POST On Session RestSession url=/admin/v1/init data=${xml_body} headers=${header} + +Create group ${group} on port ${port} + [Documentation] Creates a new group in MarkLogic using the Management API + ${auth} ${headers}= Generate digest authorization for ${DEFAULT ADMIN USER} ${DEFAULT ADMIN PASS} + ${header}= Create Dictionary Content-type=application/json Accept=application/json + ${json_body}= Create Dictionary group-name=${group} + ${response}= Create Digest Session RestSession url=http://localhost:${port} headers=${headers} auth=${auth} + ${response}= POST On Session RestSession url=/manage/v2/groups json=${json_body} headers=${header} + +Set up dynamic host group ${group} + Create group ${group} on port 7102 + Enable dynamic host feature on 7102 for group ${group} + +Get MarkLogic Host ID on port ${port} with hostname ${hostname} + [Documentation] Returns the host ID of a MarkLogic server with the specified hostname + ${auth} ${headers}= Generate digest authorization for ${DEFAULT ADMIN USER} ${DEFAULT ADMIN PASS} + ${header}= Create Dictionary Accept=application/json + ${response}= Create Digest Session RestSession url=http://localhost:${port} headers=${headers} auth=${auth} + ${response}= GET On Session RestSession url=/manage/v2/hosts?format=json headers=${header} + ${hosts}= Set Variable ${response.json()["host-default-list"]["list-items"]["list-item"]} + ${host_id}= Set Variable ${EMPTY} + FOR ${host} IN @{hosts} + IF '${host["nameref"]}' == '${hostname}' + ${host_id}= Set Variable ${host["idref"]} + Exit For Loop + END + END + + Should Not Be Empty ${host_id} Host ${hostname} not found in the host list + RETURN ${host_id} + +Get All Dynamic Host IDs on port ${port} for group ${group} + [Documentation] Gets all dynamic host IDs from the MarkLogic cluster + ${auth} ${headers}= Generate digest authorization for ${DEFAULT ADMIN USER} ${DEFAULT ADMIN PASS} + ${header}= Create Dictionary Accept=application/json + ${response}= Create Digest Session RestSession url=http://localhost:${port} headers=${headers} auth=${auth} + ${response}= GET On Session RestSession url=/manage/v2/clusters/${group}/dynamic-hosts?format=json headers=${header} + ${has_hosts}= Run Keyword And Return Status Dictionary Should Contain Key ${response.json()} dynamic-hosts + @{host_ids}= Create List + IF ${has_hosts} + ${hosts}= Set Variable ${response.json()["dynamic-hosts"]} + ${host_ids}= Set Variable ${hosts} + END + RETURN ${host_ids} + +Remove All Dynamic Hosts on port ${port} for group ${group} + [Documentation] Removes all dynamic hosts from the MarkLogic cluster for a specific group + @{host_ids}= Get All Dynamic Host IDs on port ${port} for group ${group} + ${host_count}= Get Length ${host_ids} + IF ${host_count} > 0 + ${auth} ${headers}= Generate digest authorization for ${DEFAULT ADMIN USER} ${DEFAULT ADMIN PASS} + ${header}= Create Dictionary Accept=application/json + # Create session and delete hosts one by one using URL parameters + ${response}= Create Digest Session RestSession url=http://localhost:${port} headers=${headers} auth=${auth} + FOR ${host_id} IN @{host_ids} + ${response}= DELETE On Session RestSession + ... url=/manage/v2/clusters/${group}/dynamic-hosts/${host_id} + ... headers=${header} + END + Log Removed ${host_count} dynamic hosts from group ${group} + ELSE + Log No dynamic hosts to remove from group ${group} + END + +Dynamic Host Join Successful on ${group} with ${port} + ${token}= Create dynamic host token for group ${group} on host bootstrap_3n and port 8001 using docker port 7102 with duration PT10M and comment no + Init dynamic host ${port} with token ${token} + +Dynamic Host Join Successful with d-node on ${group} with ${port} + ${token}= Create dynamic host token for group ${group} on host node2 and port 8001 using docker port 7202 with duration PT10M and comment no + Init dynamic host ${port} with token ${token} + Sleep 5s + +Dynamic Host Join Failure on ${group} with ${port} with wrong token + [Documentation] Tests that joining fails when token is corrupted + ${token}= Create dynamic host token for group ${group} on host bootstrap_3n and port 8001 using docker port 7102 with duration PT10M and comment no + ${corrupted_token}= Set Variable ${token}XYZ123InvalidTokenSuffix + ${status}= Run Keyword And Return Status Init dynamic host ${port} with token ${corrupted_token} + Should Be Equal ${status} ${FALSE} Join with corrupted token should have failed but succeeded + +Dynamic Host Join Failure on ${group} with ${port} when feature disabled + Disable dynamic host feature on 7102 for group ${group} + ${token}= Create dynamic host token for group ${group} on host bootstrap_3n and port 8001 using docker port 7102 with duration PT10M and comment no + ${status}= Run Keyword And Return Status Init dynamic host ${port} with token ${token} + Should Be Equal ${status} ${FALSE} Join should fail when this feature is disabled + Enable dynamic host feature on 7102 for group ${group} + +Dynamic Host Join Failure on ${group} with ${port} when not using the Admin app server + ${token}= Create dynamic host token for group ${group} on host bootstrap_3n and port 8002 using docker port 7102 with duration PT10M and comment no + ${status}= Run Keyword And Return Status Init dynamic host ${port} with token ${token} + Should Be Equal ${status} ${FALSE} Join should fail when this feature is disabled + +Wait Until Container Is Stopped + Sleep 50s + +Wait Until Container Is Running + Sleep 10s + +Dynamic Host Remove Successful When Host is down + Run Process docker kill node2 + Wait Until Container Is Stopped + Remove All Dynamic Hosts on port 7102 for group dynamic + Run Process docker start node2 + Wait Until Container Is Running + @{host_ids}= Get All Dynamic Host IDs on port 7202 for group dynamic + Should Be Empty ${host_ids} Dynamic hosts were found when none were expected + +Dynamic Host Remove Successful When All Node is up + # sleep to make sure the host joins successfully + Sleep 5s + Remove All Dynamic Hosts on port 7102 for group dynamic + @{host_ids}= Get All Dynamic Host IDs on port 7102 for group dynamic + Should Be Empty ${host_ids} Dynamic hosts were found when none were expected + +Dynamic Host Added When Some Host is down ${port} + Run Process docker kill node2 + Wait Until Container Is Stopped + Dynamic Host Join Successful on dynamic with ${port} + Run Process docker start node2 + Wait Until Container Is Running + @{host_ids}= Get All Dynamic Host IDs on port 7202 for group dynamic + Should Not Be Empty ${host_ids} No dynamic hosts were found when some were expected + +Dynamic Host Returns All Id ${container_name} + Run Process docker kill ${container_name} + Wait Until Container Is Stopped + @{host_ids}= Get All Dynamic Host IDs on port 7202 for group dynamic + ${host_count}= Get Length ${host_ids} + Should Be Equal As Integers ${host_count} 2 Expected exactly 2 dynamic hosts but found ${host_count} + Run Process docker start ${container_name} + Wait Until Container Is Running + +Verify Full Cluster Restart Removes Dynamic Host Configuration ${group} + [Documentation] Tests that restarting the entire cluster removes all dynamic host information + + @{host_ids}= Get All Dynamic Host IDs on port 7102 for group dynamic + ${host_count}= Get Length ${host_ids} + Should Be Equal As Integers ${host_count} 2 Expected 2 dynamic hosts but found ${host_count} + + # Restart the entire cluster + Restart compose from compose-test-16.yaml + Sleep 30s + + # Verify dynamic host information is removed + @{host_ids_after_restart}= Get All Dynamic Host IDs on port 7102 for group dynamic + Should Be Empty ${host_ids_after_restart} Dynamic hosts still exist after full cluster restart + +Verify Dynamic Host Count on port ${port} for group ${group} equals ${expected_count} + [Documentation] Verifies that the number of dynamic hosts matches the expected count + @{host_ids}= Get All Dynamic Host IDs on port ${port} for group ${group} + ${actual_count}= Get Length ${host_ids} + ${expected_count}= Convert To Integer ${expected_count} + Should Be Equal As Integers ${actual_count} ${expected_count} + ... Expected ${expected_count} dynamic hosts but found ${actual_count} + + IF ${actual_count} > 0 + Log Found dynamic hosts: @{host_ids} + END + RETURN ${host_ids} + +Dynamic Host Join Fails When Token Expires ${group} + [Documentation] Tests that a token cannot be reused after it expires + + ${token}= Create dynamic host token for group ${group} on host bootstrap_3n and port 8001 using docker port 7102 with duration PT5S and comment "short-lived token" + Init dynamic host 8001 with token ${token} + Sleep 10s # Wait longer than the PT5S token duration + ${status}= Run Keyword And Return Status Init dynamic host 8101 with token ${token} + Should Be Equal ${status} ${FALSE} Join succeeded with expired token when it should have failed + Verify Dynamic Host Count on port 7102 for group ${group} equals 1 + +Dynamic Host Join Fails After Token Revoked ${group} + [Documentation] Tests that authentication fails after the token is revoked by the admin. + + ${token}= Create dynamic host token for group ${group} on host bootstrap_3n and port 8001 using docker port 7102 with duration PT10M and comment "revoked token test" + + ${auth} ${headers}= Generate digest authorization for ${DEFAULT ADMIN USER} ${DEFAULT ADMIN PASS} + ${header}= Create Dictionary Content-Type=application/xml + ${xml_body}= Set Variable ${token} + ${response}= Create Digest Session RestSession url=http://localhost:8002 headers=${headers} auth=${auth} + ${response}= DELETE On Session RestSession url=/manage/v2/clusters/${group}/dynamic-host-token data=${xml_body} headers=${header} + + ${status}= Run Keyword And Return Status Init dynamic host 8201 with token ${token} + Should Be Equal ${status} ${FALSE} Join succeeded with revoked token when it should have failed + +Verify Dynamic Host Can Execute Query ${group} ${port} + [Documentation] Verifies that a dynamic host can execute a query using the REST API. + + # Execute a query to verify the dynamic host is functional + ${auth} ${headers}= Generate digest authorization for ${DEFAULT ADMIN USER} ${DEFAULT ADMIN PASS} + ${header}= Create Dictionary Content-type=application/x-www-form-urlencoded + ${query}= Set Variable xquery=xdmp:host-name(xdmp:host()) + ${response}= Create Digest Session RestSession url=http://localhost:${port} headers=${headers} auth=${auth} + ${response}= POST On Session RestSession url=/v1/eval data=${query} headers=${header} + + # Extract the host name from the response body + ${host_name}= Get Regexp Matches ${response.text} X-Primitive: string\r?\n\r?\n([^\r\n]+) 1 + Should Not Be Empty ${host_name} Query execution failed or returned an empty result + Log Dynamic host executed query successfully: ${host_name[0]} + +Concurrent Dynamic Host Join Test + [Documentation] Tests joining multiple dynamic hosts concurrently and verifies all hosts were added + [Arguments] ${group}=dynamic ${start_port}=7401 ${end_port}=8401 + + + # Create a list to store all tokens for concurrent joining + ${token}= Create dynamic host token for group ${group} on host bootstrap_3n and port 8001 using docker port 7102 with duration PT10M and comment "concurrent-test" + + FOR ${port} IN RANGE ${start_port} ${end_port}+100 100 + Init dynamic host ${port} with token ${token} + END + + # Give some time for all operations to complete + Sleep 10s + + # Verify the hosts were added - count how many were successful + ${hosts}= Get All Dynamic Host IDs on port 7102 for group ${group} + ${host_count}= Get Length ${hosts} + ${expected_count}= Evaluate (${end_port} - ${start_port}) / 100 + 1 + ${expected_count}= Convert To Integer ${expected_count} + + # Log the details + Log Found ${host_count} dynamic hosts from concurrent join + Log Expected approximately ${expected_count} dynamic hosts + + Should Be True ${host_count} == ${expected_count} + +Ensure Test Results Directory Exists + [Documentation] Creates 'test_results' directory if needed. + OperatingSystem.Create Directory ${TEST_RESULTS_DIR} + +Verify That marklogic.conf contains + [Arguments] @{variables} + [Documentation] Checks if a list of variables exists in /etc/marklogic.conf inside the container. + ${container name}= Remove spaces from ${TEST_NAME} + FOR ${variable} IN @{variables} + IF 'rootless' not in '${IMAGE_TYPE}' + ${output}= Run Process docker exec ${container name} sudo grep ${variable} /etc/marklogic.conf + ELSE + ${output}= Run Process docker exec ${container name} grep ${variable} /etc/marklogic.conf + END + Log STDERR: ${output.stderr} + Log STDOUT: ${output.stdout} + Should Not Be Empty ${output.stdout} Variable ${variable} not found in /etc/marklogic.conf + END From c492f5f9e5d73bc398105617c71b403b5dda3984 Mon Sep 17 00:00:00 2001 From: Vitaly Korolev Date: Fri, 27 Feb 2026 10:44:39 -0800 Subject: [PATCH 2/2] Increment Docker release version --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6560005d..35c6f651 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -537,7 +537,7 @@ pipeline { parameters { string(name: 'emailList', defaultValue: emailList, description: 'List of email for build notification', trim: true) - string(name: 'dockerVersion', defaultValue: '2.2.3', description: 'ML Docker version. This version along with ML rpm package version will be the image tag as {ML_Version}_{dockerVersion}', trim: true) + string(name: 'dockerVersion', defaultValue: '2.2.4', description: 'ML Docker version. This version along with ML rpm package version will be the image tag as {ML_Version}_{dockerVersion}', trim: true) choice(name: 'dockerImageType', choices: 'ubi-rootless\nubi\nubi9-rootless\nubi9\nubi9-arm\nubi9-rootless-arm', description: 'Platform type for Docker image. Will be made part of the docker image tag') string(name: 'upgradeDockerImage', defaultValue: '', description: 'Docker image for testing upgrades. Defaults to ubi image if left blank.\n Currently upgrading to ubi-rotless is not supported hence the test is skipped when ubi-rootless image is provided.', trim: true) choice(name: 'marklogicVersion', choices: '12\n11\n10', description: 'MarkLogic Server Branch. used to pick appropriate rpm')