From 22e4584d0e4f465ac8b6550adcf1e7d30bba18c1 Mon Sep 17 00:00:00 2001 From: Joey Zhao <5253430+joeyzhao2018@users.noreply.github.com> Date: Mon, 2 Mar 2026 11:13:11 -0500 Subject: [PATCH 1/3] enable build_layers.sh to build from specific dd-trace-py commit --- Dockerfile | 6 ++++ scripts/build_layers.sh | 69 ++++++++++++++++++++++++++++++++++------- 2 files changed, 63 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index 974419ed..ec4f29ba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,6 +25,12 @@ ENV PATH=/root/.cargo/bin:$PATH # Install datadog_lambda and dependencies from local COPY . . + +# Constrain setuptools<78 for build isolation environments so that +# pkg_resources is available when building dd-trace-py from source. +RUN echo 'setuptools<78' > /tmp/build-constraints.txt +ENV PIP_CONSTRAINT=/tmp/build-constraints.txt + RUN pip install --no-cache-dir . -t ./python/lib/$runtime/site-packages # Remove botocore (40MB) to reduce package size. aws-xray-sdk diff --git a/scripts/build_layers.sh b/scripts/build_layers.sh index f10b9dbf..c01c788e 100755 --- a/scripts/build_layers.sh +++ b/scripts/build_layers.sh @@ -6,9 +6,27 @@ # Copyright 2019 Datadog, Inc. # Builds datadog-lambda-python layers for Lambda functions - -# Usage: PYTHON_VERSION=3.11 ./build_layers.sh -# If PYTHON_VERSION is not specified, all versions will be built. +# +# Usage: +# PYTHON_VERSION=12 ARCH=arm ./scripts/build_layers.sh +# +# Environment variables: +# PYTHON_VERSION Python minor version. Accepts shorthand (e.g. 12) or full (e.g. 3.12). +# If not set, all supported versions are built. +# ARCH Target architecture. Accepts shorthand: arm, amd, x86, aarch64 +# or full: arm64, amd64. If not set, both are built. +# +# dd-trace-py overrides (mutually exclusive, highest priority first): +# DD_TRACE_COMMIT Specific dd-trace-py commit SHA to build from GitHub. +# DD_TRACE_COMMIT_BRANCH dd-trace-py branch name to build from GitHub. +# DD_TRACE_WHEEL Path to a pre-built ddtrace .whl file. +# +# Examples: +# # Build a single layer for Python 3.12 on arm64 +# PYTHON_VERSION=12 ARCH=arm ./scripts/build_layers.sh +# +# # Build with a specific dd-trace-py commit (for git bisect) +# DD_TRACE_COMMIT=abc123 PYTHON_VERSION=12 ARCH=arm ./scripts/build_layers.sh set -e @@ -17,6 +35,13 @@ LAYER_FILES_PREFIX="datadog_lambda_py" AVAILABLE_PYTHON_VERSIONS=("3.8" "3.9" "3.10" "3.11" "3.12" "3.13" "3.14") AVAILABLE_ARCHS=("arm64" "amd64") +if [ -n "$ARCH" ]; then + case "$ARCH" in + arm|arm64|aarch64) ARCH="arm64" ;; + amd|amd64|x86|x86_64) ARCH="amd64" ;; + esac +fi + if [ -z "$ARCH" ]; then echo "No architectures specified, building layers for all architectures." ARCHS=("${AVAILABLE_ARCHS[@]}") @@ -28,7 +53,14 @@ else echo "EXITING SCRIPT." exit 1 fi - ARCHS=$ARCH + ARCHS=("$ARCH") +fi + +# Normalize Python version shorthand (e.g. 12 -> 3.12, 3.12 -> 3.12) +if [ -n "$PYTHON_VERSION" ]; then + if [[ "$PYTHON_VERSION" =~ ^[0-9]+$ ]]; then + PYTHON_VERSION="3.${PYTHON_VERSION}" + fi fi # Determine which Python versions to build layers for @@ -43,16 +75,29 @@ else echo "EXITING SCRIPT." exit 1 fi - PYTHON_VERSIONS=$PYTHON_VERSION + PYTHON_VERSIONS=("$PYTHON_VERSION") fi -# Replace ddtrace wheel used if necessary -if [ -n "$DD_TRACE_COMMIT_BRANCH" ]; then - sed -z -E -i 's|(ddtrace = )\[[^]]*]|\1{ git = "https://github.com/DataDog/dd-trace-py.git", branch = \"'"$DD_TRACE_COMMIT_BRANCH"'\" }|g' pyproject.toml -else - if [ -n "$DD_TRACE_WHEEL" ]; then - sed -z -E -i 's|(ddtrace = )\[[^]]*]|\1{ file = "'"$DD_TRACE_WHEEL"'" }|g' pyproject.toml - fi +# Backup pyproject.toml so modifications don't persist across runs +cp pyproject.toml pyproject.toml.bak +cleanup() { + mv pyproject.toml.bak pyproject.toml 2>/dev/null || true +} +trap cleanup EXIT + +# Helper: replace the multi-line ddtrace dependency in pyproject.toml. +# Uses perl instead of sed -z for macOS/Linux portability. +replace_ddtrace_dep() { + perl -i -0777 -pe "s|ddtrace = \[[^\]]*\]|$1|gs" pyproject.toml +} + +# Replace ddtrace source if necessary +if [ -n "$DD_TRACE_COMMIT" ]; then + replace_ddtrace_dep "ddtrace = { git = \"https://github.com/DataDog/dd-trace-py.git\", rev = \"$DD_TRACE_COMMIT\" }" +elif [ -n "$DD_TRACE_COMMIT_BRANCH" ]; then + replace_ddtrace_dep "ddtrace = { git = \"https://github.com/DataDog/dd-trace-py.git\", branch = \"$DD_TRACE_COMMIT_BRANCH\" }" +elif [ -n "$DD_TRACE_WHEEL" ]; then + replace_ddtrace_dep "ddtrace = { file = \"$DD_TRACE_WHEEL\" }" fi function make_path_absolute { From ef104fb6086a5dd3ca40e801e77fd50b436aa09a Mon Sep 17 00:00:00 2001 From: Joey Zhao <5253430+joeyzhao2018@users.noreply.github.com> Date: Wed, 4 Mar 2026 11:54:00 -0500 Subject: [PATCH 2/3] make PIN_SETUPTOOLS optional --- Dockerfile | 12 +++++++++--- scripts/build_layers.sh | 10 ++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index ec4f29ba..8d71e667 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,9 +26,15 @@ ENV PATH=/root/.cargo/bin:$PATH # Install datadog_lambda and dependencies from local COPY . . -# Constrain setuptools<78 for build isolation environments so that -# pkg_resources is available when building dd-trace-py from source. -RUN echo 'setuptools<78' > /tmp/build-constraints.txt +# When building dd-trace-py from source (older commits), setuptools<78 is needed +# in pip's build isolation environment so that pkg_resources is available. +# An empty constraints file is a no-op for pip. +ARG pin_setuptools="" +RUN if [ -n "$pin_setuptools" ]; then \ + echo 'setuptools<78' > /tmp/build-constraints.txt; \ + else \ + touch /tmp/build-constraints.txt; \ + fi ENV PIP_CONSTRAINT=/tmp/build-constraints.txt RUN pip install --no-cache-dir . -t ./python/lib/$runtime/site-packages diff --git a/scripts/build_layers.sh b/scripts/build_layers.sh index c01c788e..cacca392 100755 --- a/scripts/build_layers.sh +++ b/scripts/build_layers.sh @@ -20,6 +20,8 @@ # DD_TRACE_COMMIT Specific dd-trace-py commit SHA to build from GitHub. # DD_TRACE_COMMIT_BRANCH dd-trace-py branch name to build from GitHub. # DD_TRACE_WHEEL Path to a pre-built ddtrace .whl file. +# PIN_SETUPTOOLS Set to "true" to constrain setuptools<78 in Docker. +# Only needed for old dd-trace-py commits that import pkg_resources. # # Examples: # # Build a single layer for Python 3.12 on arm64 @@ -92,6 +94,7 @@ replace_ddtrace_dep() { } # Replace ddtrace source if necessary +DOCKER_BUILD_ARGS="" if [ -n "$DD_TRACE_COMMIT" ]; then replace_ddtrace_dep "ddtrace = { git = \"https://github.com/DataDog/dd-trace-py.git\", rev = \"$DD_TRACE_COMMIT\" }" elif [ -n "$DD_TRACE_COMMIT_BRANCH" ]; then @@ -100,6 +103,12 @@ elif [ -n "$DD_TRACE_WHEEL" ]; then replace_ddtrace_dep "ddtrace = { file = \"$DD_TRACE_WHEEL\" }" fi +# Older dd-trace-py commits import pkg_resources, which was removed in setuptools>=78. +# Set PIN_SETUPTOOLS=true to constrain setuptools<78 in pip's build isolation. +if [ -n "$PIN_SETUPTOOLS" ]; then + DOCKER_BUILD_ARGS="--build-arg pin_setuptools=true" +fi + function make_path_absolute { echo "$(cd "$(dirname "$1")"; pwd)/$(basename "$1")" } @@ -118,6 +127,7 @@ function docker_build_zip { --build-arg runtime=python$1 \ --platform linux/${arch} \ --progress=plain \ + $DOCKER_BUILD_ARGS \ -o $temp_dir/python # Zip to destination, and keep directory structure as based in $temp_dir From f028e8ee66e3ea3e920f47f197c7b677e50537fe Mon Sep 17 00:00:00 2001 From: Joey Zhao <5253430+joeyzhao2018@users.noreply.github.com> Date: Wed, 4 Mar 2026 13:34:48 -0500 Subject: [PATCH 3/3] remove the pin_setuptools option entirely --- Dockerfile | 11 ----------- scripts/build_layers.sh | 11 ----------- 2 files changed, 22 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8d71e667..b64cd419 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,17 +26,6 @@ ENV PATH=/root/.cargo/bin:$PATH # Install datadog_lambda and dependencies from local COPY . . -# When building dd-trace-py from source (older commits), setuptools<78 is needed -# in pip's build isolation environment so that pkg_resources is available. -# An empty constraints file is a no-op for pip. -ARG pin_setuptools="" -RUN if [ -n "$pin_setuptools" ]; then \ - echo 'setuptools<78' > /tmp/build-constraints.txt; \ - else \ - touch /tmp/build-constraints.txt; \ - fi -ENV PIP_CONSTRAINT=/tmp/build-constraints.txt - RUN pip install --no-cache-dir . -t ./python/lib/$runtime/site-packages # Remove botocore (40MB) to reduce package size. aws-xray-sdk diff --git a/scripts/build_layers.sh b/scripts/build_layers.sh index cacca392..f1500275 100755 --- a/scripts/build_layers.sh +++ b/scripts/build_layers.sh @@ -20,8 +20,6 @@ # DD_TRACE_COMMIT Specific dd-trace-py commit SHA to build from GitHub. # DD_TRACE_COMMIT_BRANCH dd-trace-py branch name to build from GitHub. # DD_TRACE_WHEEL Path to a pre-built ddtrace .whl file. -# PIN_SETUPTOOLS Set to "true" to constrain setuptools<78 in Docker. -# Only needed for old dd-trace-py commits that import pkg_resources. # # Examples: # # Build a single layer for Python 3.12 on arm64 @@ -94,7 +92,6 @@ replace_ddtrace_dep() { } # Replace ddtrace source if necessary -DOCKER_BUILD_ARGS="" if [ -n "$DD_TRACE_COMMIT" ]; then replace_ddtrace_dep "ddtrace = { git = \"https://github.com/DataDog/dd-trace-py.git\", rev = \"$DD_TRACE_COMMIT\" }" elif [ -n "$DD_TRACE_COMMIT_BRANCH" ]; then @@ -102,13 +99,6 @@ elif [ -n "$DD_TRACE_COMMIT_BRANCH" ]; then elif [ -n "$DD_TRACE_WHEEL" ]; then replace_ddtrace_dep "ddtrace = { file = \"$DD_TRACE_WHEEL\" }" fi - -# Older dd-trace-py commits import pkg_resources, which was removed in setuptools>=78. -# Set PIN_SETUPTOOLS=true to constrain setuptools<78 in pip's build isolation. -if [ -n "$PIN_SETUPTOOLS" ]; then - DOCKER_BUILD_ARGS="--build-arg pin_setuptools=true" -fi - function make_path_absolute { echo "$(cd "$(dirname "$1")"; pwd)/$(basename "$1")" } @@ -127,7 +117,6 @@ function docker_build_zip { --build-arg runtime=python$1 \ --platform linux/${arch} \ --progress=plain \ - $DOCKER_BUILD_ARGS \ -o $temp_dir/python # Zip to destination, and keep directory structure as based in $temp_dir