diff --git a/sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/HttpJsonCallOptions.java b/sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/HttpJsonCallOptions.java index 4c2d8ae55e19..1f0022bb4390 100644 --- a/sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/HttpJsonCallOptions.java +++ b/sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/HttpJsonCallOptions.java @@ -35,6 +35,7 @@ import static com.google.api.gax.util.TimeConversionUtils.toThreetenInstant; import com.google.api.core.ObsoleteApi; +import com.google.api.gax.tracing.ApiTracer; import com.google.auth.Credentials; import com.google.auto.value.AutoValue; import com.google.protobuf.TypeRegistry; @@ -71,6 +72,9 @@ public final org.threeten.bp.Instant getDeadline() { @Nullable public abstract TypeRegistry getTypeRegistry(); + @Nullable + public abstract ApiTracer getTracer(); + public abstract Builder toBuilder(); public static Builder newBuilder() { @@ -106,6 +110,11 @@ public HttpJsonCallOptions merge(HttpJsonCallOptions inputOptions) { builder.setTypeRegistry(newTypeRegistry); } + ApiTracer newTracer = inputOptions.getTracer(); + if (newTracer != null) { + builder.setTracer(newTracer); + } + return builder.build(); } @@ -131,6 +140,8 @@ public final Builder setDeadline(org.threeten.bp.Instant value) { public abstract Builder setTypeRegistry(TypeRegistry value); + public abstract Builder setTracer(ApiTracer value); + public abstract HttpJsonCallOptions build(); } } diff --git a/sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/HttpJsonClientCalls.java b/sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/HttpJsonClientCalls.java index c98d00afb961..65b745f27bc1 100644 --- a/sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/HttpJsonClientCalls.java +++ b/sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/HttpJsonClientCalls.java @@ -76,8 +76,13 @@ public static HttpJsonClientCall newC // if the Universe Domain is valid. httpJsonContext.validateUniverseDomain(); + HttpJsonCallOptions callOptions = httpJsonContext.getCallOptions(); + if (httpJsonContext.getTracer() != null) { + callOptions = callOptions.toBuilder().setTracer(httpJsonContext.getTracer()).build(); + } + // TODO: add headers interceptor logic - return httpJsonContext.getChannel().newCall(methodDescriptor, httpJsonContext.getCallOptions()); + return httpJsonContext.getChannel().newCall(methodDescriptor, callOptions); } static ApiFuture futureUnaryCall( diff --git a/sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/HttpRequestRunnable.java b/sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/HttpRequestRunnable.java index 2738844bd0f4..b3b5b2612d18 100644 --- a/sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/HttpRequestRunnable.java +++ b/sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/HttpRequestRunnable.java @@ -44,6 +44,7 @@ import com.google.api.client.json.JsonObjectParser; import com.google.api.client.json.gson.GsonFactory; import com.google.api.client.util.GenericData; +import com.google.api.gax.tracing.ApiTracer; import com.google.auth.Credentials; import com.google.auth.http.HttpCredentialsAdapter; import com.google.auto.value.AutoValue; @@ -188,6 +189,11 @@ HttpRequest createHttpRequest() throws IOException { } } + ApiTracer tracer = httpJsonCallOptions.getTracer(); + if (tracer != null) { + tracer.requestUrlResolved(url.build()); + } + HttpRequest httpRequest = buildRequest(requestFactory, url, jsonHttpContent); for (Map.Entry entry : headers.getHeaders().entrySet()) { diff --git a/sdk-platform-java/gax-java/gax-httpjson/src/test/java/com/google/api/gax/httpjson/HttpJsonClientCallsTest.java b/sdk-platform-java/gax-java/gax-httpjson/src/test/java/com/google/api/gax/httpjson/HttpJsonClientCallsTest.java index 7e2d9cb946c4..5b7ea376e7e6 100644 --- a/sdk-platform-java/gax-java/gax-httpjson/src/test/java/com/google/api/gax/httpjson/HttpJsonClientCallsTest.java +++ b/sdk-platform-java/gax-java/gax-httpjson/src/test/java/com/google/api/gax/httpjson/HttpJsonClientCallsTest.java @@ -77,14 +77,13 @@ void setUp() throws IOException { endpointContext = Mockito.mock(EndpointContext.class); mockChannel = Mockito.mock(HttpJsonChannel.class); descriptor = Mockito.mock(ApiMethodDescriptor.class); - callOptions = Mockito.mock(HttpJsonCallOptions.class); + callOptions = HttpJsonCallOptions.newBuilder().setCredentials(credentials).build(); callContext = HttpJsonCallContext.of(mockChannel, callOptions) .withEndpointContext(endpointContext) .withChannel(mockChannel); - Mockito.when(callOptions.getCredentials()).thenReturn(credentials); Mockito.doNothing() .when(endpointContext) .validateUniverseDomain( diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/ApiTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/ApiTracer.java index 97f8e017db28..ce55774ae2ac 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/ApiTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/ApiTracer.java @@ -192,6 +192,14 @@ default void requestSent() {} default void batchRequestSent(long elementCount, long requestSize) {} ; + /** + * Annotates the attempt with the full resolved HTTP URL. Only relevant for HTTP transport. + * + * @param requestUrl the full URL of the request + */ + default void requestUrlResolved(String requestUrl) {} + ; + /** * A context class to be used with {@link #inScope()} and a try-with-resources block. Closing a * {@link Scope} removes any context that the underlying implementation might've set in {@link diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/ObservabilityAttributes.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/ObservabilityAttributes.java index 64095fde0099..f4e0c8b2eb3e 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/ObservabilityAttributes.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/ObservabilityAttributes.java @@ -93,4 +93,7 @@ public class ObservabilityAttributes { /** The destination resource id of the request (e.g. projects/p/locations/l/topics/t). */ public static final String DESTINATION_RESOURCE_ID_ATTRIBUTE = "gcp.resource.destination.id"; + + /** The full URL of the HTTP request, with sensitive query parameters redacted. */ + public static final String HTTP_URL_FULL_ATTRIBUTE = "url.full"; } diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/ObservabilityUtils.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/ObservabilityUtils.java index 2487964370ca..4c0cef84139d 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/ObservabilityUtils.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/ObservabilityUtils.java @@ -31,16 +31,140 @@ import com.google.api.gax.rpc.ApiException; import com.google.api.gax.rpc.StatusCode; +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableSet; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.common.AttributesBuilder; import java.util.Map; import java.util.concurrent.CancellationException; import javax.annotation.Nullable; -class ObservabilityUtils { +final class ObservabilityUtils { - /** Function to extract the status of the error as a string */ - static String extractStatus(@Nullable Throwable error) { + private ObservabilityUtils() { + } + + /** Constant for redacted values. */ + private static final String REDACTED_VALUE = "REDACTED"; + + /** + * A set of lowercase query parameter keys whose values should be + * redacted in URLs for observability. These include direct credentials + * (access keys), cryptographic signatures (to prevent replay attacks + * or leak of authorization), and session identifiers (like upload_id). + */ + private static final ImmutableSet SENSITIVE_QUERY_KEYS = + ImmutableSet.of( + "awsaccesskeyid", // AWS S3-compatible access keys + "signature", // General cryptographic signature + "sig", // General cryptographic signature (abbreviated) + "x-goog-signature", // Google Cloud specific signature + "upload_id", // Resumable upload session identifiers + "access_token", // OAuth2 explicit tokens + "key", // API Keys + "api_key"); // API Keys + + /** + * Sanitizes an HTTP URL by redacting sensitive query parameters and + * credentials in the user-info component. If the provided URL cannot + * be parsed (e.g. invalid syntax), it returns the original string. + * + *

This sanitization process conforms to the recommendations in footnote 3 + * of the OpenTelemetry semantic conventions for HTTP URL attributes: + * https://opentelemetry.io/docs/specs/semconv/registry/attributes/url/ + * + *

    + *
  • "url.full MUST NOT contain credentials passed via URL in form of + * https://user:pass@example.com/. In such case username and password + * SHOULD be redacted and attribute's value SHOULD be + * https://REDACTED:REDACTED@example.com/." + * - Handled by stripping the raw user info component. + *
  • "url.full SHOULD capture the absolute URL when it is available + * (or can be reconstructed)." + * - Handled by parsing and rebuilding the generic URI. + *
  • "When a query string value is redacted, the query string key + * SHOULD still be preserved, e.g. + * https://www.example.com/path?color=blue&sig=REDACTED." + * - Handled by the redactSensitiveQueryValues method. + *
+ * + * @param url the raw URL string + * @return the sanitized URL string, or the original if unparsable + */ + static String sanitizeUrlFull(final String url) { + try { + java.net.URI uri = new java.net.URI(url); + String sanitizedUserInfo = + uri.getRawUserInfo() != null + ? REDACTED_VALUE + ":" + REDACTED_VALUE + : null; + String sanitizedQuery = redactSensitiveQueryValues(uri.getRawQuery()); + java.net.URI sanitizedUri = + new java.net.URI( + uri.getScheme(), + sanitizedUserInfo, + uri.getHost(), + uri.getPort(), + uri.getRawPath(), + sanitizedQuery, + uri.getRawFragment()); + return sanitizedUri.toString(); + } catch (java.net.URISyntaxException | IllegalArgumentException ex) { + return url; + } + } + + /** + * Redacts the values of sensitive keys within a raw URI query string. + * + *

This logic splits the query string by the {@code &} delimiter + * without full URL decoding, ensures only values belonging to predefined + * sensitive keys are replaced with {@code REDACTED_VALUE}. + * The check is strictly case-insensitive. + * + *

Note regarding Footnote 3: The OpenTelemetry spec recommends + * case-sensitive matching for query parameters. However, we intentionally + * utilize case-insensitive matching (by lowercasing all query keys) + * to prevent credentials bypassing validation when sent with mixed + * casings (e.g., Sig=..., API_KEY=...). + * + * @param rawQuery the raw query string from a java.net.URI + * @return a reconstructed query sequence with sensitive values redacted + */ + private static String redactSensitiveQueryValues(final String rawQuery) { + if (rawQuery == null || rawQuery.isEmpty()) { + return rawQuery; + } + + java.util.List redactedParams = + Splitter.on('&').splitToList(rawQuery).stream() + .map( + param -> { + int equalsIndex = param.indexOf('='); + String key = equalsIndex >= 0 + ? param.substring(0, equalsIndex) + : param; + // Case-insensitive match utilizing the fact that all + // predefined keys are in lowercase + if (SENSITIVE_QUERY_KEYS.contains( + key.toLowerCase(java.util.Locale.US))) { + return key + "=" + REDACTED_VALUE; + } + return param; + }) + .collect(java.util.stream.Collectors.toList()); + + return Joiner.on('&').join(redactedParams); + } + + /** + * Function to extract the status of the error as a string. + * + * @param error the thrown throwable error + * @return the extracted status string + */ + static String extractStatus(@Nullable final Throwable error) { final String statusString; if (error == null) { @@ -48,7 +172,8 @@ static String extractStatus(@Nullable Throwable error) { } else if (error instanceof CancellationException) { statusString = StatusCode.Code.CANCELLED.toString(); } else if (error instanceof ApiException) { - statusString = ((ApiException) error).getStatusCode().getCode().toString(); + statusString = + ((ApiException) error).getStatusCode().getCode().toString(); } else { statusString = StatusCode.Code.UNKNOWN.toString(); } @@ -56,7 +181,7 @@ static String extractStatus(@Nullable Throwable error) { return statusString; } - static Attributes toOtelAttributes(Map attributes) { + static Attributes toOtelAttributes(final Map attributes) { AttributesBuilder attributesBuilder = Attributes.builder(); if (attributes == null) { return attributesBuilder.build(); diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/SpanTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/SpanTracer.java index a2690359dd4c..751a7114b5bf 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/SpanTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/SpanTracer.java @@ -160,4 +160,12 @@ private void endAttempt() { attemptSpan = null; } } + + @Override + public void requestUrlResolved(String url) { + if (attemptSpan != null && url != null) { + attemptSpan.setAttribute( + ObservabilityAttributes.HTTP_URL_FULL_ATTRIBUTE, ObservabilityUtils.sanitizeUrlFull(url)); + } + } } diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/ObservabilityUtilsTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/ObservabilityUtilsTest.java index 0af3be474604..6c8a6f0b31aa 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/ObservabilityUtilsTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/ObservabilityUtilsTest.java @@ -118,4 +118,36 @@ void testToOtelAttributes_shouldMapIntAttributes() { void testToOtelAttributes_shouldReturnEmptyAttributes_nullInput() { assertThat(ObservabilityUtils.toOtelAttributes(null)).isEqualTo(Attributes.empty()); } + + @Test + void testSanitizeUrlFull_redactsUserInfo() { + String url = "https://user:password@example.com/some/path?foo=bar"; + String sanitized = ObservabilityUtils.sanitizeUrlFull(url); + assertThat(sanitized).isEqualTo("https://REDACTED:REDACTED@example.com/some/path?foo=bar"); + } + + @Test + void testSanitizeUrlFull_redactsSensitiveQueryParameters_caseInsensitive() { + String url = + "https://example.com/some/path?upload_Id=secret&AWSAccessKeyId=123&foo=bar&API_KEY=my_key"; + String sanitized = ObservabilityUtils.sanitizeUrlFull(url); + assertThat(sanitized) + .isEqualTo( + "https://example.com/some/path?upload_Id=REDACTED&AWSAccessKeyId=REDACTED&foo=bar&API_KEY=REDACTED"); + } + + @Test + void testSanitizeUrlFull_handlesMalformedUrl() { + String url = "invalid::url:"; + String sanitized = ObservabilityUtils.sanitizeUrlFull(url); + // Unparsable URLs should be returned as-is + assertThat(sanitized).isEqualTo(url); + } + + @Test + void testSanitizeUrlFull_noQueryOrUserInfo() { + String url = "https://example.com/some/path"; + String sanitized = ObservabilityUtils.sanitizeUrlFull(url); + assertThat(sanitized).isEqualTo(url); + } } diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/SpanTracerTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/SpanTracerTest.java index 3e9fc53ce536..4bc6fc8beb72 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/SpanTracerTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/SpanTracerTest.java @@ -186,4 +186,17 @@ void testAttemptStarted_retryAttributes_http() { ObservabilityAttributes.HTTP_RESEND_COUNT_ATTRIBUTE), 5L); } + + @Test + void testRequestUrlResolved_setsAttribute() { + spanTracer.attemptStarted(new Object(), 1); + + String rawUrl = "https://example.com?api_key=secret"; + spanTracer.requestUrlResolved(rawUrl); + + verify(span) + .setAttribute( + ObservabilityAttributes.HTTP_URL_FULL_ATTRIBUTE, + "https://example.com?api_key=REDACTED"); + } }