CLDSRV-863: Checksums for PutObject and UploadPart

.claude/skills/review-pr/SKILL.md

@@ -79,6 +79,8 @@ Parse `$ARGUMENTS` to extract the repo and PR number:
     - **Test quality** — no `.only()` tests (eslint enforces this),
       assertions match the behavior being tested, `require()`/`import`
       at top of file (never inside `describe` or functions).
+    - **Test prefix** — Name of tests using the `it()` Mocha test function
+      should start with `should`. Only raise this criteria once per file.
 
 4. **Deliver your review:**
 

lib/api/apiUtils/integrity/validateChecksums.js

@@ -2,7 +2,21 @@ const crypto = require('crypto');
 const { Crc32 } = require('@aws-crypto/crc32');
 const { Crc32c } = require('@aws-crypto/crc32c');
 const { CrtCrc64Nvme } = require('@aws-sdk/crc64-nvme-crt');
-const { errors: ArsenalErrors } = require('arsenal');
+const { errors: ArsenalErrors, errorInstances } = require('arsenal');
+
+const errAlgoNotSupported = errorInstances.InvalidRequest.customizeDescription(
+    'The algorithm type you specified in x-amz-checksum- header is invalid.');
+const errAlgoNotSupportedSDK = errorInstances.InvalidRequest.customizeDescription(
+    'Value for x-amz-sdk-checksum-algorithm header is invalid.');
+const errMissingCorresponding = errorInstances.InvalidRequest.customizeDescription(
+    'x-amz-sdk-checksum-algorithm specified, but no corresponding x-amz-checksum-* ' +
+    'or x-amz-trailer headers were found.');
+const errMultipleChecksumTypes = errorInstances.InvalidRequest.customizeDescription(
+    'Expecting a single x-amz-checksum- header. Multiple checksum Types are not allowed.');
+const errTrailerAndChecksum = errorInstances.InvalidRequest.customizeDescription(
+    'Expecting a single x-amz-checksum- header');
+const errTrailerNotSupported = errorInstances.InvalidRequest.customizeDescription(
+    'The value specified in the x-amz-trailer header is not supported');
 const { config } = require('../../../Config');
 
 const checksumedMethods = Object.freeze({
@@ -37,6 +51,12 @@ const ChecksumError = Object.freeze({
     MultipleChecksumTypes: 'MultipleChecksumTypes',
     MissingCorresponding: 'MissingCorresponding',
     MalformedChecksum: 'MalformedChecksum',
+    TrailerAlgoMismatch: 'TrailerAlgoMismatch',
+    TrailerChecksumMalformed: 'TrailerChecksumMalformed',
+    TrailerMissing: 'TrailerMissing',
+    TrailerUnexpected: 'TrailerUnexpected',
+    TrailerAndChecksum: 'TrailerAndChecksum',
+    TrailerNotSupported: 'TrailerNotSupported',
 });
 
 const base64Regex = /^[A-Za-z0-9+/]*={0,2}$/;
@@ -56,35 +76,51 @@ const algorithms = Object.freeze({
             const result = await crc.digest();
             return Buffer.from(result).toString('base64');
         },
+        digestFromHash: async hash => {
+            const result = await hash.digest();
+            return Buffer.from(result).toString('base64');
+        },
         isValidDigest: expected => typeof expected === 'string' && expected.length === 12 && base64Regex.test(expected),
+        createHash: () => new CrtCrc64Nvme()
     },
     crc32: {
         digest: data => {
             const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
             return uint32ToBase64(new Crc32().update(input).digest() >>> 0); // >>> 0 coerce number to uint32
         },
+        digestFromHash: hash => {
+            const result = hash.digest();
+            return uint32ToBase64(result >>> 0);
+        },
         isValidDigest: expected => typeof expected === 'string' && expected.length === 8 && base64Regex.test(expected),
+        createHash: () => new Crc32()
     },
     crc32c: {
         digest: data => {
             const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
             return uint32ToBase64(new Crc32c().update(input).digest() >>> 0); // >>> 0 coerce number to uint32
         },
+        digestFromHash: hash => uint32ToBase64(hash.digest() >>> 0),
         isValidDigest: expected => typeof expected === 'string' && expected.length === 8 && base64Regex.test(expected),
+        createHash: () => new Crc32c()
     },
     sha1: {
         digest: data => {
             const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
             return crypto.createHash('sha1').update(input).digest('base64');
         },
+        digestFromHash: hash => hash.digest('base64'),
         isValidDigest: expected => typeof expected === 'string' && expected.length === 28 && base64Regex.test(expected),
+        createHash: () => crypto.createHash('sha1')
     },
     sha256: {
         digest: data => {
             const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
             return crypto.createHash('sha256').update(input).digest('base64');
         },
+        digestFromHash: hash => hash.digest('base64'),
         isValidDigest: expected => typeof expected === 'string' && expected.length === 44 && base64Regex.test(expected),
+        createHash: () => crypto.createHash('sha256')
     }
 });
 
@@ -132,7 +168,7 @@ async function validateXAmzChecksums(headers, body) {
             return { error: ChecksumError.AlgoNotSupportedSDK, details: { algorithm: sdkAlgo } };
         }
 
-        // If AWS there is a mismatch, AWS returns the same error as if the algo was invalid.
+        // If there is a mismatch, AWS returns the same error as if the algo was invalid.
         if (sdkLowerAlgo !== algo) {
             return { error: ChecksumError.AlgoNotSupportedSDK, details: { algorithm: sdkAlgo } };
         }
@@ -141,6 +177,94 @@ async function validateXAmzChecksums(headers, body) {
     return null;
 }
 
+function getChecksumDataFromHeaders(headers) {
+    const checkSdk = algo => {
+        if (!('x-amz-sdk-checksum-algorithm' in headers)) {
+            return null;
+        }
+
+        const sdkAlgo = headers['x-amz-sdk-checksum-algorithm'];
+        if (typeof sdkAlgo !== 'string') {
+            return { error: ChecksumError.AlgoNotSupportedSDK, details: { algorithm: sdkAlgo } };
+        }
+
+        const sdkLowerAlgo = sdkAlgo.toLowerCase();
+        if (!(sdkLowerAlgo in algorithms)) {
+            return { error: ChecksumError.AlgoNotSupportedSDK, details: { algorithm: sdkAlgo } };
+        }
+
+        // If there is a mismatch, AWS returns the same error as if the algo was invalid.
+        if (sdkLowerAlgo !== algo) {
+            return { error: ChecksumError.AlgoNotSupportedSDK, details: { algorithm: sdkAlgo } };
+        }
+
+        return null;
+    };
+
+    const checksumHeaders = Object.keys(headers).filter(header => header.startsWith('x-amz-checksum-'));
+    const xAmzChecksumCnt = checksumHeaders.length;
+    if (xAmzChecksumCnt > 1) {
+        return { error: ChecksumError.MultipleChecksumTypes, details: { algorithms: checksumHeaders } };
+    }
+
+    if (xAmzChecksumCnt === 0 && !('x-amz-trailer' in headers) && 'x-amz-sdk-checksum-algorithm' in headers) {
+        return {
+            error: ChecksumError.MissingCorresponding,
+            details: { expected: headers['x-amz-sdk-checksum-algorithm'] }
+        };
+    }
+
+    if ('x-amz-trailer' in headers) {
+        if (xAmzChecksumCnt !== 0) {
+            return {
+                error: ChecksumError.TrailerAndChecksum,
+                details: { trailer: headers['x-amz-trailer'], checksum: checksumHeaders },
+            };
+        }
+
+        const trailer = headers['x-amz-trailer'];
+        if (!trailer.startsWith('x-amz-checksum-')) {
+            return { error: ChecksumError.TrailerNotSupported, details: { value: trailer } };
+        }
+
+        const trailerAlgo = trailer.slice('x-amz-checksum-'.length);
+        if (!(trailerAlgo in algorithms)) {
+            return { error: ChecksumError.TrailerNotSupported, details: { value: trailer } };
+        }
+
+        const err = checkSdk(trailerAlgo);
+        if (err) {
+            return err;
+        }
+
+        return { algorithm: trailerAlgo, isTrailer: true, expected: undefined };
+    }
+
+    if (xAmzChecksumCnt === 0) {
+        // There was no x-amz-checksum- or x-amz-trailer return crc64nvme.
+        // The calculated crc64nvme will be stored in the object metadata.
+        return { algorithm: 'crc64nvme', isTrailer: false, expected: undefined };
+    }
+
+    // No x-amz-sdk-checksum-algorithm we expect one x-amz-checksum-[crc64nvme, crc32, crc32C, sha1, sha256].
+    const algo = checksumHeaders[0].slice('x-amz-checksum-'.length);
+    if (!(algo in algorithms)) {
+        return { error: ChecksumError.AlgoNotSupported, details: { algorithm: algo } };
+    }
+
+    const expected = headers[`x-amz-checksum-${algo}`];
+    if (!algorithms[algo].isValidDigest(expected)) {
+        return { error: ChecksumError.MalformedChecksum, details: { algorithm: algo, expected } };
+    }
+
+    const err = checkSdk(algo);
+    if (err) {
+        return err;
+    }
+
+    return { algorithm: algo, isTrailer: false, expected };
+}
+
 /**
  * validateChecksumsNoChunking - Validate the checksums of a request.
  * @param {object} headers - http headers
@@ -183,53 +307,59 @@ async function validateChecksumsNoChunking(headers, body) {
     return err;
 }
 
-async function defaultValidationFunc(request, body, log) {
-    const err = await validateChecksumsNoChunking(request.headers, body);
-    if (!err) {
-        return null;
-    }
-
-    if (err.error !== ChecksumError.MissingChecksum) {
-        log.debug('failed checksum validation', { method: request.apiMethod }, err);
-    }
-
+function arsenalErrorFromChecksumError(err) {
     switch (err.error) {
         case ChecksumError.MissingChecksum:
             return null;
         case ChecksumError.XAmzMismatch: {
             const algoUpper = err.details.algorithm.toUpperCase();
-            return ArsenalErrors.BadDigest.customizeDescription(
-                `The ${algoUpper} you specified did not match the calculated checksum.`
-            );
+            return errorInstances.BadDigest.customizeDescription(
+                `The ${algoUpper} you specified did not match the calculated checksum.`);
         }
         case ChecksumError.AlgoNotSupported:
-            return ArsenalErrors.InvalidRequest.customizeDescription(
-                'The algorithm type you specified in x-amz-checksum- header is invalid.'
-            );
+            return errAlgoNotSupported;
         case ChecksumError.AlgoNotSupportedSDK:
-            return ArsenalErrors.InvalidRequest.customizeDescription(
-                'Value for x-amz-sdk-checksum-algorithm header is invalid.'
-            );
+            return errAlgoNotSupportedSDK;
         case ChecksumError.MissingCorresponding:
-            return ArsenalErrors.InvalidRequest.customizeDescription(
-                'x-amz-sdk-checksum-algorithm specified, but no corresponding x-amz-checksum-* ' +
-                'or x-amz-trailer headers were found.'
-            );
+            return errMissingCorresponding;
         case ChecksumError.MultipleChecksumTypes:
-            return ArsenalErrors.InvalidRequest.customizeDescription(
-                'Expecting a single x-amz-checksum- header. Multiple checksum Types are not allowed.'
-            );
+            return errMultipleChecksumTypes;
         case ChecksumError.MalformedChecksum:
-            return ArsenalErrors.InvalidRequest.customizeDescription(
-                `Value for x-amz-checksum-${err.details.algorithm} header is invalid.`
-            );
+            return errorInstances.InvalidRequest.customizeDescription(
+                `Value for x-amz-checksum-${err.details.algorithm} header is invalid.`);
         case ChecksumError.MD5Invalid:
             return ArsenalErrors.InvalidDigest;
+        case ChecksumError.TrailerAlgoMismatch:
+            return ArsenalErrors.MalformedTrailerError;
+        case ChecksumError.TrailerMissing:
+            return ArsenalErrors.MalformedTrailerError;
+        case ChecksumError.TrailerUnexpected:
+            return ArsenalErrors.MalformedTrailerError;
+        case ChecksumError.TrailerChecksumMalformed:
+            return errorInstances.InvalidRequest.customizeDescription(
+                `Value for x-amz-checksum-${err.details.algorithm} trailing header is invalid.`);
+        case ChecksumError.TrailerAndChecksum:
+            return errTrailerAndChecksum;
+        case ChecksumError.TrailerNotSupported:
+            return errTrailerNotSupported;
         default:
             return ArsenalErrors.BadDigest;
     }
 }
 
+async function defaultValidationFunc(request, body, log) {
+    const err = await validateChecksumsNoChunking(request.headers, body);
+    if (!err) {
+        return null;
+    }
+
+    if (err.error !== ChecksumError.MissingChecksum) {
+        log.debug('failed checksum validation', { method: request.apiMethod }, err);
+    }
+
+    return arsenalErrorFromChecksumError(err);
+}
+
 /**
  * validateMethodChecksumsNoChunking - Validate the checksums of a request.
  * @param {object} request - http request
@@ -253,5 +383,8 @@ module.exports = {
     ChecksumError,
     validateChecksumsNoChunking,
     validateMethodChecksumNoChunking,
+    getChecksumDataFromHeaders,
+    arsenalErrorFromChecksumError,
+    algorithms,
     checksumedMethods,
 };