From d5fdd77a0fc8345683afbfe86ff3693933fba73c Mon Sep 17 00:00:00 2001 From: Maha Benzekri Date: Tue, 24 Feb 2026 09:51:07 +0100 Subject: [PATCH 1/7] add unit tests for createAndStoreObject Issue: CLDSRV-561 --- tests/unit/api/createAndStoreObject.js | 565 +++++++++++++++++++++++++ 1 file changed, 565 insertions(+) create mode 100644 tests/unit/api/createAndStoreObject.js diff --git a/tests/unit/api/createAndStoreObject.js b/tests/unit/api/createAndStoreObject.js new file mode 100644 index 0000000000..85c19556e5 --- /dev/null +++ b/tests/unit/api/createAndStoreObject.js @@ -0,0 +1,565 @@ +/** + * Unit tests for createAndStoreObject function + * Tests cold storage restoration, versioning, and corner cases + */ + +const assert = require('assert'); +const { storage } = require('arsenal'); +const sinon = require('sinon'); + +const { bucketPut } = require('../../../lib/api/bucketPut'); +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); +const metadata = require('../metadataswitch'); +const createAndStoreObject = require('../../../lib/api/apiUtils/object/createAndStoreObject'); +const DummyRequest = require('../DummyRequest'); + +const { ds } = storage.data.inMemory.datastore; +const log = new DummyRequestLogger(); +const authInfo = makeAuthInfo('accessKey1'); +const canonicalID = authInfo.getCanonicalID(); +const bucketName = 'test-bucket'; +const objectKey = 'test-object'; + +describe('createAndStoreObject', () => { + let testBucket; + const getStoredObjectData = () => metadata.putObjectMD.lastCall.args[2].getValue(); + const getStoredOptions = () => metadata.putObjectMD.lastCall.args[3]; + + beforeEach(done => { + cleanup(); + const bucketRequest = new DummyRequest({ + bucketName, + namespace: 'default', + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: '/', + }); + bucketPut(authInfo, bucketRequest, log, err => { + if (err) { + return done(err); + } + return metadata.getBucket(bucketName, log, (err, bucket) => { + testBucket = bucket; + done(err); + }); + }); + }); + + afterEach(() => { + cleanup(); + sinon.restore(); + }); + + describe('Regular object creation', () => { + it('should create object successfully', done => { + const request = new DummyRequest({ + bucketName, + namespace: 'default', + objectKey, + headers: { 'content-type': 'text/plain' }, + url: `/${bucketName}/${objectKey}`, + }, Buffer.from('test data', 'utf8')); + + createAndStoreObject(bucketName, testBucket, objectKey, null, + authInfo, canonicalID, null, request, false, null, + ['overhead'], log, 's3:ObjectCreated:Put', (err, result) => { + assert.ifError(err); + assert(result.contentMD5); + + // Verify object was stored + metadata.getObjectMD(bucketName, objectKey, {}, log, (err, objMD) => { + assert.ifError(err); + assert.strictEqual(objMD['content-md5'], result.contentMD5); + done(); + }); + }); + }); + + it('should handle zero-byte object', done => { + const request = new DummyRequest({ + bucketName, + namespace: 'default', + objectKey, + headers: { 'content-type': 'text/plain' }, + parsedContentLength: 0, + url: `/${bucketName}/${objectKey}`, + }, ''); + + createAndStoreObject(bucketName, testBucket, objectKey, null, + authInfo, canonicalID, null, request, false, null, + ['overhead'], log, 's3:ObjectCreated:Put', (err, result) => { + assert.ifError(err); + assert(result.contentMD5); + done(); + }); + }); + + it('should set bucketOwnerId when requester is not bucket owner', done => { + const authInfo2 = makeAuthInfo('accessKey2'); + sinon.spy(metadata, 'putObjectMD'); + + const request = new DummyRequest({ + bucketName, + namespace: 'default', + objectKey, + headers: {}, + url: `/${bucketName}/${objectKey}`, + }, Buffer.from('test', 'utf8')); + + createAndStoreObject(bucketName, testBucket, objectKey, null, + authInfo2, authInfo2.getCanonicalID(), null, request, false, null, + ['overhead'], log, 's3:ObjectCreated:Put', err => { + assert.ifError(err); + const storedObjMD = getStoredObjectData(); + assert.strictEqual(storedObjMD.bucketOwnerId, canonicalID); + done(); + }); + }); + }); + + describe('Delete marker creation', () => { + it('should create delete marker without storing data', done => { + const request = new DummyRequest({ + bucketName, + namespace: 'default', + objectKey, + headers: {}, + url: `/${bucketName}/${objectKey}`, + }); + + createAndStoreObject(bucketName, testBucket, objectKey, null, + authInfo, canonicalID, null, request, true, null, + ['overhead'], log, 's3:ObjectRemoved:DeleteMarkerCreated', err => { + assert.ifError(err); + + // Verify no data was stored + assert.deepStrictEqual(ds, []); + + // Verify delete marker metadata was created + metadata.getObjectMD(bucketName, objectKey, {}, log, (err, objMD) => { + assert.ifError(err); + assert(objMD.isDeleteMarker); + done(); + }); + }); + }); + }); + + describe('Archived object replacement', () => { + it('should trigger oplog event when replacing archived object in non-versioned bucket', done => { + const archivedObjMD = { + 'content-md5': 'abc123', + 'content-length': 100, + 'archive': { + 'archiveInfo': { 'archiveID': 'archive-123' }, + }, + }; + + sinon.spy(metadata, 'putObjectMD'); + + const request = new DummyRequest({ + bucketName, + namespace: 'default', + objectKey, + headers: {}, + url: `/${bucketName}/${objectKey}`, + }, Buffer.from('new data', 'utf8')); + + createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, + authInfo, canonicalID, null, request, false, null, + ['overhead'], log, 's3:ObjectCreated:Put', err => { + assert.ifError(err); + const options = getStoredOptions(); + assert.strictEqual(options.needOplogUpdate, true); + assert.strictEqual(options.originOp, 's3:ReplaceArchivedObject'); + done(); + }); + }); + + it('should not trigger oplog event for archived object in versioned bucket', done => { + const archivedObjMD = { + 'content-md5': 'abc123', + 'content-length': 100, + 'versionId': 'v1', + 'archive': { + 'archiveInfo': { 'archiveID': 'archive-123' }, + }, + }; + + sinon.stub(testBucket, 'isVersioningEnabled').returns(true); + sinon.spy(metadata, 'putObjectMD'); + + const request = new DummyRequest({ + bucketName, + namespace: 'default', + objectKey, + headers: {}, + url: `/${bucketName}/${objectKey}`, + }, Buffer.from('new data', 'utf8')); + + createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, + authInfo, canonicalID, null, request, false, null, + ['overhead'], log, 's3:ObjectCreated:Put', err => { + assert.ifError(err); + const options = getStoredOptions(); + assert.strictEqual(options.needOplogUpdate, undefined); + assert.strictEqual(options.originOp, undefined); + done(); + }); + }); + }); + + describe('Cold storage restoration (putObjectVersion)', () => { + it('should restore object with x-scal-s3-version-id header', done => { + const now = Date.now(); + const archivedObjMD = { + 'key': objectKey, + 'versionId': 'v123', + 'content-md5': 'original-hash', + 'content-length': 100, + 'x-amz-storage-class': 'cold-location', + 'dataStoreName': 'cold-location', + 'x-amz-meta-custom': 'preserved-value', + 'tags': { 'tagkey': 'tagvalue' }, + 'archive': { + 'archiveInfo': { 'archiveID': 'archive-123' }, + 'restoreRequestedAt': new Date(now).toISOString(), + 'restoreRequestedDays': 7, + }, + }; + + sinon.spy(metadata, 'putObjectMD'); + + const request = new DummyRequest({ + bucketName, + namespace: 'default', + objectKey, + headers: { 'x-scal-s3-version-id': 'v123' }, + url: `/${bucketName}/${objectKey}`, + }, Buffer.from('restored data', 'utf8')); + + createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, + authInfo, canonicalID, null, request, false, null, + ['overhead'], log, 's3:ObjectCreated:Put', err => { + assert.ifError(err); + // Verify archive info was updated + const storedObjMD = getStoredObjectData(); + assert(storedObjMD.archive.restoreCompletedAt, 'restoreCompletedAt should be set'); + assert(storedObjMD.archive.restoreWillExpireAt, 'restoreWillExpireAt should be set'); + assert.strictEqual(storedObjMD.archive.restoreRequestedDays, 7); + + // Verify metadata preserved + assert.strictEqual(storedObjMD['x-amz-meta-custom'], 'preserved-value'); + assert.deepStrictEqual(storedObjMD.tags, { 'tagkey': 'tagvalue' }); + + // Verify originOp set correctly + assert.strictEqual(storedObjMD.originOp, 's3:ObjectRestore:Completed'); + + done(); + }); + }); + + it('should preserve original etag for MPU restoration with different part count', done => { + const archivedObjMD = { + 'versionId': 'v123', + 'content-md5': 'original-abc123-5', // Original had 5 parts + 'archive': { + 'archiveInfo': { 'archiveID': 'archive-123' }, + 'restoreRequestedAt': new Date().toISOString(), + 'restoreRequestedDays': 7, + }, + }; + + sinon.spy(metadata, 'putObjectMD'); + + const request = new DummyRequest({ + bucketName, + namespace: 'default', + objectKey, + headers: { 'x-scal-s3-version-id': 'v123' }, + url: `/${bucketName}/${objectKey}`, + calculatedHash: 'restored-def456-3', // Restored with 3 parts + }, Buffer.from('restored data', 'utf8')); + + createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, + authInfo, canonicalID, null, request, false, null, + ['overhead'], log, 's3:ObjectCreated:Put', err => { + assert.ifError(err); + const storedObjMD = getStoredObjectData(); + + // Original etag should be preserved + assert.strictEqual(storedObjMD['content-md5'], 'original-abc123-5', + 'Original etag should be preserved'); + + // Restored ETag should be kept in restore headers for expiry processing. + assert(storedObjMD['x-amz-restore']['content-md5']); + assert.notStrictEqual(storedObjMD['x-amz-restore']['content-md5'], + storedObjMD['content-md5']); + + done(); + }); + }); + + it('should preserve replication info during restoration', done => { + const replicationInfo = { + status: 'COMPLETED', + backends: [{ site: 'site1', status: 'COMPLETED' }], + }; + + const archivedObjMD = { + 'versionId': 'v123', + replicationInfo, + 'archive': { + 'archiveInfo': { 'archiveID': 'archive-123' }, + 'restoreRequestedAt': new Date().toISOString(), + 'restoreRequestedDays': 7, + }, + }; + + sinon.spy(metadata, 'putObjectMD'); + + const request = new DummyRequest({ + bucketName, + namespace: 'default', + objectKey, + headers: { 'x-scal-s3-version-id': 'v123' }, + url: `/${bucketName}/${objectKey}`, + }, Buffer.from('restored', 'utf8')); + + createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, + authInfo, canonicalID, null, request, false, null, + ['overhead'], log, 's3:ObjectCreated:Put', err => { + assert.ifError(err); + const storedObjMD = getStoredObjectData(); + assert.strictEqual(storedObjMD.replicationInfo.status, replicationInfo.status, + 'Replication status should be preserved'); + assert.deepStrictEqual(storedObjMD.replicationInfo.backends, replicationInfo.backends, + 'Replication backends should be preserved'); + + done(); + }); + }); + + it('should preserve legal hold during restoration', done => { + const archivedObjMD = { + 'versionId': 'v123', + 'legalHold': true, + 'archive': { + 'archiveInfo': { 'archiveID': 'archive-123' }, + 'restoreRequestedAt': new Date().toISOString(), + 'restoreRequestedDays': 7, + }, + }; + + sinon.spy(metadata, 'putObjectMD'); + + const request = new DummyRequest({ + bucketName, + namespace: 'default', + objectKey, + headers: { 'x-scal-s3-version-id': 'v123' }, + url: `/${bucketName}/${objectKey}`, + }, Buffer.from('restored', 'utf8')); + + createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, + authInfo, canonicalID, null, request, false, null, + ['overhead'], log, 's3:ObjectCreated:Put', err => { + assert.ifError(err); + const storedObjMD = getStoredObjectData(); + assert.strictEqual(storedObjMD.legalHold, true, + 'Legal hold should be preserved'); + + done(); + }); + }); + + it('should preserve ACLs during restoration', done => { + const acl = { + 'Canned': '', + 'FULL_CONTROL': ['canonical-id-1'], + 'READ': ['canonical-id-2'], + }; + + const archivedObjMD = { + 'versionId': 'v123', + acl, + 'archive': { + 'archiveInfo': { 'archiveID': 'archive-123' }, + 'restoreRequestedAt': new Date().toISOString(), + 'restoreRequestedDays': 7, + }, + }; + + sinon.spy(metadata, 'putObjectMD'); + + const request = new DummyRequest({ + bucketName, + namespace: 'default', + objectKey, + headers: { 'x-scal-s3-version-id': 'v123' }, + url: `/${bucketName}/${objectKey}`, + }, Buffer.from('restored', 'utf8')); + + createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, + authInfo, canonicalID, null, request, false, null, + ['overhead'], log, 's3:ObjectCreated:Put', err => { + assert.ifError(err); + const storedObjMD = getStoredObjectData(); + assert.deepStrictEqual(storedObjMD.acl, acl, + 'ACLs should be preserved'); + + done(); + }); + }); + + it('should not preserve x-amz-meta-scal-s3-restore-attempt metadata', done => { + const archivedObjMD = { + 'versionId': 'v123', + 'x-amz-meta-custom': 'keep-this', + 'x-amz-meta-scal-s3-restore-attempt': '3', + 'archive': { + 'archiveInfo': { 'archiveID': 'archive-123' }, + 'restoreRequestedAt': new Date().toISOString(), + 'restoreRequestedDays': 7, + }, + }; + + sinon.spy(metadata, 'putObjectMD'); + + const request = new DummyRequest({ + bucketName, + namespace: 'default', + objectKey, + headers: { 'x-scal-s3-version-id': 'v123' }, + url: `/${bucketName}/${objectKey}`, + }, Buffer.from('restored', 'utf8')); + + createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, + authInfo, canonicalID, null, request, false, null, + ['overhead'], log, 's3:ObjectCreated:Put', err => { + assert.ifError(err); + const storedObjMD = getStoredObjectData(); + assert.strictEqual(storedObjMD['x-amz-meta-custom'], 'keep-this', + 'Custom metadata should be preserved'); + assert.strictEqual(storedObjMD['x-amz-meta-scal-s3-restore-attempt'], undefined, + 'Restore attempt metadata should NOT be preserved'); + + done(); + }); + }); + }); + + describe('MPU scenarios', () => { + it('should set oldReplayId when overwriting MPU object', done => { + const mpuObjMD = { + 'uploadId': 'mpu-upload-123', + 'content-md5': 'abc123', + }; + + sinon.spy(metadata, 'putObjectMD'); + + const request = new DummyRequest({ + bucketName, + namespace: 'default', + objectKey, + headers: {}, + url: `/${bucketName}/${objectKey}`, + }, Buffer.from('new data', 'utf8')); + + createAndStoreObject(bucketName, testBucket, objectKey, mpuObjMD, + authInfo, canonicalID, null, request, false, null, + ['overhead'], log, 's3:ObjectCreated:Put', err => { + assert.ifError(err); + const options = getStoredOptions(); + assert.strictEqual(options.oldReplayId, 'mpu-upload-123'); + done(); + }); + }); + }); + + describe('Azure compatibility', () => { + it('should preserve creation-time from existing object', done => { + const existingObjMD = { + 'creation-time': '2024-01-01T00:00:00.000Z', + 'last-modified': '2024-02-01T00:00:00.000Z', + }; + + sinon.spy(metadata, 'putObjectMD'); + + const request = new DummyRequest({ + bucketName, + namespace: 'default', + objectKey, + headers: {}, + url: `/${bucketName}/${objectKey}`, + }, Buffer.from('new data', 'utf8')); + + createAndStoreObject(bucketName, testBucket, objectKey, existingObjMD, + authInfo, canonicalID, null, request, false, null, + ['overhead'], log, 's3:ObjectCreated:Put', err => { + assert.ifError(err); + const storedObjMD = getStoredObjectData(); + assert.strictEqual(storedObjMD['creation-time'], '2024-01-01T00:00:00.000Z'); + done(); + }); + }); + + it('should fall back to last-modified if creation-time missing', done => { + const existingObjMD = { + 'last-modified': '2024-02-01T00:00:00.000Z', + }; + + sinon.spy(metadata, 'putObjectMD'); + + const request = new DummyRequest({ + bucketName, + namespace: 'default', + objectKey, + headers: {}, + url: `/${bucketName}/${objectKey}`, + }, Buffer.from('new data', 'utf8')); + + createAndStoreObject(bucketName, testBucket, objectKey, existingObjMD, + authInfo, canonicalID, null, request, false, null, + ['overhead'], log, 's3:ObjectCreated:Put', err => { + assert.ifError(err); + const storedObjMD = getStoredObjectData(); + assert.strictEqual(storedObjMD['creation-time'], '2024-02-01T00:00:00.000Z'); + done(); + }); + }); + }); + + describe('Integration-sensitive restore behavior', () => { + it('should keep x-amz-meta-scal-version-id when restoring to ingestion location', done => { + sinon.stub(testBucket, 'isIngestionBucket').returns(true); + sinon.spy(metadata, 'putObjectMD'); + const putVersionId = 'restore-version-id'; + const archivedObjMD = { + versionId: putVersionId, + archive: { + archiveInfo: { archiveID: 'archive-123' }, + restoreRequestedAt: new Date().toISOString(), + restoreRequestedDays: 7, + }, + }; + + const request = new DummyRequest({ + bucketName, + namespace: 'default', + objectKey, + headers: { 'x-scal-s3-version-id': putVersionId }, + url: `/${bucketName}/${objectKey}`, + }, Buffer.from('restored', 'utf8')); + + createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, + authInfo, canonicalID, null, request, false, null, + ['overhead'], log, 's3:ObjectCreated:Put', err => { + assert.ifError(err); + const storedObjMD = getStoredObjectData(); + assert.strictEqual(storedObjMD['x-amz-meta-scal-version-id'], putVersionId); + done(); + }); + }); + }); +}); + From 1ec4d6fd4fabf9f033945e7ac29d7914aad50935 Mon Sep 17 00:00:00 2001 From: Maha Benzekri Date: Tue, 24 Feb 2026 09:51:32 +0100 Subject: [PATCH 2/7] add functional restore via objectOverwite tests Issue: CLDSRV-561 --- .../test/object/objectOverwrite.js | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/tests/functional/aws-node-sdk/test/object/objectOverwrite.js b/tests/functional/aws-node-sdk/test/object/objectOverwrite.js index 8028e3d06a..d3d94ae467 100644 --- a/tests/functional/aws-node-sdk/test/object/objectOverwrite.js +++ b/tests/functional/aws-node-sdk/test/object/objectOverwrite.js @@ -1,12 +1,18 @@ const assert = require('assert'); +const { promisify } = require('util'); const { PutObjectCommand, + PutBucketVersioningCommand, HeadObjectCommand, GetObjectCommand, } = require('@aws-sdk/client-s3'); const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); +const { fakeMetadataArchive, getMetadata, initMetadata } = require('../utils/init'); +const fakeMetadataArchivePromise = promisify(fakeMetadataArchive); +const getMetadataPromise = promisify(getMetadata); +const initMetadataPromise = promisify(initMetadata); const objectName = 'someObject'; const firstPutMetadata = { @@ -30,6 +36,7 @@ describe('Put object with same key as prior object', () => { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; bucketName = await bucketUtil.createRandom(1); + await initMetadataPromise(); }); beforeEach(async () => { @@ -66,5 +73,57 @@ describe('Put object with same key as prior object', () => { const bodyText = await res.Body.transformToString(); assert.deepStrictEqual(bodyText, 'Much different'); }); + + it('should replace archived object in non-versioned bucket', async () => { + await fakeMetadataArchivePromise(bucketName, objectName, undefined, { + archiveInfo: { archiveId: 'archive-1' }, + restoreRequestedAt: new Date(0).toISOString(), + restoreRequestedDays: 5, + }); + + await s3.send(new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + Body: 'overwrite archived data', + Metadata: secondPutMetadata, + })); + + const currentMD = await getMetadataPromise(bucketName, objectName, undefined); + assert.strictEqual(currentMD.archive, undefined); + }); + + it('should overwrite archived current object in versioned bucket', async () => { + await bucketUtil.empty(bucketName); + + await s3.send(new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: { Status: 'Enabled' }, + })); + + const firstPutRes = await s3.send(new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + Body: 'versioned first payload', + Metadata: firstPutMetadata, + })); + assert(firstPutRes.VersionId); + + await fakeMetadataArchivePromise(bucketName, objectName, undefined, { + archiveInfo: { archiveId: 'archive-versioned-current' }, + restoreRequestedAt: new Date(0).toISOString(), + restoreRequestedDays: 5, + }); + + const secondPutRes = await s3.send(new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + Body: 'versioned second payload', + Metadata: secondPutMetadata, + })); + assert(secondPutRes.VersionId); + + const currentVersionMD = await getMetadataPromise(bucketName, objectName, undefined); + assert.strictEqual(currentVersionMD.archive, undefined); + }); }); }); From 57227d65923caa6cbe8ee2d4ed76e1eacc73d0c2 Mon Sep 17 00:00:00 2001 From: Maha Benzekri Date: Tue, 24 Feb 2026 09:51:47 +0100 Subject: [PATCH 3/7] add ingestion-specific restore functional test Issue: CLDSRV-561 --- .../aws-node-sdk/test/object/putVersion.js | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/tests/functional/aws-node-sdk/test/object/putVersion.js b/tests/functional/aws-node-sdk/test/object/putVersion.js index efa68708be..4519b4dbae 100644 --- a/tests/functional/aws-node-sdk/test/object/putVersion.js +++ b/tests/functional/aws-node-sdk/test/object/putVersion.js @@ -1,5 +1,6 @@ const assert = require('assert'); const async = require('async'); +const { promisify } = require('util'); const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); @@ -956,6 +957,71 @@ describe('PUT object with x-scal-s3-version-id header', () => { }, ], done); }); + + it('should set restore originOp and drop restore-attempt metadata', done => { + const params = { Bucket: bucketName, Key: objectName }; + + async.series([ + next => s3.send(new PutObjectCommand({ + ...params, + Metadata: { + 'custom-md': 'preserved-value', + }, + })).then(() => next()).catch(next), + next => fakeMetadataArchive(bucketName, objectName, undefined, archive, next), + next => getMetadata(bucketName, objectName, undefined, (err, objMD) => { + if (err) { + return next(err); + } + /* eslint-disable no-param-reassign */ + objMD['x-amz-meta-scal-s3-restore-attempt'] = '3'; + /* eslint-enable no-param-reassign */ + return metadata.putObjectMD(bucketName, objectName, objMD, undefined, log, next); + }), + next => putObjectVersion(s3, params, '', next), + next => getMetadata(bucketName, objectName, undefined, (err, objMD) => { + if (err) { + return next(err); + } + assert.strictEqual(objMD.originOp, 's3:ObjectRestore:Completed'); + assert.strictEqual(objMD['x-amz-meta-custom-md'], 'preserved-value'); + assert.strictEqual(objMD['x-amz-meta-scal-s3-restore-attempt'], undefined); + return next(); + }), + ], done); + }); + + it('should keep x-amz-meta-scal-version-id when restoring on ingestion bucket', async () => { + const ingestionBucketName = `ingestion-restore-${Date.now()}`; + const params = { Bucket: ingestionBucketName, Key: objectName }; + let putVersionId; + const fakeMetadataArchivePromise = promisify(fakeMetadataArchive); + const putObjectVersionPromise = promisify(putObjectVersion); + const getMetadataPromise = promisify(getMetadata); + try { + await s3.send(new CreateBucketCommand({ + Bucket: ingestionBucketName, + CreateBucketConfiguration: { + LocationConstraint: 'us-east-2:ingest', + }, + })); + + const putRes = await s3.send(new PutObjectCommand(params)); + putVersionId = putRes.VersionId; + + await fakeMetadataArchivePromise(ingestionBucketName, objectName, putVersionId, archive); + + await putObjectVersionPromise(s3, params, putVersionId); + + const restoredObjMD = await getMetadataPromise( + ingestionBucketName, objectName, putVersionId); + + assert.strictEqual(restoredObjMD['x-amz-meta-scal-version-id'], putVersionId); + } finally { + await bucketUtil.emptyMany([ingestionBucketName]).catch(() => {}); + await bucketUtil.deleteMany([ingestionBucketName]).catch(() => {}); + } + }); }); }); }); From 3d5ca315cd8375725cd7027eb4ff1769220edd01 Mon Sep 17 00:00:00 2001 From: Maha Benzekri Date: Tue, 24 Feb 2026 18:04:40 +0100 Subject: [PATCH 4/7] post review fixups --- .../test/object/objectOverwrite.js | 14 ++++- tests/unit/api/createAndStoreObject.js | 60 +++++++++++++++++++ 2 files changed, 71 insertions(+), 3 deletions(-) diff --git a/tests/functional/aws-node-sdk/test/object/objectOverwrite.js b/tests/functional/aws-node-sdk/test/object/objectOverwrite.js index d3d94ae467..df7c4a275e 100644 --- a/tests/functional/aws-node-sdk/test/object/objectOverwrite.js +++ b/tests/functional/aws-node-sdk/test/object/objectOverwrite.js @@ -92,7 +92,7 @@ describe('Put object with same key as prior object', () => { assert.strictEqual(currentMD.archive, undefined); }); - it('should overwrite archived current object in versioned bucket', async () => { + it('should create a new version when replacing archived current object in versioned bucket', async () => { await bucketUtil.empty(bucketName); await s3.send(new PutBucketVersioningCommand({ @@ -121,9 +121,17 @@ describe('Put object with same key as prior object', () => { Metadata: secondPutMetadata, })); assert(secondPutRes.VersionId); + assert.notStrictEqual(secondPutRes.VersionId, firstPutRes.VersionId); - const currentVersionMD = await getMetadataPromise(bucketName, objectName, undefined); - assert.strictEqual(currentVersionMD.archive, undefined); + const headRes = await s3.send(new HeadObjectCommand({ + Bucket: bucketName, + Key: objectName, + })); + assert.deepStrictEqual(headRes.Metadata, secondPutMetadata); + + const currentMD = await getMetadataPromise(bucketName, objectName, undefined); + assert.strictEqual(currentMD.archive, undefined); }); + }); }); diff --git a/tests/unit/api/createAndStoreObject.js b/tests/unit/api/createAndStoreObject.js index 85c19556e5..a019ea4892 100644 --- a/tests/unit/api/createAndStoreObject.js +++ b/tests/unit/api/createAndStoreObject.js @@ -206,6 +206,66 @@ describe('createAndStoreObject', () => { done(); }); }); + + it('should trigger oplog event for archived object in version-suspended bucket', done => { + const archivedObjMD = { + 'content-md5': 'abc123', + 'content-length': 100, + archive: { + archiveInfo: { archiveID: 'archive-123' }, + }, + }; + + sinon.stub(testBucket, 'isVersioningEnabled').returns(false); + sinon.spy(metadata, 'putObjectMD'); + + const request = new DummyRequest({ + bucketName, + namespace: 'default', + objectKey, + headers: {}, + url: `/${bucketName}/${objectKey}`, + }, Buffer.from('new data', 'utf8')); + + createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, + authInfo, canonicalID, null, request, false, null, + ['overhead'], log, 's3:ObjectCreated:Put', err => { + assert.ifError(err); + const options = getStoredOptions(); + assert.strictEqual(options.needOplogUpdate, true); + assert.strictEqual(options.originOp, 's3:ReplaceArchivedObject'); + done(); + }); + }); + + it('should not trigger oplog event when archiveInfo is absent', done => { + const archivedObjMD = { + 'content-md5': 'abc123', + 'content-length': 100, + archive: { + restoreRequestedAt: new Date().toISOString(), + }, + }; + sinon.spy(metadata, 'putObjectMD'); + + const request = new DummyRequest({ + bucketName, + namespace: 'default', + objectKey, + headers: {}, + url: `/${bucketName}/${objectKey}`, + }, Buffer.from('new data', 'utf8')); + + createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, + authInfo, canonicalID, null, request, false, null, + ['overhead'], log, 's3:ObjectCreated:Put', err => { + assert.ifError(err); + const options = getStoredOptions(); + assert.strictEqual(options.needOplogUpdate, undefined); + assert.strictEqual(options.originOp, undefined); + done(); + }); + }); }); describe('Cold storage restoration (putObjectVersion)', () => { From ac530232b1ad81e3912559a8ea71b486fb481bd4 Mon Sep 17 00:00:00 2001 From: Maha Benzekri Date: Wed, 25 Feb 2026 17:38:36 +0100 Subject: [PATCH 5/7] tests adding --- .../test/object/objectOverwrite.js | 104 +++++++++++++++--- tests/unit/api/createAndStoreObject.js | 4 + 2 files changed, 93 insertions(+), 15 deletions(-) diff --git a/tests/functional/aws-node-sdk/test/object/objectOverwrite.js b/tests/functional/aws-node-sdk/test/object/objectOverwrite.js index df7c4a275e..1bb78fcb8d 100644 --- a/tests/functional/aws-node-sdk/test/object/objectOverwrite.js +++ b/tests/functional/aws-node-sdk/test/object/objectOverwrite.js @@ -9,8 +9,9 @@ const { const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); -const { fakeMetadataArchive, getMetadata, initMetadata } = require('../utils/init'); +const { fakeMetadataArchive, fakeMetadataTransition, getMetadata, initMetadata } = require('../utils/init'); const fakeMetadataArchivePromise = promisify(fakeMetadataArchive); +const fakeMetadataTransitionPromise = promisify(fakeMetadataTransition); const getMetadataPromise = promisify(getMetadata); const initMetadataPromise = promisify(initMetadata); @@ -74,22 +75,54 @@ describe('Put object with same key as prior object', () => { assert.deepStrictEqual(bodyText, 'Much different'); }); - it('should replace archived object in non-versioned bucket', async () => { - await fakeMetadataArchivePromise(bucketName, objectName, undefined, { - archiveInfo: { archiveId: 'archive-1' }, - restoreRequestedAt: new Date(0).toISOString(), - restoreRequestedDays: 5, - }); + [ + { + name: 'transition in progress', + setup: () => fakeMetadataTransitionPromise(bucketName, objectName, undefined), + }, + { + name: 'archived', + setup: () => fakeMetadataArchivePromise(bucketName, objectName, undefined, { + archiveInfo: { archiveId: 'archive-1' }, + restoreRequestedAt: new Date(0).toISOString(), + restoreRequestedDays: 5, + }), + }, + { + name: 'restored (not expired)', + setup: () => fakeMetadataArchivePromise(bucketName, objectName, undefined, { + archiveInfo: { archiveId: 'archive-restored' }, + restoreRequestedAt: new Date(0).toISOString(), + restoreRequestedDays: 5, + restoreCompletedAt: new Date(Date.now() - 60 * 60 * 1000).toISOString(), + restoreWillExpireAt: new Date(Date.now() + 24 * 60 * 60 * 1000).toISOString(), + }), + }, + { + name: 'restored (expired)', + setup: () => fakeMetadataArchivePromise(bucketName, objectName, undefined, { + archiveInfo: { archiveId: 'archive-expired' }, + restoreRequestedAt: new Date(0).toISOString(), + restoreRequestedDays: 5, + restoreCompletedAt: new Date(Date.now() - 48 * 60 * 60 * 1000).toISOString(), + restoreWillExpireAt: new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString(), + }), + }, + ].forEach(({ name, setup }) => { + it(`should replace object with cold-state metadata (${name}) in non-versioned bucket`, async () => { + await setup(); - await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName, - Body: 'overwrite archived data', - Metadata: secondPutMetadata, - })); + await s3.send(new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + Body: `overwrite cold state ${name}`, + Metadata: secondPutMetadata, + })); - const currentMD = await getMetadataPromise(bucketName, objectName, undefined); - assert.strictEqual(currentMD.archive, undefined); + const currentMD = await getMetadataPromise(bucketName, objectName, undefined); + assert.strictEqual(currentMD.archive, undefined); + assert.strictEqual(currentMD['x-amz-scal-transition-in-progress'], undefined); + }); }); it('should create a new version when replacing archived current object in versioned bucket', async () => { @@ -133,5 +166,46 @@ describe('Put object with same key as prior object', () => { assert.strictEqual(currentMD.archive, undefined); }); + it('should replace archived current null version in version-suspended bucket', async () => { + await bucketUtil.empty(bucketName); + + await s3.send(new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: { Status: 'Enabled' }, + })); + await s3.send(new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + Body: 'enabled-version-payload', + })); + await s3.send(new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: { Status: 'Suspended' }, + })); + + await s3.send(new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + Body: 'null-current-before-archive', + })); + + await fakeMetadataArchivePromise(bucketName, objectName, undefined, { + archiveInfo: { archiveId: 'archive-null-current' }, + restoreRequestedAt: new Date(0).toISOString(), + restoreRequestedDays: 5, + }); + + await s3.send(new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + Body: 'replace archived null current', + Metadata: secondPutMetadata, + })); + + const currentMD = await getMetadataPromise(bucketName, objectName, undefined); + assert.strictEqual(currentMD.archive, undefined); + assert.deepStrictEqual(currentMD['x-amz-meta-secondput'], secondPutMetadata.secondput); + }); + }); }); diff --git a/tests/unit/api/createAndStoreObject.js b/tests/unit/api/createAndStoreObject.js index a019ea4892..fa236933b6 100644 --- a/tests/unit/api/createAndStoreObject.js +++ b/tests/unit/api/createAndStoreObject.js @@ -303,6 +303,7 @@ describe('createAndStoreObject', () => { assert.ifError(err); // Verify archive info was updated const storedObjMD = getStoredObjectData(); + const options = getStoredOptions(); assert(storedObjMD.archive.restoreCompletedAt, 'restoreCompletedAt should be set'); assert(storedObjMD.archive.restoreWillExpireAt, 'restoreWillExpireAt should be set'); assert.strictEqual(storedObjMD.archive.restoreRequestedDays, 7); @@ -313,6 +314,9 @@ describe('createAndStoreObject', () => { // Verify originOp set correctly assert.strictEqual(storedObjMD.originOp, 's3:ObjectRestore:Completed'); + // PutObjectVersion must not use archived-overwrite oplog path. + assert.strictEqual(options.needOplogUpdate, undefined); + assert.strictEqual(options.originOp, undefined); done(); }); From cebeb28efdd0de234eb047729b168510a43e203b Mon Sep 17 00:00:00 2001 From: Maha Benzekri Date: Wed, 25 Mar 2026 09:47:16 +0100 Subject: [PATCH 6/7] modernize metadata helpers and async tests Issue: CLDSRV-561 --- .../test/object/objectOverwrite.js | 93 ++--- .../aws-node-sdk/test/utils/init.js | 73 ++-- tests/unit/api/createAndStoreObject.js | 346 ++++++++---------- 3 files changed, 244 insertions(+), 268 deletions(-) diff --git a/tests/functional/aws-node-sdk/test/object/objectOverwrite.js b/tests/functional/aws-node-sdk/test/object/objectOverwrite.js index 1bb78fcb8d..177339898c 100644 --- a/tests/functional/aws-node-sdk/test/object/objectOverwrite.js +++ b/tests/functional/aws-node-sdk/test/object/objectOverwrite.js @@ -1,5 +1,4 @@ const assert = require('assert'); -const { promisify } = require('util'); const { PutObjectCommand, PutBucketVersioningCommand, @@ -10,10 +9,41 @@ const { const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); const { fakeMetadataArchive, fakeMetadataTransition, getMetadata, initMetadata } = require('../utils/init'); -const fakeMetadataArchivePromise = promisify(fakeMetadataArchive); -const fakeMetadataTransitionPromise = promisify(fakeMetadataTransition); -const getMetadataPromise = promisify(getMetadata); -const initMetadataPromise = promisify(initMetadata); + +const coldStateScenarios = [ + { + name: 'transition in progress', + transitionInProgress: true, + }, + { + name: 'archived', + archiveState: { + archiveInfo: { archiveId: 'archive-1' }, + restoreRequestedAt: new Date(0).toISOString(), + restoreRequestedDays: 5, + }, + }, + { + name: 'restored (not expired)', + archiveState: { + archiveInfo: { archiveId: 'archive-restored' }, + restoreRequestedAt: new Date(0).toISOString(), + restoreRequestedDays: 5, + restoreCompletedAt: new Date(Date.now() - 60 * 60 * 1000).toISOString(), + restoreWillExpireAt: new Date(Date.now() + 24 * 60 * 60 * 1000).toISOString(), + }, + }, + { + name: 'restored (expired)', + archiveState: { + archiveInfo: { archiveId: 'archive-expired' }, + restoreRequestedAt: new Date(0).toISOString(), + restoreRequestedDays: 5, + restoreCompletedAt: new Date(Date.now() - 48 * 60 * 60 * 1000).toISOString(), + restoreWillExpireAt: new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString(), + }, + }, +]; const objectName = 'someObject'; const firstPutMetadata = { @@ -37,7 +67,7 @@ describe('Put object with same key as prior object', () => { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; bucketName = await bucketUtil.createRandom(1); - await initMetadataPromise(); + await initMetadata(); }); beforeEach(async () => { @@ -75,42 +105,13 @@ describe('Put object with same key as prior object', () => { assert.deepStrictEqual(bodyText, 'Much different'); }); - [ - { - name: 'transition in progress', - setup: () => fakeMetadataTransitionPromise(bucketName, objectName, undefined), - }, - { - name: 'archived', - setup: () => fakeMetadataArchivePromise(bucketName, objectName, undefined, { - archiveInfo: { archiveId: 'archive-1' }, - restoreRequestedAt: new Date(0).toISOString(), - restoreRequestedDays: 5, - }), - }, - { - name: 'restored (not expired)', - setup: () => fakeMetadataArchivePromise(bucketName, objectName, undefined, { - archiveInfo: { archiveId: 'archive-restored' }, - restoreRequestedAt: new Date(0).toISOString(), - restoreRequestedDays: 5, - restoreCompletedAt: new Date(Date.now() - 60 * 60 * 1000).toISOString(), - restoreWillExpireAt: new Date(Date.now() + 24 * 60 * 60 * 1000).toISOString(), - }), - }, - { - name: 'restored (expired)', - setup: () => fakeMetadataArchivePromise(bucketName, objectName, undefined, { - archiveInfo: { archiveId: 'archive-expired' }, - restoreRequestedAt: new Date(0).toISOString(), - restoreRequestedDays: 5, - restoreCompletedAt: new Date(Date.now() - 48 * 60 * 60 * 1000).toISOString(), - restoreWillExpireAt: new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString(), - }), - }, - ].forEach(({ name, setup }) => { + coldStateScenarios.forEach(({ name, transitionInProgress, archiveState }) => { it(`should replace object with cold-state metadata (${name}) in non-versioned bucket`, async () => { - await setup(); + if (transitionInProgress) { + await fakeMetadataTransition(bucketName, objectName, undefined); + } else { + await fakeMetadataArchive(bucketName, objectName, undefined, archiveState); + } await s3.send(new PutObjectCommand({ Bucket: bucketName, @@ -119,7 +120,7 @@ describe('Put object with same key as prior object', () => { Metadata: secondPutMetadata, })); - const currentMD = await getMetadataPromise(bucketName, objectName, undefined); + const currentMD = await getMetadata(bucketName, objectName, undefined); assert.strictEqual(currentMD.archive, undefined); assert.strictEqual(currentMD['x-amz-scal-transition-in-progress'], undefined); }); @@ -141,7 +142,7 @@ describe('Put object with same key as prior object', () => { })); assert(firstPutRes.VersionId); - await fakeMetadataArchivePromise(bucketName, objectName, undefined, { + await fakeMetadataArchive(bucketName, objectName, undefined, { archiveInfo: { archiveId: 'archive-versioned-current' }, restoreRequestedAt: new Date(0).toISOString(), restoreRequestedDays: 5, @@ -162,7 +163,7 @@ describe('Put object with same key as prior object', () => { })); assert.deepStrictEqual(headRes.Metadata, secondPutMetadata); - const currentMD = await getMetadataPromise(bucketName, objectName, undefined); + const currentMD = await getMetadata(bucketName, objectName, undefined); assert.strictEqual(currentMD.archive, undefined); }); @@ -189,7 +190,7 @@ describe('Put object with same key as prior object', () => { Body: 'null-current-before-archive', })); - await fakeMetadataArchivePromise(bucketName, objectName, undefined, { + await fakeMetadataArchive(bucketName, objectName, undefined, { archiveInfo: { archiveId: 'archive-null-current' }, restoreRequestedAt: new Date(0).toISOString(), restoreRequestedDays: 5, @@ -202,7 +203,7 @@ describe('Put object with same key as prior object', () => { Metadata: secondPutMetadata, })); - const currentMD = await getMetadataPromise(bucketName, objectName, undefined); + const currentMD = await getMetadata(bucketName, objectName, undefined); assert.strictEqual(currentMD.archive, undefined); assert.deepStrictEqual(currentMD['x-amz-meta-secondput'], secondPutMetadata.secondput); }); diff --git a/tests/functional/aws-node-sdk/test/utils/init.js b/tests/functional/aws-node-sdk/test/utils/init.js index b9301a5fb4..81ebf747fd 100644 --- a/tests/functional/aws-node-sdk/test/utils/init.js +++ b/tests/functional/aws-node-sdk/test/utils/init.js @@ -38,22 +38,33 @@ function decodeVersionId(versionId) { let metadataInit = false; -function initMetadata(done) { - if (metadataInit === true) { - return done(); - } - return metadata.setup(err => { - if (err) { - return done(err); +function initMetadata(cb) { + const promise = (async () => { + if (metadataInit) { + return; } - metadataInit = true; - return done(); - }); + await new Promise((resolve, reject) => { + metadata.setup(err => { + if (err) { + return reject(err); + } + metadataInit = true; + return resolve(); + }); + }); + })(); + return cb ? promise.then(() => cb(), cb) : promise; } function getMetadata(bucketName, objectName, versionId, cb) { - return metadata.getObjectMD(bucketName, objectName, { versionId: decodeVersionId(versionId) }, - log, cb); + const promise = new Promise((resolve, reject) => metadata.getObjectMD( + bucketName, + objectName, + { versionId: decodeVersionId(versionId) }, + log, + (err, data) => (err ? reject(err) : resolve(data)), + )); + return cb ? promise.then(res => cb(null, res), cb) : promise; } /** @@ -66,16 +77,21 @@ function getMetadata(bucketName, objectName, versionId, cb) { * @returns {undefined} */ function fakeMetadataTransition(bucketName, objectName, versionId, cb) { - return getMetadata(bucketName, objectName, versionId, (err, objMD) => { - if (err) { - return cb(err); - } + const promise = (async () => { + const objMD = await getMetadata(bucketName, objectName, versionId); /* eslint-disable no-param-reassign */ objMD['x-amz-scal-transition-in-progress'] = true; /* eslint-enable no-param-reassign */ - return metadata.putObjectMD(bucketName, objectName, objMD, { versionId: decodeVersionId(versionId) }, - log, err => cb(err)); - }); + await new Promise((resolve, reject) => metadata.putObjectMD( + bucketName, + objectName, + objMD, + { versionId: decodeVersionId(versionId) }, + log, + err => (err ? reject(err) : resolve()), + )); + })(); + return cb ? promise.then(() => cb(), cb) : promise; } /** @@ -89,18 +105,23 @@ function fakeMetadataTransition(bucketName, objectName, versionId, cb) { * @returns {undefined} */ function fakeMetadataArchive(bucketName, objectName, versionId, archive, cb) { - return getMetadata(bucketName, objectName, versionId, (err, objMD) => { - if (err) { - return cb(err); - } + const promise = (async () => { + const objMD = await getMetadata(bucketName, objectName, versionId); /* eslint-disable no-param-reassign */ objMD['x-amz-storage-class'] = LOCATION_NAME_DMF; objMD.dataStoreName = LOCATION_NAME_DMF; objMD.archive = archive; /* eslint-enable no-param-reassign */ - return metadata.putObjectMD(bucketName, objectName, objMD, { versionId: decodeVersionId(versionId) }, - log, err => cb(err)); - }); + await new Promise((resolve, reject) => metadata.putObjectMD( + bucketName, + objectName, + objMD, + { versionId: decodeVersionId(versionId) }, + log, + err => (err ? reject(err) : resolve()), + )); + })(); + return cb ? promise.then(() => cb(), cb) : promise; } module.exports = { diff --git a/tests/unit/api/createAndStoreObject.js b/tests/unit/api/createAndStoreObject.js index fa236933b6..25105595c5 100644 --- a/tests/unit/api/createAndStoreObject.js +++ b/tests/unit/api/createAndStoreObject.js @@ -4,15 +4,18 @@ */ const assert = require('assert'); +const { promisify } = require('util'); const { storage } = require('arsenal'); const sinon = require('sinon'); const { bucketPut } = require('../../../lib/api/bucketPut'); const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); const metadata = require('../metadataswitch'); -const createAndStoreObject = require('../../../lib/api/apiUtils/object/createAndStoreObject'); +const rawCreateAndStoreObject = require('../../../lib/api/apiUtils/object/createAndStoreObject'); const DummyRequest = require('../DummyRequest'); +const createAndStoreObject = promisify(rawCreateAndStoreObject); + const { ds } = storage.data.inMemory.datastore; const log = new DummyRequestLogger(); const authInfo = makeAuthInfo('accessKey1'); @@ -20,6 +23,16 @@ const canonicalID = authInfo.getCanonicalID(); const bucketName = 'test-bucket'; const objectKey = 'test-object'; +const getObjectMDAsync = (bucket, key, params = {}) => new Promise((resolve, reject) => { + metadata.getObjectMD(bucket, key, params, log, (err, data) => { + if (err) { + reject(err); + } else { + resolve(data); + } + }); +}); + describe('createAndStoreObject', () => { let testBucket; const getStoredObjectData = () => metadata.putObjectMD.lastCall.args[2].getValue(); @@ -50,7 +63,7 @@ describe('createAndStoreObject', () => { }); describe('Regular object creation', () => { - it('should create object successfully', done => { + it('should create object successfully', async () => { const request = new DummyRequest({ bucketName, namespace: 'default', @@ -59,22 +72,17 @@ describe('createAndStoreObject', () => { url: `/${bucketName}/${objectKey}`, }, Buffer.from('test data', 'utf8')); - createAndStoreObject(bucketName, testBucket, objectKey, null, + const result = await createAndStoreObject(bucketName, testBucket, objectKey, null, authInfo, canonicalID, null, request, false, null, - ['overhead'], log, 's3:ObjectCreated:Put', (err, result) => { - assert.ifError(err); - assert(result.contentMD5); - - // Verify object was stored - metadata.getObjectMD(bucketName, objectKey, {}, log, (err, objMD) => { - assert.ifError(err); - assert.strictEqual(objMD['content-md5'], result.contentMD5); - done(); - }); - }); + ['overhead'], log, 's3:ObjectCreated:Put'); + + assert(result.contentMD5); + + const objMD = await getObjectMDAsync(bucketName, objectKey, {}); + assert.strictEqual(objMD['content-md5'], result.contentMD5); }); - it('should handle zero-byte object', done => { + it('should handle zero-byte object', async () => { const request = new DummyRequest({ bucketName, namespace: 'default', @@ -84,16 +92,14 @@ describe('createAndStoreObject', () => { url: `/${bucketName}/${objectKey}`, }, ''); - createAndStoreObject(bucketName, testBucket, objectKey, null, + const result = await createAndStoreObject(bucketName, testBucket, objectKey, null, authInfo, canonicalID, null, request, false, null, - ['overhead'], log, 's3:ObjectCreated:Put', (err, result) => { - assert.ifError(err); - assert(result.contentMD5); - done(); - }); + ['overhead'], log, 's3:ObjectCreated:Put'); + + assert(result.contentMD5); }); - it('should set bucketOwnerId when requester is not bucket owner', done => { + it('should set bucketOwnerId when requester is not bucket owner', async () => { const authInfo2 = makeAuthInfo('accessKey2'); sinon.spy(metadata, 'putObjectMD'); @@ -105,19 +111,18 @@ describe('createAndStoreObject', () => { url: `/${bucketName}/${objectKey}`, }, Buffer.from('test', 'utf8')); - createAndStoreObject(bucketName, testBucket, objectKey, null, + await createAndStoreObject(bucketName, testBucket, objectKey, null, authInfo2, authInfo2.getCanonicalID(), null, request, false, null, - ['overhead'], log, 's3:ObjectCreated:Put', err => { - assert.ifError(err); - const storedObjMD = getStoredObjectData(); - assert.strictEqual(storedObjMD.bucketOwnerId, canonicalID); - done(); - }); + ['overhead'], log, 's3:ObjectCreated:Put'); + + const storedObjMD = getStoredObjectData(); + assert.strictEqual(storedObjMD.bucketOwnerId, canonicalID); + }); }); }); describe('Delete marker creation', () => { - it('should create delete marker without storing data', done => { + it('should create delete marker without storing data', async () => { const request = new DummyRequest({ bucketName, namespace: 'default', @@ -126,26 +131,19 @@ describe('createAndStoreObject', () => { url: `/${bucketName}/${objectKey}`, }); - createAndStoreObject(bucketName, testBucket, objectKey, null, + await createAndStoreObject(bucketName, testBucket, objectKey, null, authInfo, canonicalID, null, request, true, null, - ['overhead'], log, 's3:ObjectRemoved:DeleteMarkerCreated', err => { - assert.ifError(err); - - // Verify no data was stored - assert.deepStrictEqual(ds, []); - - // Verify delete marker metadata was created - metadata.getObjectMD(bucketName, objectKey, {}, log, (err, objMD) => { - assert.ifError(err); - assert(objMD.isDeleteMarker); - done(); - }); - }); + ['overhead'], log, 's3:ObjectRemoved:DeleteMarkerCreated'); + + assert.deepStrictEqual(ds, []); + + const objMD = await getObjectMDAsync(bucketName, objectKey, {}); + assert(objMD.isDeleteMarker); }); }); describe('Archived object replacement', () => { - it('should trigger oplog event when replacing archived object in non-versioned bucket', done => { + it('should trigger oplog event when replacing archived object in non-versioned bucket', async () => { const archivedObjMD = { 'content-md5': 'abc123', 'content-length': 100, @@ -164,18 +162,16 @@ describe('createAndStoreObject', () => { url: `/${bucketName}/${objectKey}`, }, Buffer.from('new data', 'utf8')); - createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, + await createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, authInfo, canonicalID, null, request, false, null, - ['overhead'], log, 's3:ObjectCreated:Put', err => { - assert.ifError(err); - const options = getStoredOptions(); - assert.strictEqual(options.needOplogUpdate, true); - assert.strictEqual(options.originOp, 's3:ReplaceArchivedObject'); - done(); - }); + ['overhead'], log, 's3:ObjectCreated:Put'); + + const options = getStoredOptions(); + assert.strictEqual(options.needOplogUpdate, true); + assert.strictEqual(options.originOp, 's3:ReplaceArchivedObject'); }); - it('should not trigger oplog event for archived object in versioned bucket', done => { + it('should not trigger oplog event for archived object in versioned bucket', async () => { const archivedObjMD = { 'content-md5': 'abc123', 'content-length': 100, @@ -196,18 +192,16 @@ describe('createAndStoreObject', () => { url: `/${bucketName}/${objectKey}`, }, Buffer.from('new data', 'utf8')); - createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, + await createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, authInfo, canonicalID, null, request, false, null, - ['overhead'], log, 's3:ObjectCreated:Put', err => { - assert.ifError(err); - const options = getStoredOptions(); - assert.strictEqual(options.needOplogUpdate, undefined); - assert.strictEqual(options.originOp, undefined); - done(); - }); + ['overhead'], log, 's3:ObjectCreated:Put'); + + const options = getStoredOptions(); + assert.strictEqual(options.needOplogUpdate, undefined); + assert.strictEqual(options.originOp, undefined); }); - it('should trigger oplog event for archived object in version-suspended bucket', done => { + it('should trigger oplog event for archived object in version-suspended bucket', async () => { const archivedObjMD = { 'content-md5': 'abc123', 'content-length': 100, @@ -227,18 +221,16 @@ describe('createAndStoreObject', () => { url: `/${bucketName}/${objectKey}`, }, Buffer.from('new data', 'utf8')); - createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, + await createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, authInfo, canonicalID, null, request, false, null, - ['overhead'], log, 's3:ObjectCreated:Put', err => { - assert.ifError(err); - const options = getStoredOptions(); - assert.strictEqual(options.needOplogUpdate, true); - assert.strictEqual(options.originOp, 's3:ReplaceArchivedObject'); - done(); - }); + ['overhead'], log, 's3:ObjectCreated:Put'); + + const options = getStoredOptions(); + assert.strictEqual(options.needOplogUpdate, true); + assert.strictEqual(options.originOp, 's3:ReplaceArchivedObject'); }); - it('should not trigger oplog event when archiveInfo is absent', done => { + it('should not trigger oplog event when archiveInfo is absent', async () => { const archivedObjMD = { 'content-md5': 'abc123', 'content-length': 100, @@ -256,20 +248,18 @@ describe('createAndStoreObject', () => { url: `/${bucketName}/${objectKey}`, }, Buffer.from('new data', 'utf8')); - createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, + await createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, authInfo, canonicalID, null, request, false, null, - ['overhead'], log, 's3:ObjectCreated:Put', err => { - assert.ifError(err); - const options = getStoredOptions(); - assert.strictEqual(options.needOplogUpdate, undefined); - assert.strictEqual(options.originOp, undefined); - done(); - }); + ['overhead'], log, 's3:ObjectCreated:Put'); + + const options = getStoredOptions(); + assert.strictEqual(options.needOplogUpdate, undefined); + assert.strictEqual(options.originOp, undefined); }); }); describe('Cold storage restoration (putObjectVersion)', () => { - it('should restore object with x-scal-s3-version-id header', done => { + it('should restore object with x-scal-s3-version-id header', async () => { const now = Date.now(); const archivedObjMD = { 'key': objectKey, @@ -297,32 +287,23 @@ describe('createAndStoreObject', () => { url: `/${bucketName}/${objectKey}`, }, Buffer.from('restored data', 'utf8')); - createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, + await createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, authInfo, canonicalID, null, request, false, null, - ['overhead'], log, 's3:ObjectCreated:Put', err => { - assert.ifError(err); - // Verify archive info was updated - const storedObjMD = getStoredObjectData(); - const options = getStoredOptions(); - assert(storedObjMD.archive.restoreCompletedAt, 'restoreCompletedAt should be set'); - assert(storedObjMD.archive.restoreWillExpireAt, 'restoreWillExpireAt should be set'); - assert.strictEqual(storedObjMD.archive.restoreRequestedDays, 7); - - // Verify metadata preserved - assert.strictEqual(storedObjMD['x-amz-meta-custom'], 'preserved-value'); - assert.deepStrictEqual(storedObjMD.tags, { 'tagkey': 'tagvalue' }); - - // Verify originOp set correctly - assert.strictEqual(storedObjMD.originOp, 's3:ObjectRestore:Completed'); - // PutObjectVersion must not use archived-overwrite oplog path. - assert.strictEqual(options.needOplogUpdate, undefined); - assert.strictEqual(options.originOp, undefined); - - done(); - }); + ['overhead'], log, 's3:ObjectCreated:Put'); + + const storedObjMD = getStoredObjectData(); + const options = getStoredOptions(); + assert(storedObjMD.archive.restoreCompletedAt, 'restoreCompletedAt should be set'); + assert(storedObjMD.archive.restoreWillExpireAt, 'restoreWillExpireAt should be set'); + assert.strictEqual(storedObjMD.archive.restoreRequestedDays, 7); + assert.strictEqual(storedObjMD['x-amz-meta-custom'], 'preserved-value'); + assert.deepStrictEqual(storedObjMD.tags, { 'tagkey': 'tagvalue' }); + assert.strictEqual(storedObjMD.originOp, 's3:ObjectRestore:Completed'); + assert.strictEqual(options.needOplogUpdate, undefined); + assert.strictEqual(options.originOp, undefined); }); - it('should preserve original etag for MPU restoration with different part count', done => { + it('should preserve original etag for MPU restoration with different part count', async () => { const archivedObjMD = { 'versionId': 'v123', 'content-md5': 'original-abc123-5', // Original had 5 parts @@ -344,26 +325,19 @@ describe('createAndStoreObject', () => { calculatedHash: 'restored-def456-3', // Restored with 3 parts }, Buffer.from('restored data', 'utf8')); - createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, + await createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, authInfo, canonicalID, null, request, false, null, - ['overhead'], log, 's3:ObjectCreated:Put', err => { - assert.ifError(err); - const storedObjMD = getStoredObjectData(); - - // Original etag should be preserved - assert.strictEqual(storedObjMD['content-md5'], 'original-abc123-5', - 'Original etag should be preserved'); - - // Restored ETag should be kept in restore headers for expiry processing. - assert(storedObjMD['x-amz-restore']['content-md5']); - assert.notStrictEqual(storedObjMD['x-amz-restore']['content-md5'], - storedObjMD['content-md5']); - - done(); - }); + ['overhead'], log, 's3:ObjectCreated:Put'); + + const storedObjMD = getStoredObjectData(); + assert.strictEqual(storedObjMD['content-md5'], 'original-abc123-5', + 'Original etag should be preserved'); + assert(storedObjMD['x-amz-restore']['content-md5']); + assert.notStrictEqual(storedObjMD['x-amz-restore']['content-md5'], + storedObjMD['content-md5']); }); - it('should preserve replication info during restoration', done => { + it('should preserve replication info during restoration', async () => { const replicationInfo = { status: 'COMPLETED', backends: [{ site: 'site1', status: 'COMPLETED' }], @@ -389,21 +363,18 @@ describe('createAndStoreObject', () => { url: `/${bucketName}/${objectKey}`, }, Buffer.from('restored', 'utf8')); - createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, + await createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, authInfo, canonicalID, null, request, false, null, - ['overhead'], log, 's3:ObjectCreated:Put', err => { - assert.ifError(err); - const storedObjMD = getStoredObjectData(); - assert.strictEqual(storedObjMD.replicationInfo.status, replicationInfo.status, - 'Replication status should be preserved'); - assert.deepStrictEqual(storedObjMD.replicationInfo.backends, replicationInfo.backends, - 'Replication backends should be preserved'); - - done(); - }); + ['overhead'], log, 's3:ObjectCreated:Put'); + + const storedObjMD = getStoredObjectData(); + assert.strictEqual(storedObjMD.replicationInfo.status, replicationInfo.status, + 'Replication status should be preserved'); + assert.deepStrictEqual(storedObjMD.replicationInfo.backends, replicationInfo.backends, + 'Replication backends should be preserved'); }); - it('should preserve legal hold during restoration', done => { + it('should preserve legal hold during restoration', async () => { const archivedObjMD = { 'versionId': 'v123', 'legalHold': true, @@ -424,19 +395,16 @@ describe('createAndStoreObject', () => { url: `/${bucketName}/${objectKey}`, }, Buffer.from('restored', 'utf8')); - createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, + await createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, authInfo, canonicalID, null, request, false, null, - ['overhead'], log, 's3:ObjectCreated:Put', err => { - assert.ifError(err); - const storedObjMD = getStoredObjectData(); - assert.strictEqual(storedObjMD.legalHold, true, - 'Legal hold should be preserved'); - - done(); - }); + ['overhead'], log, 's3:ObjectCreated:Put'); + + const storedObjMD = getStoredObjectData(); + assert.strictEqual(storedObjMD.legalHold, true, + 'Legal hold should be preserved'); }); - it('should preserve ACLs during restoration', done => { + it('should preserve ACLs during restoration', async () => { const acl = { 'Canned': '', 'FULL_CONTROL': ['canonical-id-1'], @@ -463,19 +431,16 @@ describe('createAndStoreObject', () => { url: `/${bucketName}/${objectKey}`, }, Buffer.from('restored', 'utf8')); - createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, + await createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, authInfo, canonicalID, null, request, false, null, - ['overhead'], log, 's3:ObjectCreated:Put', err => { - assert.ifError(err); - const storedObjMD = getStoredObjectData(); - assert.deepStrictEqual(storedObjMD.acl, acl, - 'ACLs should be preserved'); - - done(); - }); + ['overhead'], log, 's3:ObjectCreated:Put'); + + const storedObjMD = getStoredObjectData(); + assert.deepStrictEqual(storedObjMD.acl, acl, + 'ACLs should be preserved'); }); - it('should not preserve x-amz-meta-scal-s3-restore-attempt metadata', done => { + it('should not preserve x-amz-meta-scal-s3-restore-attempt metadata', async () => { const archivedObjMD = { 'versionId': 'v123', 'x-amz-meta-custom': 'keep-this', @@ -497,23 +462,20 @@ describe('createAndStoreObject', () => { url: `/${bucketName}/${objectKey}`, }, Buffer.from('restored', 'utf8')); - createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, + await createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, authInfo, canonicalID, null, request, false, null, - ['overhead'], log, 's3:ObjectCreated:Put', err => { - assert.ifError(err); - const storedObjMD = getStoredObjectData(); - assert.strictEqual(storedObjMD['x-amz-meta-custom'], 'keep-this', - 'Custom metadata should be preserved'); - assert.strictEqual(storedObjMD['x-amz-meta-scal-s3-restore-attempt'], undefined, - 'Restore attempt metadata should NOT be preserved'); - - done(); - }); + ['overhead'], log, 's3:ObjectCreated:Put'); + + const storedObjMD = getStoredObjectData(); + assert.strictEqual(storedObjMD['x-amz-meta-custom'], 'keep-this', + 'Custom metadata should be preserved'); + assert.strictEqual(storedObjMD['x-amz-meta-scal-s3-restore-attempt'], undefined, + 'Restore attempt metadata should NOT be preserved'); }); }); describe('MPU scenarios', () => { - it('should set oldReplayId when overwriting MPU object', done => { + it('should set oldReplayId when overwriting MPU object', async () => { const mpuObjMD = { 'uploadId': 'mpu-upload-123', 'content-md5': 'abc123', @@ -529,19 +491,17 @@ describe('createAndStoreObject', () => { url: `/${bucketName}/${objectKey}`, }, Buffer.from('new data', 'utf8')); - createAndStoreObject(bucketName, testBucket, objectKey, mpuObjMD, + await createAndStoreObject(bucketName, testBucket, objectKey, mpuObjMD, authInfo, canonicalID, null, request, false, null, - ['overhead'], log, 's3:ObjectCreated:Put', err => { - assert.ifError(err); - const options = getStoredOptions(); - assert.strictEqual(options.oldReplayId, 'mpu-upload-123'); - done(); - }); + ['overhead'], log, 's3:ObjectCreated:Put'); + + const options = getStoredOptions(); + assert.strictEqual(options.oldReplayId, 'mpu-upload-123'); }); }); describe('Azure compatibility', () => { - it('should preserve creation-time from existing object', done => { + it('should preserve creation-time from existing object', async () => { const existingObjMD = { 'creation-time': '2024-01-01T00:00:00.000Z', 'last-modified': '2024-02-01T00:00:00.000Z', @@ -557,17 +517,15 @@ describe('createAndStoreObject', () => { url: `/${bucketName}/${objectKey}`, }, Buffer.from('new data', 'utf8')); - createAndStoreObject(bucketName, testBucket, objectKey, existingObjMD, + await createAndStoreObject(bucketName, testBucket, objectKey, existingObjMD, authInfo, canonicalID, null, request, false, null, - ['overhead'], log, 's3:ObjectCreated:Put', err => { - assert.ifError(err); - const storedObjMD = getStoredObjectData(); - assert.strictEqual(storedObjMD['creation-time'], '2024-01-01T00:00:00.000Z'); - done(); - }); + ['overhead'], log, 's3:ObjectCreated:Put'); + + const storedObjMD = getStoredObjectData(); + assert.strictEqual(storedObjMD['creation-time'], '2024-01-01T00:00:00.000Z'); }); - it('should fall back to last-modified if creation-time missing', done => { + it('should fall back to last-modified if creation-time missing', async () => { const existingObjMD = { 'last-modified': '2024-02-01T00:00:00.000Z', }; @@ -582,19 +540,17 @@ describe('createAndStoreObject', () => { url: `/${bucketName}/${objectKey}`, }, Buffer.from('new data', 'utf8')); - createAndStoreObject(bucketName, testBucket, objectKey, existingObjMD, + await createAndStoreObject(bucketName, testBucket, objectKey, existingObjMD, authInfo, canonicalID, null, request, false, null, - ['overhead'], log, 's3:ObjectCreated:Put', err => { - assert.ifError(err); - const storedObjMD = getStoredObjectData(); - assert.strictEqual(storedObjMD['creation-time'], '2024-02-01T00:00:00.000Z'); - done(); - }); + ['overhead'], log, 's3:ObjectCreated:Put'); + + const storedObjMD = getStoredObjectData(); + assert.strictEqual(storedObjMD['creation-time'], '2024-02-01T00:00:00.000Z'); }); }); describe('Integration-sensitive restore behavior', () => { - it('should keep x-amz-meta-scal-version-id when restoring to ingestion location', done => { + it('should keep x-amz-meta-scal-version-id when restoring to ingestion location', async () => { sinon.stub(testBucket, 'isIngestionBucket').returns(true); sinon.spy(metadata, 'putObjectMD'); const putVersionId = 'restore-version-id'; @@ -615,14 +571,12 @@ describe('createAndStoreObject', () => { url: `/${bucketName}/${objectKey}`, }, Buffer.from('restored', 'utf8')); - createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, + await createAndStoreObject(bucketName, testBucket, objectKey, archivedObjMD, authInfo, canonicalID, null, request, false, null, - ['overhead'], log, 's3:ObjectCreated:Put', err => { - assert.ifError(err); - const storedObjMD = getStoredObjectData(); - assert.strictEqual(storedObjMD['x-amz-meta-scal-version-id'], putVersionId); - done(); - }); + ['overhead'], log, 's3:ObjectCreated:Put'); + + const storedObjMD = getStoredObjectData(); + assert.strictEqual(storedObjMD['x-amz-meta-scal-version-id'], putVersionId); }); }); }); From 1ac13d8c60db531ede5bacfe7f5d00af40a354c6 Mon Sep 17 00:00:00 2001 From: Maha Benzekri Date: Wed, 25 Mar 2026 10:00:53 +0100 Subject: [PATCH 7/7] fixup! modernize metadata helpers and async tests --- tests/functional/aws-node-sdk/test/utils/init.js | 4 ---- tests/unit/api/createAndStoreObject.js | 2 -- 2 files changed, 6 deletions(-) diff --git a/tests/functional/aws-node-sdk/test/utils/init.js b/tests/functional/aws-node-sdk/test/utils/init.js index 81ebf747fd..1cfd6a67a8 100644 --- a/tests/functional/aws-node-sdk/test/utils/init.js +++ b/tests/functional/aws-node-sdk/test/utils/init.js @@ -79,9 +79,7 @@ function getMetadata(bucketName, objectName, versionId, cb) { function fakeMetadataTransition(bucketName, objectName, versionId, cb) { const promise = (async () => { const objMD = await getMetadata(bucketName, objectName, versionId); - /* eslint-disable no-param-reassign */ objMD['x-amz-scal-transition-in-progress'] = true; - /* eslint-enable no-param-reassign */ await new Promise((resolve, reject) => metadata.putObjectMD( bucketName, objectName, @@ -107,11 +105,9 @@ function fakeMetadataTransition(bucketName, objectName, versionId, cb) { function fakeMetadataArchive(bucketName, objectName, versionId, archive, cb) { const promise = (async () => { const objMD = await getMetadata(bucketName, objectName, versionId); - /* eslint-disable no-param-reassign */ objMD['x-amz-storage-class'] = LOCATION_NAME_DMF; objMD.dataStoreName = LOCATION_NAME_DMF; objMD.archive = archive; - /* eslint-enable no-param-reassign */ await new Promise((resolve, reject) => metadata.putObjectMD( bucketName, objectName, diff --git a/tests/unit/api/createAndStoreObject.js b/tests/unit/api/createAndStoreObject.js index 25105595c5..1c28cce4f1 100644 --- a/tests/unit/api/createAndStoreObject.js +++ b/tests/unit/api/createAndStoreObject.js @@ -118,7 +118,6 @@ describe('createAndStoreObject', () => { const storedObjMD = getStoredObjectData(); assert.strictEqual(storedObjMD.bucketOwnerId, canonicalID); }); - }); }); describe('Delete marker creation', () => { @@ -580,4 +579,3 @@ describe('createAndStoreObject', () => { }); }); }); -