From 07c43735503c663f524d6328dd5e220b1ca80cd8 Mon Sep 17 00:00:00 2001 From: williamlardier Date: Mon, 6 May 2024 10:40:29 +0200 Subject: [PATCH] testing deletes --- tests/ctst/common/common.ts | 87 ++++++++++++++- tests/ctst/common/utils.ts | 17 +++ tests/ctst/features/quotas/Quotas.feature | 120 ++++++++++++++++----- tests/ctst/package.json | 2 +- tests/ctst/steps/bucket-policies/common.ts | 8 -- tests/ctst/steps/dmf.ts | 24 +++++ tests/ctst/steps/quotas/quotas.ts | 23 ++-- tests/ctst/steps/utils/utils.ts | 4 +- tests/ctst/yarn.lock | 6 +- 9 files changed, 238 insertions(+), 53 deletions(-) create mode 100644 tests/ctst/steps/dmf.ts diff --git a/tests/ctst/common/common.ts b/tests/ctst/common/common.ts index c845530ff8..9670a70daa 100644 --- a/tests/ctst/common/common.ts +++ b/tests/ctst/common/common.ts @@ -8,6 +8,35 @@ import { ActionPermissionsType } from 'steps/bucket-policies/utils'; setDefaultTimeout(Constants.DEFAULT_TIMEOUT); +/** + * @param {Zenko} this world object + * @param {string} objectName object name + * @returns {string} the object name based on the backend flakyness + */ +function getObjectNameWithBackendFlakiness(this: Zenko, objectName: string) { + let objectNameFinal; + const backendFlakinessRetryNumber = this.getSaved('backendFlakinessRetryNumber'); + const backendFlakiness = this.getSaved('backendFlakiness'); + + if (!backendFlakiness || !backendFlakinessRetryNumber || !objectName) { + return objectName; + } + + switch (backendFlakiness) { + case 'command': + objectNameFinal = `${objectName}.scal-retry-command-${backendFlakinessRetryNumber}`; + break; + case 'archive': + case 'restore': + objectNameFinal = `${objectName}.scal-retry-${backendFlakiness}-job-${backendFlakinessRetryNumber}`; + break; + default: + process.stdout.write(`Unknown backend flakyness ${backendFlakiness}\n`); + return objectName; + } + return objectNameFinal; +} + async function getTopicsOffsets(topics: string[], kafkaAdmin: Admin) { const offsets = []; for (const topic of topics) { @@ -90,8 +119,8 @@ Given('an object {string} that {string}', async function (this: Zenko, objectNam } }); -When('the user tries to perform the current S3 action on the bucket {string} times with a {string} ms delay', - async function (this: Zenko, numberOfRuns: string, delay: string) { +When('the user tries to perform the current S3 action on the bucket {int} times with a {int} ms delay', + async function (this: Zenko, numberOfRuns: number, delay: number) { this.setAuthMode('test_identity'); const action = { ...this.getSaved('currentAction'), @@ -104,7 +133,7 @@ When('the user tries to perform the current S3 action on the bucket {string} tim action.action = action.action.replace('Version', ''); this.addToSaved('currentAction', action); } - for (let i = 0; i < Number(numberOfRuns); i++) { + for (let i = 0; i < numberOfRuns; i++) { // For repeated WRITE actions, we want to change the object name if (action.action === 'PutObject') { this.addToSaved('objectName', `objectrepeat-${Utils.randomString()}`); @@ -116,7 +145,7 @@ When('the user tries to perform the current S3 action on the bucket {string} tim // stop at any error, the error will be evaluated in a separated step return; } - await Utils.sleep(Number(delay)); + await Utils.sleep(delay); } }); @@ -144,3 +173,53 @@ Then('the operation finished without error', function (this: Zenko) { this.cleanupEntity(); assert.strictEqual(!!this.getResult().err, false); }); + +When('i restore object {string} for {int} days', async function (this: Zenko, objectName: string, days: number) { + const objName = getObjectNameWithBackendFlakiness.call(this, objectName) || this.getSaved('objectName'); + this.resetCommand(); + this.addCommandParameter({ bucket: this.getSaved('bucketName') }); + this.addCommandParameter({ key: objName }); + const versionId = this.getSaved>('createdObjects')?.get(objName); + if (versionId) { + this.addCommandParameter({ versionId }); + } + this.addCommandParameter({ restoreRequest: `Days=${days}` }); + await S3.restoreObject(this.getCommandParameters()); +}); + +Given('an upload size of {int} B for the object {string}', async function ( + this: Zenko, + size: number, + objectName: string +) { + this.addToSaved('objectSize', size); + if (this.getSaved('preExistingObject')) { + if (objectName) { + this.addToSaved('objectName', objectName); + } else { + this.addToSaved('objectName', `object-${Utils.randomString()}`); + } + await putObject(this, this.getSaved('objectName')); + } +}); + +When('I PUT an object with size {int}', async function (this: Zenko, size: number) { + if (size > 0) { + this.addToSaved('objectSize', size); + } + this.addToSaved('objectName', `object-${Utils.randomString()}`); + const result = await putObject(this, this.getSaved('objectName')); + this.setResult(result); +}); + +When('i delete object {string}', async function (this: Zenko, objectName: string) { + const objName = objectName || this.getSaved('objectName'); + this.resetCommand(); + this.addCommandParameter({ bucket: this.getSaved('bucketName') }); + this.addCommandParameter({ key: objName }); + const versionId = this.getSaved>('createdObjects')?.get(objName); + if (versionId) { + this.addCommandParameter({ versionId }); + } + await S3.deleteObject(this.getCommandParameters()); +}); diff --git a/tests/ctst/common/utils.ts b/tests/ctst/common/utils.ts index a0b4d758e3..cfa3c2e773 100644 --- a/tests/ctst/common/utils.ts +++ b/tests/ctst/common/utils.ts @@ -1,3 +1,4 @@ +import { exec } from 'child_process'; import { Utils, } from 'cli-testing'; @@ -87,3 +88,19 @@ export const s3FunctionExtraParams: { [key: string]: Record[] } }), }], }; + +/** + * Executes a shell command and return it as a Promise. + * @param {string} cmd The command to execute + * @return {Promise} the command output + */ +export function execShellCommand(cmd: string): Promise { + return new Promise((resolve, reject) => { + exec(cmd, (error, stdout, stderr) => { + if (error) { + reject(error); + } + resolve(stdout || stderr); + }); + }); +} diff --git a/tests/ctst/features/quotas/Quotas.feature b/tests/ctst/features/quotas/Quotas.feature index 134b09cb01..148fd57f57 100644 --- a/tests/ctst/features/quotas/Quotas.feature +++ b/tests/ctst/features/quotas/Quotas.feature @@ -2,11 +2,6 @@ Feature: Quota Management for APIs This feature ensures that quotas are correctly set and honored for different APIs. - # TODO: - # ONGOING - Authz of Quota APIs: only storage manager should be allowed to edit by default - # ONGOING - Quota evaluation: should work for all types of identities (roles, accounts, users) - # ONGOING - Inflights must be properly handled in a test pushing in 2 steps - # ONGOING - Same as above but should not wait, for Restore Object @2.6.0 @PreMerge @Quotas @@ -14,14 +9,14 @@ Feature: Quota Management for APIs @DataWrite Scenario Outline: Quotas are evaluated during write operations Given an action "" - And an upload size of "" B + And an upload size of B for the object "" And a "STORAGE_MANAGER" type - And a bucket quota set to "" B - And an account quota set to "" B + And a bucket quota set to B + And an account quota set to B And a "" type And an environment setup for the API And an "existing" IAM Policy that "applies" with "ALLOW" effect for the current API - And the user tries to perform the current S3 action on the bucket "20" times with a "400" ms delay + When the user tries to perform the current S3 action on the bucket 20 times with a 400 ms delay Then the API should "" with "" Examples: @@ -62,29 +57,98 @@ Feature: Quota Management for APIs @Quotas @CronJob @Restore - Scenario Outline: Object restoration implements strict quotas - Given an action "" - And an upload size of "" B + Scenario Outline: Object restoration (fake) implements strict quotas + Given an action "RestoreObject" + And an upload size of B for the object "" And a "STORAGE_MANAGER" type - And a bucket quota set to "" B - And an account quota set to "" B + And a bucket quota set to B + And an account quota set to B And a "" type And an environment setup for the API And an "existing" IAM Policy that "applies" with "ALLOW" effect for the current API - And the user tries to perform the current S3 action on the bucket "1" times with a "0" ms delay + And the user tries to perform the current S3 action on the bucket 1 times with a 0 ms delay Then the API should "" with "" Examples: - | action | uploadSize | bucketQuota | accountQuota | userType | result | expectedError | - | RestoreObject | 100 | 0 | 0 | ACCOUNT | succeed | | - | RestoreObject | 100 | 99 | 0 | ACCOUNT | fail | QuotaExceeded | - | RestoreObject | 100 | 0 | 99 | ACCOUNT | fail | QuotaExceeded | - | RestoreObject | 100 | 99 | 99 | ACCOUNT | fail | QuotaExceeded | - | RestoreObject | 100 | 101 | 101 | ACCOUNT | succeed | | - | RestoreObject | 100 | 0 | 0 | IAM_USER | succeed | | - | RestoreObject | 100 | 99 | 0 | IAM_USER | fail | QuotaExceeded | - | RestoreObject | 100 | 0 | 99 | IAM_USER | fail | QuotaExceeded | - | RestoreObject | 100 | 99 | 99 | IAM_USER | fail | QuotaExceeded | - | RestoreObject | 100 | 101 | 101 | IAM_USER | succeed | | - # TODO test with real restores - # TODO test the deletion + | uploadSize | bucketQuota | accountQuota | userType | result | expectedError | + | 100 | 0 | 0 | ACCOUNT | succeed | | + | 100 | 99 | 0 | ACCOUNT | fail | QuotaExceeded | + | 100 | 0 | 99 | ACCOUNT | fail | QuotaExceeded | + | 100 | 99 | 99 | ACCOUNT | fail | QuotaExceeded | + | 100 | 101 | 101 | ACCOUNT | succeed | | + | 100 | 0 | 0 | IAM_USER | succeed | | + | 100 | 99 | 0 | IAM_USER | fail | QuotaExceeded | + | 100 | 0 | 99 | IAM_USER | fail | QuotaExceeded | + | 100 | 99 | 99 | IAM_USER | fail | QuotaExceeded | + | 100 | 101 | 101 | IAM_USER | succeed | | + + @2.6.0 + @PreMerge + @Quotas + @CronJob + @DataDeletion + @NonVersioned + Scenario Outline: Quotas are affected by deletion operations + Given an action "DeleteObject" + # First set a big quota to enable the inflights + # and ensure the initial PUT is accepted + And a "STORAGE_MANAGER" type + And a bucket quota set to 10000 B + And an account quota set to 10000 B + # Put an object: fill fill the quota directly + And an upload size of 1000 B for the object "obj-1" + # Set a small quota + And a bucket quota set to B + And an account quota set to B + And a "" type + And an environment setup for the API + And an "existing" IAM Policy that "applies" with "ALLOW" effect for the current API + When I PUT an object with size + Then the API should "fail" with "QuotaExceeded" + When i delete object "obj-1" + And I wait 3 seconds + And I PUT an object with size + Then the API should "succeed" with "" + + Examples: + | uploadSize | bucketQuota | accountQuota | userType | + | 100 | 99 | 0 | ACCOUNT | + | 100 | 0 | 99 | ACCOUNT | + | 100 | 99 | 99 | ACCOUNT | + | 100 | 99 | 0 | IAM_USER | + | 100 | 0 | 99 | IAM_USER | + | 100 | 99 | 99 | IAM_USER | +# @2.6.0 +# @PreMerge +# @Quotas +# @Restore +# @Dmf +# @ColdStorage +# Scenario Outline: Object restoration implements strict quotas +# Given an action "" +# And a flaky backend that will require retries for "restore" +# And an upload size of "" B +# And a "STORAGE_MANAGER" type +# And a bucket quota set to "" B +# And an account quota set to "" B +# And a "" type +# And an environment setup for the API +# And an "existing" IAM Policy that "applies" with "ALLOW" effect for the current API +# And a transition workflow to "e2e-cold" location +# When i restore object "" for 5 days +# Then the API should "" with "" + +# Examples: +# | action | uploadSize | bucketQuota | accountQuota | userType | result | expectedError | retryNumber | +# | RestoreObject | 100 | 0 | 0 | ACCOUNT | succeed | | 3 | +# | RestoreObject | 100 | 99 | 0 | ACCOUNT | fail | QuotaExceeded | 3 | +# | RestoreObject | 100 | 0 | 99 | ACCOUNT | fail | QuotaExceeded | 3 | +# | RestoreObject | 100 | 99 | 99 | ACCOUNT | fail | QuotaExceeded | 3 | +# | RestoreObject | 100 | 101 | 101 | ACCOUNT | succeed | | 3 | +# | RestoreObject | 100 | 0 | 0 | IAM_USER | succeed | | 3 | +# | RestoreObject | 100 | 99 | 0 | IAM_USER | fail | QuotaExceeded | 3 | +# | RestoreObject | 100 | 0 | 99 | IAM_USER | fail | QuotaExceeded | 3 | +# | RestoreObject | 100 | 99 | 99 | IAM_USER | fail | QuotaExceeded | 3 | +# | RestoreObject | 100 | 101 | 101 | IAM_USER | succeed | | 3 | +# TODO test with real restores +# TODO test the deletion diff --git a/tests/ctst/package.json b/tests/ctst/package.json index 8c3dff8c66..a53c3e03e5 100644 --- a/tests/ctst/package.json +++ b/tests/ctst/package.json @@ -23,7 +23,7 @@ "@typescript-eslint/eslint-plugin": "^5.45.0", "@typescript-eslint/parser": "^5.45.0", "babel-jest": "^29.3.1", - "cli-testing": "github:scality/cli-testing.git#cc24312c636a50059295d36ad12ab587b96fbcda", + "cli-testing": "github:scality/cli-testing.git#7fa94cea9369944e53cc2db454d40a103abf28c5", "eslint": "^8.28.0" }, "scripts": { diff --git a/tests/ctst/steps/bucket-policies/common.ts b/tests/ctst/steps/bucket-policies/common.ts index f5515afa8c..bccbcf0b21 100644 --- a/tests/ctst/steps/bucket-policies/common.ts +++ b/tests/ctst/steps/bucket-policies/common.ts @@ -46,14 +46,6 @@ Given('an action {string}', function (this: Zenko, apiName: string) { } }); -Given('an upload size of {string} B', async function (this: Zenko, size: string) { - this.addToSaved('objectSize', parseInt(size, 10)); - if (this.getSaved('preExistingObject')) { - this.addToSaved('objectName', `objectforbptests-${Utils.randomString()}`); - await putObject(this, this.getSaved('objectName')); - } -}); - Given('an existing bucket prepared for the action', async function (this: Zenko) { await createBucketWithConfiguration(this, this.getSaved('bucketName'), diff --git a/tests/ctst/steps/dmf.ts b/tests/ctst/steps/dmf.ts new file mode 100644 index 0000000000..4e3501c001 --- /dev/null +++ b/tests/ctst/steps/dmf.ts @@ -0,0 +1,24 @@ +import { Given, setDefaultTimeout, After } from '@cucumber/cucumber'; +import assert from 'assert'; +import { Constants } from 'cli-testing'; +import { execShellCommand } from 'common/utils'; +import Zenko from 'world/Zenko'; + +setDefaultTimeout(Constants.DEFAULT_TIMEOUT); + +async function cleanDmfVolume() { + await execShellCommand('rm -rf /cold-data/*'); +} + +Given('a flaky backend that will require {int} retries for {string}', + function (this: Zenko, retryNumber: number, op: string) { + assert(['restore', 'archive', 'command'].includes(op), `Invalid operation ${op}`); + assert(retryNumber > 0, `Invalid retry number ${retryNumber}`); + + this.addToSaved('backendFlakinessRetryNumber', retryNumber); + this.addToSaved('backendFlakiness', op); + }); + +After({ tags: '@Dmf' }, async () => { + await cleanDmfVolume(); +}); diff --git a/tests/ctst/steps/quotas/quotas.ts b/tests/ctst/steps/quotas/quotas.ts index e5da88563f..b866a09d15 100644 --- a/tests/ctst/steps/quotas/quotas.ts +++ b/tests/ctst/steps/quotas/quotas.ts @@ -2,7 +2,7 @@ import fs from 'fs'; import lockFile from 'proper-lockfile'; import { createHash } from 'crypto'; -import { Given, Before } from '@cucumber/cucumber'; +import { Given, Before, When } from '@cucumber/cucumber'; import Zenko from '../../world/Zenko'; import { Scality, Command, CacheHelper, Constants, Utils } from 'cli-testing'; import { createJobAndWaitForCompletion } from 'steps/utils/kubernetes'; @@ -46,12 +46,15 @@ Before({tags: '@Quotas'}, async function ({ gherkinDocument, pickle }) { } if (releaseLock) { + const isBucketNonVersioned = gherkinDocument.feature?.tags?.find( + tag => tag.name === 'NonVersioned') === undefined; for (const scenario of gherkinDocument.feature?.children || []) { for (const example of scenario.scenario?.examples || []) { for (const values of example.tableBody || []) { const scenarioWithExampleID = hashStringAndKeepFirst40Characters(`${values.id}`); await world.createAccount(scenarioWithExampleID); - await createBucketWithConfiguration(world, scenarioWithExampleID, 'with'); + await createBucketWithConfiguration(world, scenarioWithExampleID, + isBucketNonVersioned ? '' : 'with'); await putObject(world); output[scenarioWithExampleID] = { AccessKey: CacheHelper.parameters?.AccessKey || Constants.DEFAULT_ACCESS_KEY, @@ -90,12 +93,12 @@ Before({tags: '@Quotas'}, async function ({ gherkinDocument, pickle }) { world.addToSaved('bucketName', key); }); -Given('a bucket quota set to {string} B', async function (this: Zenko, quota: string) { - if (quota === '0') { +Given('a bucket quota set to {int} B', async function (this: Zenko, quota: number) { + if (quota === 0) { return; } this.addCommandParameter({ - quota, + quota: String(quota), }); this.addCommandParameter({ bucket: this.getSaved('bucketName'), @@ -114,12 +117,12 @@ Given('a bucket quota set to {string} B', async function (this: Zenko, quota: st } }); -Given('an account quota set to {string} B', async function (this: Zenko, quota: string) { - if (quota === '0') { +Given('an account quota set to {int} B', async function (this: Zenko, quota: number) { + if (quota === 0) { return; } this.addCommandParameter({ - quotaMax: quota, + quotaMax: String(quota), }); const result: Command = await Scality.updateAccountQuota( this.parameters, @@ -134,3 +137,7 @@ Given('an account quota set to {string} B', async function (this: Zenko, quota: throw new Error(result.err); } }); + +When('I wait {int} seconds', async function (this: Zenko, seconds: number) { + await Utils.sleep(seconds * 1000); +}); diff --git a/tests/ctst/steps/utils/utils.ts b/tests/ctst/steps/utils/utils.ts index 04a41c2dd0..10e3d9d15e 100644 --- a/tests/ctst/steps/utils/utils.ts +++ b/tests/ctst/steps/utils/utils.ts @@ -189,10 +189,12 @@ async function putObject(context: Zenko, objectName?: string) { await uploadSetup(context, 'PutObject'); context.addCommandParameter({ key: context.getSaved('objectName') }); context.addCommandParameter({ bucket: context.getSaved('bucketName') }); + const result = await S3.putObject(context.getCommandParameters()); context.addToSaved('versionId', extractPropertyFromResults( - await S3.putObject(context.getCommandParameters()), 'VersionId' + result, 'VersionId' )); await uploadTeardown(context, 'PutObject'); + return result; } function getAuthorizationConfiguration(context: Zenko): AuthorizationConfiguration { diff --git a/tests/ctst/yarn.lock b/tests/ctst/yarn.lock index 6ec90b2f21..156c667174 100644 --- a/tests/ctst/yarn.lock +++ b/tests/ctst/yarn.lock @@ -3546,9 +3546,9 @@ cli-table3@^0.6.0: optionalDependencies: "@colors/colors" "1.5.0" -"cli-testing@github:scality/cli-testing.git#cc24312c636a50059295d36ad12ab587b96fbcda": - version "0.4.1" - resolved "git+ssh://git@github.com/scality/cli-testing.git#cc24312c636a50059295d36ad12ab587b96fbcda" +"cli-testing@github:scality/cli-testing.git#7fa94cea9369944e53cc2db454d40a103abf28c5": + version "0.4.3" + resolved "git+ssh://git@github.com/scality/cli-testing.git#7fa94cea9369944e53cc2db454d40a103abf28c5" dependencies: "@aws-crypto/sha256-universal" "^5.2.0" "@aws-sdk/client-iam" "^3.484.0"