Skip to content

Commit

Permalink
Merge branch 'bugfix/BB-511' into q/8.6
Browse files Browse the repository at this point in the history
  • Loading branch information
@bert-e
bert-e committed Mar 27, 2024
2 parents a2b4001 + b6cb31d commit 37d5eb7
Show file tree
Hide file tree
Showing 5 changed files with 189 additions and 27 deletions.
33 changes: 13 additions & 20 deletions extensions/lifecycle/bucketProcessor/LifecycleBucketProcessor.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
const async = require('async');
const { Logger } = require('werelogs');
const { errors } = require('arsenal');
const { supportedLifecycleRules } = require('arsenal').constants;

const BackbeatProducer = require('../../../lib/BackbeatProducer');
const BackbeatTask = require('../../../lib/tasks/BackbeatTask');
Expand All @@ -15,6 +14,7 @@ const safeJsonParse = require('../util/safeJsonParse');
const ClientManager = require('../../../lib/clients/ClientManager');
const { authTypeAssumeRole } = require('../../../lib/constants');
const LocationStatusStream = require('../../utils/LocationStatusStream');
const { getFormattedSupportedLifecycleRules } = require('../util/rules');
const {
updateCircuitBreakerConfigForImplicitOutputQueue,
} = require('../../../lib/CircuitBreaker');
Expand Down Expand Up @@ -85,17 +85,7 @@ class LifecycleBucketProcessor {
this._producer = null;
this._kafkaBacklogMetrics = null;

this._supportedRulesObject = {};
this._hasSupportedRules = Array.isArray(supportedLifecycleRules) &&
supportedLifecycleRules.length > 0;

if (this._hasSupportedRules) {
supportedLifecycleRules.forEach(rule => {
this._supportedRulesObject[rule] = { enabled: true };
});
} else {
this._log.debug('no lifecycle rules enabled');
}
this._supportedRules = getFormattedSupportedLifecycleRules();

this._producerReady = false;
this._consumerReady = false;
Expand Down Expand Up @@ -184,7 +174,6 @@ class LifecycleBucketProcessor {
return {
producer: this._producer,
bootstrapList: this._repConfig.destination.bootstrapList,
enabledRules: this._supportedRulesObject,
bucketTasksTopic: this._lcConfig.bucketTasksTopic,
objectTasksTopic: this._lcConfig.objectTasksTopic,
transitionTasksTopic: this._lcConfig.transitionTasksTopic,
Expand Down Expand Up @@ -236,14 +225,18 @@ class LifecycleBucketProcessor {
* @return {Boolean} Whether the config should be processed
*/
_shouldProcessConfig(config) {
if (config.Rules.length === 0) {
this._log.debug('bucket lifecycle config has no rules to process', {
config,
const rulesEnabled = config.Rules.some(rule => {
if (rule.Status === 'Disabled') {
return false;
}
return Object.keys(rule).some(key => {
if (!this._supportedRules.includes(key)) {
return false;
}
return !rule[key].StorageClass || !this._pausedLocations.has(rule[key].StorageClass);
});
return false;
}

return this._hasSupportedRules;
});
return rulesEnabled;
}

/**
Expand Down
17 changes: 17 additions & 0 deletions extensions/lifecycle/util/rules.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ const { RulesReducer } = require('./RulesReducer');
const { lifecycleListing: { NON_CURRENT_TYPE, CURRENT_TYPE, ORPHAN_DM_TYPE } } = require('../../../lib/constants');

const { s3middleware } = require('arsenal');
const { supportedLifecycleRules } = require('arsenal').constants;
const { scaleMsPerDay } = s3middleware.objectUtils;

// Default max AWS limit is 1000 for both list objects and list object versions
Expand Down Expand Up @@ -193,6 +194,22 @@ function rulesToParams(versioningStatus, currentDate, bucketLCRules, bucketData,
return _getParamsFromListings(bucketName, currentDate, listings, options);
}

/**
* Formats the array of supported lifecycle rules in Arsenal so that
* they correspond to how they are written in the lifecycle configuration.
* @param {string[]} supportedLifecycleRules list of supported lifecycle rules
* @returns {string[]} formatted supported lifecycle rules
*/
function getFormattedSupportedLifecycleRules() {
return supportedLifecycleRules.map(rule => {
if (rule === 'transitions') {
return 'Transition';
}
return rule.charAt(0).toUpperCase() + rule.slice(1);
});
}

module.exports = {
rulesToParams,
getFormattedSupportedLifecycleRules,
};
1 change: 0 additions & 1 deletion tests/functional/lifecycle/LifecycleTask.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -429,7 +429,6 @@ class LifecycleBucketProcessorMock {
return {
producer: this._producer,
removeBucketFromQueued: () => {},
enabledRules: this._lcConfig.rules,
// Corresponds to the default endpoint in the cloudserver config.
bootstrapList: [{ site: 'us-east-2', type: 'aws_s3' }],
s3Endpoint: s3config.endpoint,
Expand Down
159 changes: 159 additions & 0 deletions tests/unit/lifecycle/LifecycleBucketProcessor.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,4 +38,163 @@ describe('Lifecycle Bucket Processor', () => {
lbp._resumeServiceForLocation('new-location');
assert(!lbp._pausedLocations.has('new-location'));
});

describe('_shouldProcessConfig', () => {
[
{
title: 'should return false if no lifecycle rules were set',
lcConfig: {
Rules: [],
},
supportedRules: [
'Expiration',
'Noncurrentversionexpiration',
'AbortincompletemultipartUpload',
'Transition',
'Noncurrentversiontransition',
],
pauseLocations: [],
expected: false,
}, {
title: 'should return false if all lifecyle rules are disabled',
lcConfig: {
Rules: [
{
Status: 'Disabled',
AbortIncompleteMultipartUpload: {
DaysAfterInitiation: 7
},
NoncurrentVersionExpiration: {
NoncurrentDays: 7
},
Expiration: {
Days: 1
},
ID: '64bf6794-55fd-43f4-8d44-54a54cd2bf09'
},
{
Status: 'Disabled',
Transition: {
Days: 10,
StorageClass: 'azure'
},
ID: 'dac36d89-0005-4c78-8e00-7e9ace06a9c4'
}
],
},
supportedRules: [
'Expiration',
'Noncurrentversionexpiration',
'AbortincompletemultipartUpload',
'Transition',
'Noncurrentversiontransition',
],
pauseLocations: [],
expected: false,
}, {
title: 'should return true if at least one lifecycle rule is enabled',
lcConfig: {
Rules: [
{
Status: 'Disabled',
Expiration: {
Days: 1
},
ID: '64bf6794-55fd-43f4-8d44-54a54cd2bf09'
},
{
Status: 'Enabled',
Transition: {
Days: 10,
StorageClass: 'azure'
},
ID: 'dac36d89-0005-4c78-8e00-7e9ace06a9c4'
}
],
},
supportedRules: [
'Expiration',
'Noncurrentversionexpiration',
'AbortincompletemultipartUpload',
'Transition',
'Noncurrentversiontransition',
],
pauseLocations: [],
expected: true,
}, {
title: 'should return false if transition rule\'s location is paused',
lcConfig: {
Rules: [
{
Status: 'Enabled',
Transition: {
Days: 10,
StorageClass: 'azure'
},
ID: 'dac36d89-0005-4c78-8e00-7e9ace06a9c4'
}
],
},
supportedRules: [
'Expiration',
'Noncurrentversionexpiration',
'AbortincompletemultipartUpload',
'Transition',
'Noncurrentversiontransition',
],
pauseLocations: ['azure'],
expected: false,
}, {
title: 'should return false if non current transition rule\'s location is paused',
lcConfig: {
Rules: [
{
Status: 'Enabled',
NoncurrentVersionTransition: {
NoncurrentDays: 10,
StorageClass: 'azure'
},
ID: 'dac36d89-0005-4c78-8e00-7e9ace06a9c4'
}
],
},
supportedRules: [
'Expiration',
'Noncurrentversionexpiration',
'AbortincompletemultipartUpload',
'Transition',
'Noncurrentversiontransition',
],
pauseLocations: ['azure'],
expected: false,
}, {
title: 'should return false if no supported rule is enabled',
lcConfig: {
Rules: [
{
Status: 'Enabled',
Transition: {
Days: 10,
StorageClass: 'azure'
},
ID: 'dac36d89-0005-4c78-8e00-7e9ace06a9c4'
}
],
},
supportedRules: [
'Expiration',
'Noncurrentversionexpiration',
'AbortincompletemultipartUpload',
],
pauseLocations: [],
expected: false,
},
].forEach(params => {
it(params.title, () => {
lbp._supportedRules = params.supportedRules;
lbp._pausedLocations = new Set(params.pauseLocations);
assert.strictEqual(lbp._shouldProcessConfig(params.lcConfig), params.expected);
});
});
});
});
6 changes: 0 additions & 6 deletions tests/unit/lifecycle/LifecycleTask.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,12 +81,6 @@ const NON_CURRENT_DELETE_MARKER = {
const lp = {
getStateVars: () => (
{
enabledRules: {
expiration: { enabled: true },
transitions: { enabled: true },
noncurrentVersionExpiration: { enabled: true },
abortIncompleteMultipartUpload: { enabled: true },
},
ncvHeap: new Map(),
lcOptions: timeOptions,
}
Expand Down

0 comments on commit 37d5eb7

Please sign in to comment.