Skip to content

Commit

Permalink
Prune execution data when more than cofnfigured limit
Browse files Browse the repository at this point in the history
  • Loading branch information
ahsanv committed Feb 17, 2023
1 parent 9a1e7b5 commit ee4fa58
Show file tree
Hide file tree
Showing 2 changed files with 30 additions and 1 deletion.
21 changes: 20 additions & 1 deletion packages/cli/src/WorkflowExecuteAdditionalData.ts
Original file line number Diff line number Diff line change
Expand Up @@ -202,14 +202,33 @@ async function pruneExecutionData(this: WorkflowHooks): Promise<void> {
throttling = true;
const timeout = config.getEnv('executions.pruneDataTimeout'); // in seconds
const maxAge = config.getEnv('executions.pruneDataMaxAge'); // in h
const maxCount = config.getEnv('executions.pruneDataCountMax');
const date = new Date(); // today
date.setHours(date.getHours() - maxAge);

// date reformatting needed - see https://github.com/typeorm/typeorm/issues/2286
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
const utcDate = DateUtils.mixedDateToUtcDatetimeString(date);

const toPrune = { stoppedAt: LessThanOrEqual(utcDate) };
let toPrune: Array<any> = [
{ stoppedAt: LessThanOrEqual(utcDate) },
];

if (maxCount > 0) {
const executions = await Db.collections.Execution.find({
select: ['id'],
skip: maxCount,
take: 1,
order: { id: 'DESC' },
});

if (executions[0]) {
toPrune.push({
id: LessThanOrEqual(executions[0].id),
})
}
}

const isBinaryModeDefaultMode = config.getEnv('binaryDataManager.mode') === 'default';
try {
const executions = isBinaryModeDefaultMode
Expand Down
10 changes: 10 additions & 0 deletions packages/cli/src/config/schema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -325,6 +325,16 @@ export const schema = {
default: 3600,
env: 'EXECUTIONS_DATA_PRUNE_TIMEOUT',
},

// Additional pruning option to delete executions if total count exceeds the configured max.
// Deletes the oldest entries first
// Default is 0 = No limit
pruneDataCountMax: {
doc: 'Maximum number of executions to keep in DB. Default 0 = no limit',
format: Number,
default: 0,
env: 'EXECUTIONS_DATA_PRUNE_COUNT_MAX',
},
},

queue: {
Expand Down

0 comments on commit ee4fa58

Please # to comment.