Skip to content

Commit 2af3734

Browse files
vlbeevlbee
committed
Add DLQ alarms
1 parent f3b0fd6 commit 2af3734

2 files changed

Lines changed: 237 additions & 6 deletions

File tree

‎cdk/lib/__snapshots__/liveactivities.test.ts.snap‎

Lines changed: 195 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,12 @@ exports[`The LiveActivities stack matches the snapshot for CODE 1`] = `
44
{
55
"Metadata": {
66
"gu:cdk:constructs": [
7+
"GuAlarm",
78
"GuDistributionBucketParameter",
89
"GuLambdaFunction",
10+
"GuAlarm",
911
"GuLambdaFunction",
12+
"GuAlarm",
1013
"GuLambdaFunction",
1114
],
1215
"gu:cdk:version": "TEST",
@@ -22,7 +25,7 @@ exports[`The LiveActivities stack matches the snapshot for CODE 1`] = `
2225
"BroadcastDlq8DE17B20": {
2326
"DeletionPolicy": "Delete",
2427
"Properties": {
25-
"MessageRetentionPeriod": 604800,
28+
"MessageRetentionPeriod": 1209600,
2629
"QueueName": "liveactivities-broadcast-dlq-CODE",
2730
"Tags": [
2831
{
@@ -116,7 +119,7 @@ exports[`The LiveActivities stack matches the snapshot for CODE 1`] = `
116119
"ChannelCleanerDlq9CA723C0": {
117120
"DeletionPolicy": "Delete",
118121
"Properties": {
119-
"MessageRetentionPeriod": 604800,
122+
"MessageRetentionPeriod": 1209600,
120123
"QueueName": "liveactivities-channel-cleaner-dlq-CODE",
121124
"Tags": [
122125
{
@@ -199,7 +202,7 @@ exports[`The LiveActivities stack matches the snapshot for CODE 1`] = `
199202
"ChannelManagerDlq58D33B75": {
200203
"DeletionPolicy": "Delete",
201204
"Properties": {
202-
"MessageRetentionPeriod": 604800,
205+
"MessageRetentionPeriod": 1209600,
203206
"QueueName": "liveactivities-channel-manager-dlq-CODE",
204207
"Tags": [
205208
{
@@ -380,6 +383,69 @@ exports[`The LiveActivities stack matches the snapshot for CODE 1`] = `
380383
},
381384
"Type": "AWS::Lambda::Permission",
382385
},
386+
"liveactivitiesbroadcastdlqCODEAlarm3AB74C54": {
387+
"Properties": {
388+
"ActionsEnabled": true,
389+
"AlarmActions": [
390+
{
391+
"Fn::Join": [
392+
"",
393+
[
394+
"arn:aws:sns:",
395+
{
396+
"Ref": "AWS::Region",
397+
},
398+
":",
399+
{
400+
"Ref": "AWS::AccountId",
401+
},
402+
":mobile-server-side",
403+
],
404+
],
405+
},
406+
],
407+
"AlarmDescription": "More than 1 message in liveactivities-broadcast-dlq-CODE",
408+
"AlarmName": "liveactivities-broadcast-dlq-CODE-alarm",
409+
"ComparisonOperator": "GreaterThanOrEqualToThreshold",
410+
"Dimensions": [
411+
{
412+
"Name": "QueueName",
413+
"Value": {
414+
"Fn::GetAtt": [
415+
"BroadcastDlq8DE17B20",
416+
"QueueName",
417+
],
418+
},
419+
},
420+
],
421+
"EvaluationPeriods": 1,
422+
"MetricName": "ApproximateNumberOfMessagesVisible",
423+
"Namespace": "AWS/SQS",
424+
"Period": 60,
425+
"Statistic": "Sum",
426+
"Tags": [
427+
{
428+
"Key": "gu:cdk:version",
429+
"Value": "TEST",
430+
},
431+
{
432+
"Key": "gu:repo",
433+
"Value": "guardian/mobile-n10n",
434+
},
435+
{
436+
"Key": "Stack",
437+
"Value": "mobile-notifications",
438+
},
439+
{
440+
"Key": "Stage",
441+
"Value": "CODE",
442+
},
443+
],
444+
"Threshold": 1,
445+
"TreatMissingData": "notBreaching",
446+
},
447+
"Type": "AWS::CloudWatch::Alarm",
448+
},
383449
"liveactivitiesbroadcastlambdaC288C367": {
384450
"DependsOn": [
385451
"liveactivitiesbroadcastlambdaServiceRoleDefaultPolicyB2BCF85A",
@@ -665,6 +731,69 @@ exports[`The LiveActivities stack matches the snapshot for CODE 1`] = `
665731
},
666732
"Type": "AWS::IAM::Policy",
667733
},
734+
"liveactivitieschannelcleanerdlqCODEAlarm3EA37F66": {
735+
"Properties": {
736+
"ActionsEnabled": true,
737+
"AlarmActions": [
738+
{
739+
"Fn::Join": [
740+
"",
741+
[
742+
"arn:aws:sns:",
743+
{
744+
"Ref": "AWS::Region",
745+
},
746+
":",
747+
{
748+
"Ref": "AWS::AccountId",
749+
},
750+
":mobile-server-side",
751+
],
752+
],
753+
},
754+
],
755+
"AlarmDescription": "More than 1 message in liveactivities-channel-cleaner-dlq-CODE",
756+
"AlarmName": "liveactivities-channel-cleaner-dlq-CODE-alarm",
757+
"ComparisonOperator": "GreaterThanOrEqualToThreshold",
758+
"Dimensions": [
759+
{
760+
"Name": "QueueName",
761+
"Value": {
762+
"Fn::GetAtt": [
763+
"ChannelCleanerDlq9CA723C0",
764+
"QueueName",
765+
],
766+
},
767+
},
768+
],
769+
"EvaluationPeriods": 1,
770+
"MetricName": "ApproximateNumberOfMessagesVisible",
771+
"Namespace": "AWS/SQS",
772+
"Period": 60,
773+
"Statistic": "Sum",
774+
"Tags": [
775+
{
776+
"Key": "gu:cdk:version",
777+
"Value": "TEST",
778+
},
779+
{
780+
"Key": "gu:repo",
781+
"Value": "guardian/mobile-n10n",
782+
},
783+
{
784+
"Key": "Stack",
785+
"Value": "mobile-notifications",
786+
},
787+
{
788+
"Key": "Stage",
789+
"Value": "CODE",
790+
},
791+
],
792+
"Threshold": 1,
793+
"TreatMissingData": "notBreaching",
794+
},
795+
"Type": "AWS::CloudWatch::Alarm",
796+
},
668797
"liveactivitieschannelcleanerlambda569C0617": {
669798
"DependsOn": [
670799
"liveactivitieschannelcleanerlambdaServiceRoleDefaultPolicy2D93C4FB",
@@ -950,6 +1079,69 @@ exports[`The LiveActivities stack matches the snapshot for CODE 1`] = `
9501079
},
9511080
"Type": "AWS::IAM::Policy",
9521081
},
1082+
"liveactivitieschannelmanagerdlqCODEAlarmB1763C34": {
1083+
"Properties": {
1084+
"ActionsEnabled": true,
1085+
"AlarmActions": [
1086+
{
1087+
"Fn::Join": [
1088+
"",
1089+
[
1090+
"arn:aws:sns:",
1091+
{
1092+
"Ref": "AWS::Region",
1093+
},
1094+
":",
1095+
{
1096+
"Ref": "AWS::AccountId",
1097+
},
1098+
":mobile-server-side",
1099+
],
1100+
],
1101+
},
1102+
],
1103+
"AlarmDescription": "More than 1 message in liveactivities-channel-manager-dlq-CODE",
1104+
"AlarmName": "liveactivities-channel-manager-dlq-CODE-alarm",
1105+
"ComparisonOperator": "GreaterThanOrEqualToThreshold",
1106+
"Dimensions": [
1107+
{
1108+
"Name": "QueueName",
1109+
"Value": {
1110+
"Fn::GetAtt": [
1111+
"ChannelManagerDlq58D33B75",
1112+
"QueueName",
1113+
],
1114+
},
1115+
},
1116+
],
1117+
"EvaluationPeriods": 1,
1118+
"MetricName": "ApproximateNumberOfMessagesVisible",
1119+
"Namespace": "AWS/SQS",
1120+
"Period": 60,
1121+
"Statistic": "Sum",
1122+
"Tags": [
1123+
{
1124+
"Key": "gu:cdk:version",
1125+
"Value": "TEST",
1126+
},
1127+
{
1128+
"Key": "gu:repo",
1129+
"Value": "guardian/mobile-n10n",
1130+
},
1131+
{
1132+
"Key": "Stack",
1133+
"Value": "mobile-notifications",
1134+
},
1135+
{
1136+
"Key": "Stage",
1137+
"Value": "CODE",
1138+
},
1139+
],
1140+
"Threshold": 1,
1141+
"TreatMissingData": "notBreaching",
1142+
},
1143+
"Type": "AWS::CloudWatch::Alarm",
1144+
},
9531145
"liveactivitieschannelmanagerlambda398DC149": {
9541146
"DependsOn": [
9551147
"liveactivitieschannelmanagerlambdaServiceRoleDefaultPolicy2C093A08",

‎cdk/lib/liveactivities.ts‎

Lines changed: 42 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,14 @@
1+
import { GuAlarm } from '@guardian/cdk/lib/constructs/cloudwatch';
12
import type { GuStackProps } from '@guardian/cdk/lib/constructs/core';
23
import { GuStack } from '@guardian/cdk/lib/constructs/core';
34
import { GuLambdaFunction } from '@guardian/cdk/lib/constructs/lambda';
45
import type { App } from 'aws-cdk-lib';
56
import { Duration, Tags } from 'aws-cdk-lib';
7+
import {
8+
ComparisonOperator,
9+
Metric,
10+
TreatMissingData,
11+
} from 'aws-cdk-lib/aws-cloudwatch';
612
import { AttributeType, BillingMode, Table } from 'aws-cdk-lib/aws-dynamodb';
713
import { EventBus, Rule, Schedule } from 'aws-cdk-lib/aws-events';
814
import { LambdaFunction } from 'aws-cdk-lib/aws-events-targets';
@@ -11,6 +17,26 @@ import { Runtime } from 'aws-cdk-lib/aws-lambda';
1117
import { SqsDestination } from 'aws-cdk-lib/aws-lambda-destinations';
1218
import { Queue } from 'aws-cdk-lib/aws-sqs';
1319

20+
const addDlqAlarm = (scope: GuStack, app: string, dlq: Queue, name: string) => {
21+
new GuAlarm(scope, `${name}Alarm`, {
22+
app: app,
23+
alarmName: `${name}-alarm`,
24+
alarmDescription: `More than 1 message in ${name}`,
25+
metric: new Metric({
26+
namespace: 'AWS/SQS',
27+
metricName: 'ApproximateNumberOfMessagesVisible',
28+
dimensionsMap: { QueueName: dlq.queueName },
29+
statistic: 'Sum',
30+
period: Duration.minutes(1),
31+
}),
32+
threshold: 1,
33+
evaluationPeriods: 1,
34+
comparisonOperator: ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
35+
treatMissingData: TreatMissingData.NOT_BREACHING,
36+
snsTopicName: 'mobile-server-side',
37+
});
38+
};
39+
1440
export class LiveActivities extends GuStack {
1541
constructor(scope: App, id: string, props: GuStackProps) {
1642
super(scope, id, props);
@@ -32,8 +58,14 @@ export class LiveActivities extends GuStack {
3258
const channelManagerDlq = new Queue(this, 'ChannelManagerDlq', {
3359
queueName: `${app}-channel-manager-dlq-${stage}`,
3460
visibilityTimeout: Duration.minutes(4),
35-
retentionPeriod: Duration.days(7),
61+
retentionPeriod: Duration.days(14),
3662
});
63+
addDlqAlarm(
64+
this,
65+
app,
66+
channelManagerDlq,
67+
`${app}-channel-manager-dlq-${stage}`,
68+
);
3769

3870
const channelLambda = new GuLambdaFunction(
3971
this,
@@ -92,8 +124,9 @@ export class LiveActivities extends GuStack {
92124
const broadcastDlq = new Queue(this, 'BroadcastDlq', {
93125
queueName: `${app}-broadcast-dlq-${stage}`,
94126
visibilityTimeout: Duration.minutes(4),
95-
retentionPeriod: Duration.days(7),
127+
retentionPeriod: Duration.days(14),
96128
});
129+
addDlqAlarm(this, app, broadcastDlq, `${app}-broadcast-dlq-${stage}`);
97130

98131
const broadcastLambda = new GuLambdaFunction(
99132
this,
@@ -145,8 +178,14 @@ export class LiveActivities extends GuStack {
145178
const channelCleanerDlq = new Queue(this, 'ChannelCleanerDlq', {
146179
queueName: `${app}-channel-cleaner-dlq-${stage}`,
147180
visibilityTimeout: Duration.minutes(4),
148-
retentionPeriod: Duration.days(7),
181+
retentionPeriod: Duration.days(14),
149182
});
183+
addDlqAlarm(
184+
this,
185+
app,
186+
channelCleanerDlq,
187+
`${app}-channel-cleaner-dlq-${stage}`,
188+
);
150189

151190
const channelCleanerLambda = new GuLambdaFunction(
152191
this,

0 commit comments

Comments
 (0)