Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,11 @@ build-and-push: # Build lambda docker images and pushes them to ECR
deploy: # Deploys whole project - mandatory: PROFILE
eval "$$(make -s populate-tagging-variables)"
make terraform-apply-auto-approve STACKS=api-key,shared-resources,application,blue-green-link
make terraform-apply-auto-approve STACKS=cloudwatch-queries TF_CLI_ARGS="-parallelism=3"

undeploy: # Undeploys whole project - mandatory: PROFILE
eval "$$(make -s populate-tagging-variables)"
make terraform-destroy-auto-approve STACKS=cloudwatch-queries
make terraform-destroy-auto-approve STACKS=blue-green-link,application,shared-resources VERSION=any
if [ "$(PROFILE)" != "live" ]; then
make terraform-destroy-auto-approve STACKS=api-key
Expand Down Expand Up @@ -464,6 +466,10 @@ link-blue-green-environment: # Links blue green environment - mandatory: PROFILE
eval "$$(make -s populate-tagging-variables)"
make terraform-apply-auto-approve STACKS=blue-green-link

deploy-cloudwatch-queries: # Deploys CloudWatch query definitions with reduced parallelism - mandatory: PROFILE, ENVIRONMENT, SHARED_ENVIRONMENT, BLUE_GREEN_ENVIRONMENT
eval "$$(make -s populate-tagging-variables)"
make terraform-apply-auto-approve STACKS=cloudwatch-queries TF_CLI_ARGS="-parallelism=3"

undeploy-shared-resources: # Undeploys shared resources (Only intended to run in pipeline) - mandatory: PROFILE, ENVIRONMENT, SHARED_ENVIRONMENT, BLUE_GREEN_ENVIRONMENT
eval "$$(make -s populate-tagging-variables)"
make terraform-destroy-auto-approve STACKS=shared-resources
Expand All @@ -475,6 +481,10 @@ undeploy-blue-green-environment: # Undeploys blue/green resources (Only intended
eval "$$(make -s populate-tagging-variables)"
make terraform-destroy-auto-approve STACKS=application

undeploy-cloudwatch-queries: # Undeploys CloudWatch query definitions - mandatory: PROFILE, ENVIRONMENT, SHARED_ENVIRONMENT, BLUE_GREEN_ENVIRONMENT
eval "$$(make -s populate-tagging-variables)"
make terraform-destroy-auto-approve STACKS=cloudwatch-queries

unlink-blue-green-environment: # Un-Links blue green environment - mandatory: PROFILE, ENVIRONMENT, SHARED_ENVIRONMENT, BLUE_GREEN_ENVIRONMENT
eval "$$(make -s populate-tagging-variables)"
make terraform-destroy-auto-approve STACKS=blue-green-link
Expand Down
25 changes: 25 additions & 0 deletions infrastructure/stacks/cloudwatch-queries/.terraform.lock.hcl

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,6 @@ fields @timestamp, correlation_id, ods_code, function_name, message
| filter level == 'ERROR'
| sort @timestamp
EOF

provisioner "local-exec" {
command = "sleep 30"
}
}

resource "aws_cloudwatch_query_definition" "search_by_correlation_id" {
Expand All @@ -40,10 +36,6 @@ fields @timestamp, message
| filter correlation_id == 'REPLACE'
| sort @timestamp
EOF

provisioner "local-exec" {
command = "sleep 30"
}
}

resource "aws_cloudwatch_query_definition" "search_by_correlation_id_expanded" {
Expand All @@ -64,10 +56,6 @@ fields @timestamp,correlation_id,ods_code,level,message_received,function_name,
| filter correlation_id == 'REPLACE'
| sort @timestamp
EOF

provisioner "local-exec" {
command = "sleep 30"
}
}

resource "aws_cloudwatch_query_definition" "search_by_odscode" {
Expand All @@ -88,10 +76,6 @@ fields @timestamp, message
| filter ods_code == 'REPLACE'
| sort @timestamp
EOF

provisioner "local-exec" {
command = "sleep 30"
}
}

resource "aws_cloudwatch_query_definition" "search_by_odscode_expanded" {
Expand All @@ -112,10 +96,6 @@ fields @timestamp,correlation_id,ods_code,level,message_received,function_name,
| filter ods_code == 'REPLACE'
| sort @timestamp
EOF

provisioner "local-exec" {
command = "sleep 30"
}
}

resource "aws_cloudwatch_query_definition" "search_for_invalid_postcode" {
Expand All @@ -130,10 +110,6 @@ fields @timestamp,correlation_id,ods_code,level,message_received,function_name,
| filter report_key == 'INVALID_POSTCODE'
| sort @timestamp
EOF

provisioner "local-exec" {
command = "sleep 30"
}
}

resource "aws_cloudwatch_query_definition" "search_for_invalid_opening_times" {
Expand All @@ -148,10 +124,6 @@ fields @timestamp,correlation_id,ods_code,level,message_received,function_name,
| filter report_key == 'INVALID_OPEN_TIMES'
| sort @timestamp
EOF

provisioner "local-exec" {
command = "sleep 30"
}
}

resource "aws_cloudwatch_query_definition" "search_by_email_correlation_id" {
Expand All @@ -167,10 +139,6 @@ fields correlation_id
| filter message =="Email Correlation Id"
| filter email_correlation_id == "ADD_EMAIL_CORRELATION_ID"
EOF

provisioner "local-exec" {
command = "sleep 30"
}
}

resource "aws_cloudwatch_query_definition" "search_by_update_request_success" {
Expand All @@ -185,10 +153,6 @@ fields @timestamp, correlation_id
| filter ServiceUpdateSuccess == 1
| sort @timestamp desc
EOF

provisioner "local-exec" {
command = "sleep 30"
}
}

resource "aws_cloudwatch_query_definition" "search_by_update_request_failed" {
Expand All @@ -203,10 +167,6 @@ fields @timestamp, correlation_id, report_key
| filter report_key == DOS_DB_UPDATE_DLQ_HANDLER_RECEIVED_EVENT
| sort @timestamp desc
EOF

provisioner "local-exec" {
command = "sleep 30"
}
}

resource "aws_cloudwatch_query_definition" "search_by_dos_data_item_updates" {
Expand All @@ -223,10 +183,6 @@ fields @timestamp, correlation_id
| filter field == 'REPLACE'
| sort @timestamp desc
EOF

provisioner "local-exec" {
command = "sleep 30"
}
}

resource "aws_cloudwatch_query_definition" "search_for_report_warnings" {
Expand All @@ -248,13 +204,8 @@ fields @timestamp, correlation_id, message
| filter level == 'WARNING'
| sort @timestamp desc
EOF

provisioner "local-exec" {
command = "sleep 30"
}
}


resource "aws_cloudwatch_query_definition" "search_for_quality_checker_logs_with_odscode" {
name = "${var.project_id}/${var.blue_green_environment}/search-for-quality-checker-logs-with-odscode"

Expand All @@ -267,8 +218,4 @@ fields @timestamp, level, message
| filter odscode = 'TO_ADD'
| sort @timestamp asc
EOF

provisioner "local-exec" {
command = "sleep 30"
}
}
11 changes: 11 additions & 0 deletions infrastructure/stacks/cloudwatch-queries/terraform.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
terraform {
backend "s3" {
encrypt = true
}
required_providers {
aws = {
source = "hashicorp/aws"
version = "~> 6.14.1"
}
}
}
43 changes: 43 additions & 0 deletions infrastructure/stacks/cloudwatch-queries/variables.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
# ##############
# # LAMBDAS
# ##############

variable "change_event_dlq_handler_lambda" {
type = string
description = "Name of fifo dlq handler lambda"
}

variable "dos_db_update_dlq_handler_lambda" {
type = string
description = "Name of cr_fifo dlq handler lambda"
}

variable "event_replay_lambda" {
type = string
description = "Name of event replay lambda"
}

variable "ingest_change_event_lambda" {
type = string
description = "Name of ingest change event lambda"
}

variable "send_email_lambda" {
type = string
description = "Name of send email lambda"
}

variable "service_matcher_lambda" {
type = string
description = "Name of event processor lambda"
}

variable "service_sync_lambda" {
type = string
description = "Name of event sender lambda"
}

variable "quality_checker_lambda" {
type = string
description = "Name of quality checker lambda"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
# For documentation see here - https://docs.aws.amazon.com/codebuild/latest/userguide/build-spec-ref.html
version: 0.2

phases:
pre_build:
commands:
- chown -R 999:999 $CODEBUILD_SRC_DIR
- temp_role=$(aws sts assume-role --role-arn "arn:aws:iam::$AWS_ACCOUNT_ID_NONPROD:role/$PIPELINE_BUILD_ROLE" --role-session-name "CodeBuildSessionNonProd")
- export AWS_ACCESS_KEY_ID=$(echo $temp_role | jq -r .Credentials.AccessKeyId)
- export AWS_SECRET_ACCESS_KEY=$(echo $temp_role | jq -r .Credentials.SecretAccessKey)
- export AWS_SESSION_TOKEN=$(echo $temp_role | jq -r .Credentials.SessionToken)
- make docker-hub-signin
- unset AWS_ACCESS_KEY_ID
- unset AWS_SECRET_ACCESS_KEY
- unset AWS_SESSION_TOKEN
- |
if [[ "$AWS_ACCOUNT" == "PROD" ]]; then
temp_role=$(aws sts assume-role --role-arn "arn:aws:iam::$AWS_ACCOUNT_ID_PROD:role/$PIPELINE_BUILD_ROLE" --role-session-name "CodeBuildSession")
else
temp_role=$(aws sts assume-role --role-arn "arn:aws:iam::$AWS_ACCOUNT_ID_NONPROD:role/$PIPELINE_BUILD_ROLE" --role-session-name "CodeBuildSession")
fi
- export AWS_ACCESS_KEY_ID=$(echo $temp_role | jq -r .Credentials.AccessKeyId)
- export AWS_SECRET_ACCESS_KEY=$(echo $temp_role | jq -r .Credentials.SecretAccessKey)
- export AWS_SESSION_TOKEN=$(echo $temp_role | jq -r .Credentials.SessionToken)
- aws sts get-caller-identity
- make -s clean
build:
on-failure: CONTINUE
commands:
- echo "[Deploying CloudWatch Queries]"
- echo AWS_ACCOUNT=$AWS_ACCOUNT
- echo ENVIRONMENT=$ENVIRONMENT
- echo PROFILE=$PROFILE
- eval "$$(make -s populate-tagging-variables)"
- make deploy-cloudwatch-queries VERSION=$CODEBUILD_RESOLVED_SOURCE_VERSION PROFILE=$PROFILE ENVIRONMENT=$ENVIRONMENT
Original file line number Diff line number Diff line change
Expand Up @@ -30,4 +30,4 @@ phases:
- echo AWS_ACCOUNT=$AWS_ACCOUNT
- echo ENVIRONMENT=$ENVIRONMENT
- echo PROFILE=$PROFILE
- make deploy VERSION=$CODEBUILD_RESOLVED_SOURCE_VERSION PROFILE=$PROFILE ENVIRONMENT=$ENVIRONMENT TF_CLI_ARGS="-parallelism=30"
- make terraform-apply-auto-approve STACKS=api-key,shared-resources,application,blue-green-link VERSION=$CODEBUILD_RESOLVED_SOURCE_VERSION PROFILE=$PROFILE ENVIRONMENT=$ENVIRONMENT TF_CLI_ARGS="-parallelism=30"
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,19 @@ batch:
- build_slack_messenger
- build_quality_checker

- identifier: deploy_cloudwatch_queries
env:
compute-type: BUILD_GENERAL1_SMALL
image: aws/codebuild/amazonlinux2-x86_64-standard:5.0
type: LINUX_CONTAINER
privileged-mode: true
variables:
PROFILE: dev
ENVIRONMENT: release
buildspec: infrastructure/stacks/development-and-deployment-tools/batch-buildspecs/build-deploy-test-release-batch-jobs/deploy-cloudwatch-queries.yml
depend-on:
- deploy

- identifier: integration_test
env:
compute-type: BUILD_GENERAL1_LARGE
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
# For documentation see here - https://docs.aws.amazon.com/codebuild/latest/userguide/build-spec-ref.html
version: 0.2

phases:
pre_build:
commands:
- temp_role=$(aws sts assume-role --role-arn "arn:aws:iam::$AWS_ACCOUNT_ID_NONPROD:role/$PIPELINE_BUILD_ROLE" --role-session-name "CodeBuildSession")
- export AWS_ACCESS_KEY_ID=$(echo $temp_role | jq -r .Credentials.AccessKeyId)
- export AWS_SECRET_ACCESS_KEY=$(echo $temp_role | jq -r .Credentials.SecretAccessKey)
- export AWS_SESSION_TOKEN=$(echo $temp_role | jq -r .Credentials.SessionToken)
- make docker-hub-signin
- unset AWS_ACCESS_KEY_ID
- unset AWS_SECRET_ACCESS_KEY
- unset AWS_SESSION_TOKEN

- temp_role=$(aws sts assume-role --role-arn "arn:aws:iam::$AWS_ACCOUNT_ID_NONPROD:role/$PIPELINE_BUILD_ROLE" --role-session-name "CodeBuildSession")
- export AWS_ACCESS_KEY_ID=$(echo $temp_role | jq -r .Credentials.AccessKeyId)
- export AWS_SECRET_ACCESS_KEY=$(echo $temp_role | jq -r .Credentials.SecretAccessKey)
- export AWS_SESSION_TOKEN=$(echo $temp_role | jq -r .Credentials.SessionToken)
- echo PROFILE = $PROFILE
build:
on-failure: CONTINUE
commands:
- export BUILD_BRANCH=$(echo $CODEBUILD_WEBHOOK_TRIGGER | sed -e "s|^refs/heads/||")
- echo Branch = $BUILD_BRANCH
- |
if [ -z "$BUILD_BRANCH" ]; then
echo "Not triggered by branch will stop deploying CloudWatch queries"
exit
fi
- echo "[Deploying CloudWatch Queries]"
- make deploy-cloudwatch-queries VERSION=$(make commit-date-hash-tag) BUILD_BRANCH=$BUILD_BRANCH PROFILE=$PROFILE
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,18 @@ batch:
- build_quality_checker
- deploy_prerequisites

- identifier: deploy_cloudwatch_queries
env:
compute-type: BUILD_GENERAL1_SMALL
image: aws/codebuild/amazonlinux2-x86_64-standard:5.0
type: LINUX_CONTAINER
privileged-mode: true
variables:
PROFILE: dev
buildspec: infrastructure/stacks/development-and-deployment-tools/batch-buildspecs/task-env-deploy-and-test-batch-jobs/deploy-cloudwatch-queries.yml
depend-on:
- deploy

- identifier: integration_test_reporting
env:
compute-type: BUILD_GENERAL1_LARGE
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,9 @@ phases:
# Only destroy the environment if it exists
- |
if [ -n "$ENVIRONMENT_DEPLOYED" ]; then
echo "[Destroying CloudWatch Queries]";
make undeploy-cloudwatch-queries PROFILE=$PROFILE ENVIRONMENT=$ENVIRONMENT SHARED_ENVIRONMENT=$SHARED_ENVIRONMENT BLUE_GREEN_ENVIRONMENT=$BLUE_GREEN_ENVIRONMENT TF_CLI_ARGS="-parallelism=30" || echo "CloudWatch queries already destroyed or don't exist";
echo "[Destroying Application Stack]";
make undeploy-blue-green-environment PROFILE=$PROFILE ENVIRONMENT=$ENVIRONMENT SHARED_ENVIRONMENT=$SHARED_ENVIRONMENT BLUE_GREEN_ENVIRONMENT=$BLUE_GREEN_ENVIRONMENT TF_CLI_ARGS="-parallelism=30";
fi
-
Loading
Loading