CCM-16644: Record Events

.gitleaksignore

@@ -19,3 +19,9 @@ e44436c05bb859820c89b241df2e589ab760595b:infrastructure/terraform/components/rep
 056d713eefe2305444059e99d707bd902ed6f532:infrastructure/terraform/components/reporting/variables.tf:ipv4:109
 d38af4e4f6c36ca9c3d843193b434386a9bad5ee:infrastructure/terraform/etc/env_eu-west-2_int.tfvars:generic-api-key:29
 d38af4e4f6c36ca9c3d843193b434386a9bad5ee:infrastructure/terraform/etc/env_eu-west-2_prod.tfvars:generic-api-key:43
+e4f41f458ca66d94f1fcebdff9579e2ce81d1d5e:infrastructure/terraform/etc/env_eu-west-2_int.tfvars:generic-api-key:29
+e4f41f458ca66d94f1fcebdff9579e2ce81d1d5e:infrastructure/terraform/etc/env_eu-west-2_prod.tfvars:generic-api-key:43
+9469a5a10e20b5c3275ba055e65ba98e7d11e9d2:infrastructure/terraform/components/reporting/README.md:ipv4:16
+9469a5a10e20b5c3275ba055e65ba98e7d11e9d2:infrastructure/terraform/components/reporting/README.md:ipv4:16
+9469a5a10e20b5c3275ba055e65ba98e7d11e9d2:infrastructure/terraform/components/reporting/variables.tf:ipv4:109
+9469a5a10e20b5c3275ba055e65ba98e7d11e9d2:infrastructure/terraform/components/reporting/variables.tf:ipv4:109

infrastructure/terraform/components/reporting/README.md

@@ -31,6 +31,7 @@ No requirements.
 | <a name="input_enable_spot"></a> [enable\_spot](#input\_enable\_spot) | run Power BI On-Premises Gateway as spot instances | `bool` | `false` | no |
 | <a name="input_enable_vault_lock_configuration"></a> [enable\_vault\_lock\_configuration](#input\_enable\_vault\_lock\_configuration) | Enable vault lock, preventing the deletion of a vault that contains 1 or more Recovery Points | `bool` | `false` | no |
 | <a name="input_environment"></a> [environment](#input\_environment) | The name of the environment | `string` | n/a | yes |
+| <a name="input_event_staging_retention_config"></a> [event\_staging\_retention\_config](#input\_event\_staging\_retention\_config) | The number of days for data retention policy for events in the staging table | <pre>object({<br/>    current_days     = number,<br/>    non_current_days = number,<br/>  })</pre> | <pre>{<br/>  "current_days": 534,<br/>  "non_current_days": 14<br/>}</pre> | no |
 | <a name="input_group"></a> [group](#input\_group) | The group variables are being inherited from (often synonmous with account short-name) | `string` | `"n/a"` | no |
 | <a name="input_instance_type"></a> [instance\_type](#input\_instance\_type) | The EC2 instance type. | `string` | `"t3.medium"` | no |
 | <a name="input_log_retention_days"></a> [log\_retention\_days](#input\_log\_retention\_days) | How many days to retain the logs generated by the step function | `number` | `30` | no |

infrastructure/terraform/components/reporting/cloudwatch_log_group_kinesis_firehose_events.tf

@@ -0,0 +1,9 @@
+resource "aws_cloudwatch_log_group" "kinesis_firehose_events" {
+  name              = "/aws/kinesisfirehose/${local.csi}-events"
+  retention_in_days = var.log_retention_days
+}
+
+resource "aws_cloudwatch_log_stream" "kinesis_firehose_events_extended_s3" {
+  name           = "${local.csi}-events-extended-s3"
+  log_group_name = aws_cloudwatch_log_group.kinesis_firehose_events.name
+}

infrastructure/terraform/components/reporting/glue_catalog_table_event_staging.tf

@@ -0,0 +1,140 @@
+resource "aws_glue_catalog_table" "event_staging" {
+  name          = "event_staging"
+  description   = "Staging table for all event records."
+  database_name = aws_glue_catalog_database.reporting.name
+
+  table_type = "EXTERNAL_TABLE"
+
+  storage_descriptor {
+    location = "s3://${aws_s3_bucket.events.bucket}/${local.firehose_output_path_events}"
+
+    input_format  = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat"
+    output_format = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat"
+
+    # additional columns must be added at the end of the list
+    columns {
+      name = "specversion"
+      type = "string"
+    }
+    columns {
+      name = "id"
+      type = "string"
+    }
+    columns {
+      name = "source"
+      type = "string"
+    }
+    columns {
+      name = "subject"
+      type = "string"
+    }
+    columns {
+      name = "type"
+      type = "string"
+    }
+    columns {
+      name = "time"
+      type = "string"
+    }
+    columns {
+      name = "datacontenttype"
+      type = "string"
+    }
+    columns {
+      name = "dataschema"
+      type = "string"
+    }
+    columns {
+      name = "data"
+      type = "string"
+    }
+    columns {
+      name = "traceparent"
+      type = "string"
+    }
+    columns {
+      name = "tracestate"
+      type = "string"
+    }
+    columns {
+      name = "partitionkey"
+      type = "string"
+    }
+    columns {
+      name = "recordedtime"
+      type = "string"
+    }
+    columns {
+      name = "sampledrate"
+      type = "string"
+    }
+    columns {
+      name = "sampledrate"
+      type = "int"
+    }
+    columns {
+      name = "sequence"
+      type = "string"
+    }
+    columns {
+      name = "severitytext"
+      type = "string"
+    }
+    columns {
+      name = "severitynumber"
+      type = "int"
+    }
+    columns {
+      name = "dataclassification"
+      type = "string"
+    }
+    columns {
+      name = "dataregulation"
+      type = "string"
+    }
+    columns {
+      name = "datacategory"
+      type = "string"
+    }
+  }
+
+  partition_keys {
+    name = "type"
+    type = "string"
+  }
+
+  partition_keys {
+    name = "year"
+    type = "int"
+  }
+  partition_keys {
+    name = "month"
+    type = "int"
+  }
+  partition_keys {
+    name = "day"
+    type = "int"
+  }
+
+  parameters = {
+    EXTERNAL              = "TRUE"
+    "parquet.compression" = "SNAPPY"
+    compressionType       = "none"
+    classification        = "parquet"
+  }
+}
+
+resource "aws_glue_partition_index" "event_record" {
+  database_name = aws_glue_catalog_database.reporting.name
+  table_name    = aws_glue_catalog_table.event_staging.name
+
+  partition_index {
+    index_name = "data"
+    keys       = ["type", "year", "month", "day"]
+  }
+
+  timeouts {
+    create = "60m"
+    delete = "60m"
+  }
+}

infrastructure/terraform/components/reporting/iam_role_firehose_events.tf

@@ -0,0 +1,107 @@
+resource "aws_iam_role" "firehose_events" {
+  name               = "${local.csi}-firehose-events-role"
+  description        = "Role used by Firehose to deliver events into the event staging S3 bucket"
+  assume_role_policy = data.aws_iam_policy_document.firehose_events_assume_role.json
+}
+
+data "aws_iam_policy_document" "firehose_events_assume_role" {
+  statement {
+    effect = "Allow"
+
+    principals {
+      type        = "Service"
+      identifiers = ["firehose.amazonaws.com"]
+    }
+
+    actions = ["sts:AssumeRole"]
+  }
+}
+
+resource "aws_iam_role_policy_attachment" "firehose_events" {
+  role       = aws_iam_role.firehose_events.name
+  policy_arn = aws_iam_policy.firehose_events.arn
+}
+
+resource "aws_iam_policy" "firehose_events" {
+  name        = "${local.csi}-firehose-events-policy"
+  description = "Permissions for Firehose delivery to the event staging S3 bucket"
+  policy      = data.aws_iam_policy_document.firehose_events.json
+}
+
+data "aws_iam_policy_document" "firehose_events" {
+  statement {
+    sid    = "AllowS3BucketMeta"
+    effect = "Allow"
+
+    actions = [
+      "s3:GetBucketLocation",
+      "s3:ListBucket",
+      "s3:ListBucketMultipartUploads"
+    ]
+
+    resources = [
+      aws_s3_bucket.events.arn,
+    ]
+  }
+
+  statement {
+    sid    = "AllowS3ObjectWrite"
+    effect = "Allow"
+
+    actions = [
+      "s3:AbortMultipartUpload",
+      "s3:GetObject",
+      "s3:PutObject"
+    ]
+
+    resources = [
+      "${aws_s3_bucket.events.arn}/${local.firehose_output_path_prefix}/*",
+    ]
+  }
+
+  statement {
+    sid    = "AllowCloudWatchLogging"
+    effect = "Allow"
+
+    actions = [
+      "logs:CreateLogStream",
+      "logs:PutLogEvents"
+    ]
+
+    resources = [
+      aws_cloudwatch_log_group.kinesis_firehose_events.arn,
+      aws_cloudwatch_log_stream.kinesis_firehose_events_extended_s3.arn,
+    ]
+  }
+
+  statement {
+    sid    = "AllowGlueSchemaAccess"
+    effect = "Allow"
+
+    actions = [
+      "glue:GetTable",
+      "glue:GetTableVersion",
+      "glue:GetTableVersions"
+    ]
+
+    resources = [
+      "arn:aws:glue:${var.region}:${var.aws_account_id}:catalog"
+    ]
+  }
+
+  statement {
+    sid    = "AllowKMSEncryption"
+    effect = "Allow"
+
+    actions = [
+      "kms:Decrypt",
+      "kms:DescribeKey",
+      "kms:Encrypt",
+      "kms:GenerateDataKey"
+    ]
+
+    resources = [
+      aws_kms_key.s3.arn,
+    ]
+  }
+}

infrastructure/terraform/components/reporting/iam_role_sns_events_firehose.tf

@@ -0,0 +1,44 @@
+resource "aws_iam_role" "sns_events_firehose" {
+  name               = "${local.csi}-sns-events-firehose-role"
+  assume_role_policy = data.aws_iam_policy_document.sns_events_firehose_assume_role.json
+}
+
+data "aws_iam_policy_document" "sns_events_firehose_assume_role" {
+  statement {
+    effect = "Allow"
+
+    principals {
+      type        = "Service"
+      identifiers = ["sns.amazonaws.com"]
+    }
+
+    actions = ["sts:AssumeRole"]
+  }
+}
+
+resource "aws_iam_policy" "sns_events_firehose_delivery" {
+  name        = "${local.csi}-sns-events-firehose-delivery"
+  description = "Allows SNS to publish events to the events Firehose stream"
+  policy      = data.aws_iam_policy_document.sns_events_firehose_delivery.json
+}
+
+data "aws_iam_policy_document" "sns_events_firehose_delivery" {
+  statement {
+    sid    = "AllowFirehoseDelivery"
+    effect = "Allow"
+
+    actions = [
+      "firehose:PutRecord",
+      "firehose:PutRecordBatch",
+    ]
+
+    resources = [
+      aws_kinesis_firehose_delivery_stream.events.arn,
+    ]
+  }
+}
+
+resource "aws_iam_role_policy_attachment" "sns_events_firehose_delivery" {
+  role       = aws_iam_role.sns_events_firehose.name
+  policy_arn = aws_iam_policy.sns_events_firehose_delivery.arn
+}

infrastructure/terraform/components/reporting/kinesis_firehose_delivery_stream_eventsub_to_s3_reporting.tf

@@ -0,0 +1,50 @@
+resource "aws_kinesis_firehose_delivery_stream" "events" {
+  name        = "${local.csi}-events"
+  destination = "extended_s3"
+
+  server_side_encryption {
+    enabled  = true
+    key_type = "CUSTOMER_MANAGED_CMK"
+    key_arn  = aws_kms_key.s3.arn
+  }
+
+  extended_s3_configuration {
+    role_arn    = aws_iam_role.firehose_events.arn
+    bucket_arn  = aws_s3_bucket.events.arn
+    kms_key_arn = aws_kms_key.s3.arn
+
+    buffering_interval = 300
+    buffering_size     = 128
+
+    prefix              = "${local.firehose_output_path_events}/!{partitionKeyFromQuery:type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/"
+    error_output_prefix = "${local.firehose_output_path_prefix}/firehose-errors/!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/"
+
+    dynamic_partitioning_configuration {
+      enabled = true
+    }
+
+    processing_configuration {
+      enabled = true
+
+      processors {
+        type = "MetadataExtraction"
+
+        parameters {
+          parameter_name  = "JsonParsingEngine"
+          parameter_value = "JQ-1.6"
+        }
+
+        parameters {
+          parameter_name  = "MetadataExtractionQuery"
+          parameter_value = "{type:.type}"
+        }
+      }
+    }
+
+    cloudwatch_logging_options {
+      enabled         = true
+      log_group_name  = aws_cloudwatch_log_group.kinesis_firehose_events.name
+      log_stream_name = aws_cloudwatch_log_stream.kinesis_firehose_events_extended_s3.name
+    }
+  }
+}

infrastructure/terraform/components/reporting/kms_key_s3.tf

@@ -71,4 +71,24 @@ data "aws_iam_policy_document" "s3" {
       ]
     }
   }
+
+  statement {
+    sid    = "AllowUsageFromSharedInfraAccount"
+    effect = "Allow"
+
+    principals {
+      type        = "AWS"
+      identifiers = ["arn:aws:iam::${var.shared_infra_account_id}:root"]
+    }
+
+    actions = [
+      "kms:Encrypt",
+      "kms:Decrypt",
+      "kms:GenerateDataKey"
+    ]
+
+    resources = [
+      "*",
+    ]
+  }
 }