Merge pull request #366 from mdr-engineering/feature/bp_MSOCI-1991_portal-lambda-logs

Adds moose_cloudwatch_logs Module

base/moose_cloudwatch_logs/README.md

@@ -0,0 +1,6 @@
+# Moose Cloudwatch Logs
+
+The purpose of this module is to contain all the bits to pull cloudwatch logs into Splunk via the HEC. The module is designed to pull an already existing cloudwatch log group. Before trying to collect a new cloudwatch log group, be sure to create the HEC token first. This module is dependent on ../../thirdparty/terraform-aws-kinesis-firehose-splunk.
+
+This is the flow of the logs:
+Cloudwatch logs ( gzipped logs ) > Kinesis firehose > Lambda ( to be decompressed ) > Kinesis firehose > Splunk HEC.

base/moose_cloudwatch_logs/main.tf

@@ -0,0 +1,27 @@
+
+# Spit logs to splunk for each of the log groups
+module "kinesis_firehose" {
+  source = "../../thirdparty/terraform-aws-kinesis-firehose-splunk"
+  for_each = var.moose_cloudwatch_log_groups
+  region = var.aws_region
+  arn_cloudwatch_logs_to_ship = "arn:${var.aws_partition}:logs:${var.aws_region}::log-group:/${each.key}/*"
+  name_cloudwatch_logs_to_ship = each.key
+  hec_token = each.value.hec_token
+  hec_url = "https://${var.hec_pub_ack}:8088"
+  firehose_name = each.value.firehose_name
+  tags = merge(var.standard_tags, var.tags)
+  cloudwatch_log_retention = 30 # keep kinesis logs this long
+  lambda_function_name = each.value.lambda_function_name
+  log_stream_name = each.value.log_stream_name
+  kinesis_firehose_lambda_role_name = each.value.kinesis_firehose_lambda_role_name
+  lambda_iam_policy_name = each.value.lambda_iam_policy_name
+  kinesis_firehose_iam_policy_name = each.value.kinesis_firehose_iam_policy_name
+  kinesis_firehose_role_name = each.value.kinesis_firehose_role_name
+  cloudwatch_to_firehose_trust_iam_role_name = each.value.cloudwatch_to_firehose_trust_iam_role_name
+  cloudwatch_to_fh_access_policy_name = each.value.cloudwatch_to_fh_access_policy_name
+  s3_bucket_name = each.value.s3_bucket_name
+  s3_bucket_block_public_access_enabled = 1
+  s3_backup_mode = "FailedEventsOnly"
+  s3_expiration = 30
+}
+

base/moose_cloudwatch_logs/vars.tf

@@ -0,0 +1,55 @@
+variable "tags" {
+  description = "Tags to add to the resource (in addition to global standard tags)"
+  type        = map
+  default     = { }
+}
+
+variable "moose_cloudwatch_log_groups" {
+  type        = map
+  default     = {}
+  description = <<EOF
+Map of Cloudwatch Log groups to loop over and create. Sturucture looks like this:
+```
+moose_cloudwatch_log_groups = {
+    "MyCloudwatchLogGroup" = {
+      hec_token = "myhectoken"
+      firehose_name = "portal_customer_sync_firehose"
+      lambda_function_name = "portal_customer_sync_kinesis_firehose_transform"
+      s3_bucket_name = "kinesis-flowlogs-portal-customer-sync-s3"
+      log_stream_name = "SplunkDelivery_portal_customer_sync"
+      kinesis_firehose_lambda_role_name = "KinesisFirehoseToLambaRole-portal_customer_sync"
+      kinesis_firehose_role_name = "kinesis-firehose-role-name-portal-customer-sync"
+      lambda_iam_policy_name = "Kinesis-Firehose-to-Splunk-Policy-portal_customer_sync"
+      kinesis_firehose_iam_policy_name = "KinesisFirehose-Policy-portal_customer_sync"
+      cloudwatch_to_firehose_trust_iam_role_name = "CloudWatchToSplunkFirehoseTrust-portal_customer_sync"
+      cloudwatch_to_fh_access_policy_name = "KinesisCloudWatchToFirehosePolicy-portal_customer_sync"
+    }
+    "MyCloudwatchLogGroup2" = {
+      hec_token = "myhectoken"
+      firehose_name = "portal_scheduler_firehose"
+      lambda_function_name = "portal_scheduler_kinesis_firehose_transform"
+      s3_bucket_name = "kinesis-flowlogs-portal-scheduler-s3"
+      log_stream_name = "SplunkDelivery_portal_scheduler"
+      kinesis_firehose_lambda_role_name = "KinesisFirehoseToLambaRole-portal_scheduler"
+      kinesis_firehose_role_name = "kinesis-firehose-role-name-portal-scheduler"
+      lambda_iam_policy_name = "Kinesis-Firehose-to-Splunk-Policy-portal_scheduler"
+      kinesis_firehose_iam_policy_name = "KinesisFirehose-Policy-portal_scheduler"
+      cloudwatch_to_firehose_trust_iam_role_name = "CloudWatchToSplunkFirehoseTrust-portal_scheduler"
+      cloudwatch_to_fh_access_policy_name = "KinesisCloudWatchToFirehosePolicy-portal_scheduler"
+    }
+  }
+```
+EOF
+}
+
+variable "splunk_prefix" { type = string }
+variable "hec_pub_ack" { type = string }
+variable "standard_tags" { type = map }
+variable "account_name" { type = string }
+variable "aws_account_id" { type = string }
+variable "aws_partition_alias" { type = string }
+variable "environment" { type = string }
+variable "account_map" { type = map }
+variable "aws_region" { type = string }
+variable "aws_partition" { type = string }
+