Merge pull request #43 from mdr-engineering/feature/ftd_MSOCI-1364_DataIntoSplunk

Adds Functionality for Moose Data Ingestion

base/account_standards/iam.tf

@@ -143,3 +143,124 @@ resource "aws_iam_role_policy" "dlm_lifecycle" {
 }
 EOF
 }
+
+##########################
+# moose
+# 
+# See https://docs.splunk.com/Documentation/AddOns/released/AWS/ConfigureAWSpermissions
+locals {
+  trusted_principals_govcloud   = [
+    "arn:${var.aws_partition}:iam::${local.c2_account}:role/instance/moose-hf",
+    "arn:${var.aws_partition}:iam::${local.c2_account}:user/instance/moose-hf"
+  ]
+  trusted_principals_commercial = [ 
+    "arn:${var.aws_partition}:iam::${var.legacy_account}:role/splunk-aws-instance-role",
+    "arn:${var.aws_partition}:iam::${local.c2_account}:user/instance/moose-hf",
+  ]
+  trusted_principals = var.aws_partition == "aws" ? local.trusted_principals_commercial : local.trusted_principals_govcloud
+}
+
+
+resource "aws_iam_role" "splunk_addon_for_aws" {
+  name = "splunk-addon-for-aws"
+  path = "/instance/"
+
+  assume_role_policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Sid": "",
+      "Effect": "Allow",
+      "Principal": {
+        "AWS": ${jsonencode(local.trusted_principals)}
+      },
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}
+EOF
+}
+
+resource "aws_iam_role_policy" "splunk_addon_for_aws" {
+  name = "splunk-addon-for-aws"
+  role = aws_iam_role.splunk_addon_for_aws.id
+  policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "sqs:GetQueueAttributes",
+        "sqs:ListQueues",
+        "sqs:ReceiveMessage",
+        "sqs:GetQueueUrl",
+        "sqs:SendMessage",
+        "sqs:DeleteMessage",
+        "s3:ListBucket",
+        "s3:GetObject",
+        "s3:GetBucketLocation",
+        "s3:ListAllMyBuckets",
+        "s3:GetBucketTagging", 
+        "s3:GetAccelerateConfiguration", 
+        "s3:GetBucketLogging", 
+        "s3:GetLifecycleConfiguration", 
+        "s3:GetBucketCORS",
+        "config:DeliverConfigSnapshot",
+        "config:DescribeConfigRules",
+        "config:DescribeConfigRuleEvaluationStatus",
+        "config:GetComplianceDetailsByConfigRule",
+        "config:GetComplianceSummaryByConfigRule",
+        "iam:GetUser",
+        "iam:ListUsers",
+        "iam:GetAccountPasswordPolicy",
+        "iam:ListAccessKeys",
+        "iam:GetAccessKeyLastUsed", 
+        "autoscaling:Describe*",
+        "cloudwatch:Describe*",
+        "cloudwatch:Get*",
+        "cloudwatch:List*",
+        "sns:Get*",
+        "sns:List*",
+        "sns:Publish",
+        "logs:DescribeLogGroups",
+        "logs:DescribeLogStreams",
+        "logs:GetLogEvents",
+        "ec2:DescribeInstances",
+        "ec2:DescribeReservedInstances",
+        "ec2:DescribeSnapshots",
+        "ec2:DescribeRegions",
+        "ec2:DescribeKeyPairs",
+        "ec2:DescribeNetworkAcls",
+        "ec2:DescribeSecurityGroups",
+        "ec2:DescribeSubnets",
+        "ec2:DescribeVolumes",
+        "ec2:DescribeVpcs",
+        "ec2:DescribeImages",
+        "ec2:DescribeAddresses",
+        "lambda:ListFunctions",
+        "rds:DescribeDBInstances",
+        "cloudfront:ListDistributions",
+        "elasticloadbalancing:DescribeLoadBalancers",
+        "elasticloadbalancing:DescribeInstanceHealth",
+        "elasticloadbalancing:DescribeTags",
+        "elasticloadbalancing:DescribeTargetGroups",
+        "elasticloadbalancing:DescribeTargetHealth",
+        "elasticloadbalancing:DescribeListeners",
+        "inspector:Describe*",
+        "inspector:List*",
+        "kinesis:Get*",
+        "kinesis:DescribeStream",
+        "kinesis:ListStreams",
+        "kms:Decrypt",
+        "sts:AssumeRole"
+      ],
+      "Resource": [
+        "*"
+      ]
+    }
+  ]
+}
+EOF
+}

base/account_standards/vars.tf

@@ -31,6 +31,7 @@ variable extra_ebs_key_attachers {
 # Below this line are variables inherited from higher levels, so they
 # do not need to be explicitly passed to this module.
 variable "is_legacy" { type = bool }
+variable "legacy_account" { type = string }
 variable "standard_tags" { type = map }
 variable "account_list" { type = list }
 variable "aws_account_id" { type = string }

base/account_standards_c2/config_bucket.tf

@@ -158,3 +158,149 @@ data "aws_iam_policy_document" "config_encryption_key_policy" {
     }
   }
 }
+
+#### SQS Queue for Splunk
+resource "aws_s3_bucket_notification" "on_new_config_object" {
+  bucket = aws_s3_bucket.xdr_config_bucket.bucket
+
+  topic {
+    topic_arn = aws_sns_topic.new_config_object_event.arn
+
+    events = [
+      "s3:ObjectCreated:*",
+    ]
+
+    filter_suffix = ""
+  }
+}
+
+resource "aws_sns_topic" "new_config_object_event" {
+  name = "s3-notification-topic-${aws_s3_bucket.xdr_config_bucket.bucket}"
+  kms_master_key_id = aws_kms_key.new_object_key.id
+}
+
+resource "aws_sns_topic_policy" "this_config" {
+  arn    = aws_sns_topic.new_config_object_event.arn
+  policy = data.aws_iam_policy_document.config_bucket_can_publish.json
+}
+
+data "aws_iam_policy_document" "config_bucket_can_publish" {
+  statement {
+    actions = [
+      "SNS:Publish",
+    ]
+
+    effect = "Allow"
+
+    condition {
+      test     = "ArnLike"
+      variable = "aws:SourceArn"
+
+      values = [
+        aws_s3_bucket.xdr_config_bucket.arn
+      ]
+    }
+
+    principals {
+      type        = "AWS"
+      identifiers = ["*"]
+    }
+
+    resources = [
+      aws_sns_topic.new_config_object_event.arn
+    ]
+
+    sid = "allowpublish"
+  }
+
+  statement {
+    actions = [
+      "SNS:Subscribe",
+      "SNS:Receive",
+    ]
+
+    effect = "Allow"
+
+    principals {
+      type        = "AWS"
+      identifiers = ["*"]
+    }
+
+    condition {
+      test     = "ArnEquals"
+      values   = [ aws_sqs_queue.new_s3_config_object.arn ]
+      variable = "aws:SourceArn"
+    }
+
+    resources = [
+      aws_sns_topic.new_config_object_event.arn
+    ]
+
+    sid = "sid_allow_subscribe"
+  }
+}
+
+# This is the config queue for splunk to subscribe to
+resource "aws_sqs_queue" "new_s3_config_object" {
+  name                       = "new-objects-for-${aws_s3_bucket.xdr_config_bucket.bucket}"
+  visibility_timeout_seconds = 300 # wait 5 minutes before allowing a different splunk instance to process the same message
+  message_retention_seconds  = 604800 # Keep a message in the queue for 7 days
+  receive_wait_time_seconds  = 0 # how long to wait for a message before returning
+  redrive_policy             = "{\"deadLetterTargetArn\":\"${aws_sqs_queue.config-dlq.arn}\",\"maxReceiveCount\":4}"
+  tags                       = merge(var.standard_tags, var.tags)
+  kms_master_key_id = aws_kms_key.new_object_key.id
+  kms_data_key_reuse_period_seconds = 3600
+}
+
+data "aws_iam_policy_document" "sns_topic_config_can_publish" {
+  statement {
+    effect = "Allow"
+
+    principals {
+      identifiers = [
+        "*",
+      ]
+
+      type = "AWS"
+    }
+
+    actions = [
+      "SQS:SendMessage",
+    ]
+
+    resources = [
+      aws_sqs_queue.new_s3_config_object.arn
+    ]
+
+    condition {
+      test = "ArnEquals"
+
+      values = [
+        aws_sns_topic.new_config_object_event.arn
+      ]
+
+      variable = "aws:SourceArn"
+    }
+  }
+}
+
+// Dead Letter queue, use same parameters as main queue
+resource "aws_sqs_queue" "config-dlq" {
+  name                      = "new-objects-for-${aws_s3_bucket.xdr_config_bucket.bucket}-dlq"
+  message_retention_seconds = 300
+  receive_wait_time_seconds = 0
+  tags                      = merge(var.standard_tags, var.tags)
+  kms_master_key_id = aws_kms_key.new_object_key.id
+  kms_data_key_reuse_period_seconds = 3600
+}
+
+resource "aws_sqs_queue_policy" "config_bucket_can_publish" {
+  policy    = data.aws_iam_policy_document.sns_topic_config_can_publish.json
+  queue_url = aws_sqs_queue.new_s3_object.id
+}
+
+resource "aws_sns_topic_subscription" "config_bucket_change_notification_to_queue" {
+  topic_arn = aws_sns_topic.new_config_object_event.arn
+  protocol  = "sqs"
+  endpoint  = aws_sqs_queue.new_s3_config_object.arn
+}

base/account_standards_c2/main.tf

@@ -45,7 +45,6 @@ resource "aws_s3_bucket_notification" "on_new_object" {
       "s3:ObjectCreated:*",
     ]
 
-    # TODO: Can we filter out the digests?
     filter_suffix = ""
   }
 }