Merge pull request #349 from mdr-engineering/feature/ftd_MSOCI-2002_ClientCodeBuildDeployments

Adds Codebuild Module for Splunk Apps

base/codebuild_splunk_apps/README.md

@@ -0,0 +1,34 @@
+# Codebuild Splunk Apps
+
+Based heavily off codebuild_artifact, this module creates multiple CodeBuild jobs to deposit apps into the `xdr-[prefix]-[environment]-splunk-apps` bucket.
+
+It uses an ECR container from common services and creates multiple build jobs to deposit into the S3 bucket.
+
+# How to Use this Module
+
+This module is where the CodeBuild artifacts are created. It uses the Terraform GitHub Provider and requires a Personal Access Token. This should be your Personal Access Token not mdr-aws-codebuild's token (see how-to below). The provider will look in the environmental variables for the token. 
+
+```
+export GITHUB_TOKEN=<gihub_token>
+```
+
+This module should NOT create the github repo. That is a manual process. I am not comfortable with terraform adding/removing github repos. The github repo should have the same name as the name variable in the terrafgrunt.hcl file. The user associated with the Personal Access Token needs to have admin permissions on the github repo. 
+
+## Github Service Account ( mdr-aws-codebuild )
+
+AWS CodeBuild needs a Github Personal Access Token to pull code after the code in a repository has been updated. Terraform is capable of storing the Github Personal Access Token, but that is a bad idea. A better idea is a service account in Github that gives CodeBuild access to specific repositories. This user will need access to repositories in different organizations. The login credentials as well as the Personal Access Token for mdr-aws-codebuild are stored in Vault. 
+
+The service account (mdr-aws-codebuild) needs to have a personal access token manually placed into the aws console. 
+
+## Creating a Personal Access Token
+
+1) In github, go to `settings->Developer Settings->Personal Access Tokens`
+2) Click 'Generate Token'
+3) Give it a name (e.g. `terragrunt`)
+4) Give it the following permission groups:
+  * repo
+  * admin:repo_hook
+5) Create the token.
+
+Record it someplace safe.
+

base/codebuild_splunk_apps/iam.tf

@@ -0,0 +1,177 @@
+resource "aws_iam_role" "codebuild_splunk_apps_role" {
+  name     = "codebuild_splunk_apps_role"
+  path     = "/aws_services/"
+
+  assume_role_policy = <<EOF
+{
+    "Version": "2012-10-17",
+    "Statement": [
+      {
+        "Effect": "Allow",
+        "Principal": {
+          "Service": [
+            "codebuild.amazonaws.com"
+            ]
+        },
+        "Action": "sts:AssumeRole"
+      }
+    ]
+  }
+EOF
+}
+
+resource "aws_iam_role_policy_attachment" "codebuild_splunk_apps_role_policy_attach" {
+  role       = aws_iam_role.codebuild_splunk_apps_role.name
+  policy_arn = aws_iam_policy.codebuild_splunk_apps_policy.arn
+}
+
+# Some things about this policy I'm not perfectly sure about, like
+# should the account number be hardcoded?  Also, it reads like we'll have to
+# update it each time we have a new repository added to codecommit - that
+# or we'll need to authorize the codebuild role to be able to pull from any 
+# codecommit repo.  Which may be fine?
+resource "aws_iam_policy" "codebuild_splunk_apps_policy" {
+  name        = "codebuild_splunk_apps_policy"
+  description = "Policy for AWS codebuild to build and store artifacts"
+  path     = "/aws_services/"
+
+  policy = <<EOF
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "arn:${var.aws_partition}:logs:${var.aws_region}:${var.aws_account_id}:log-group:/aws/codebuild/*"
+            ],
+            "Action": [
+                "logs:CreateLogGroup",
+                "logs:CreateLogStream",
+                "logs:PutLogEvents"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "arn:${var.aws_partition}:s3:::codepipeline-${var.aws_region}-*"
+            ],
+            "Action": [
+                "s3:PutObject",
+                "s3:GetObject",
+                "s3:GetObjectVersion"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "arn:${var.aws_partition}:codecommit:${var.aws_region}:${var.aws_account_id}:*"
+            ],
+            "Action": [
+                "codecommit:GitPull"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Resource": [
+                "arn:${var.aws_partition}:s3:::xdr-${var.splunk_prefix}-${var.environment}-splunk-apps/*",
+                "arn:${var.aws_partition}:s3:::*"
+            ],
+            "Action": [
+                "s3:PutObject",
+                "s3:GetObject*",
+                "s3:ListBucket"
+            ]
+        },
+        {
+            "Sid": "WriteToECR",
+            "Effect": "Allow",
+            "Resource": [
+                "*"
+            ],
+            "Action": [
+              "ecr:GetAuthorizationToken",
+              "ecr:BatchCheckLayerAvailability",
+              "ecr:CompleteLayerUpload",
+              "ecr:GetAuthorizationToken",
+              "ecr:InitiateLayerUpload",
+              "ecr:PutImage",
+              "ecr:UploadLayerPart"
+            ]
+        },
+        {
+            "Sid": "PullFromECR",
+            "Effect": "Allow",
+            "Resource": [
+                "*"
+            ],
+            "Action": [
+              "ecr:GetDownloadUrlForLayer",
+              "ecr:BatchGetImage",
+              "ecr:BatchCheckLayerAvailability"
+            ]
+        }
+    ]
+}
+EOF
+}
+
+# !!!!!     RETAINED FOR FUTURE USE   !!!!!
+# Defines an IAM user that can only download ECR images, intended for
+# use in POP nodes where we need containers, but won't necessarily have
+# EC2 instance role credentials.  Maybe one day this goes to vault, I
+# hope.   It would be nice.
+
+# data "aws_iam_policy_document" "ecr_policy_pop" {
+#   statement {
+#     sid       = "AllowECRReadOnly"
+#     effect    = "Allow"
+
+#     actions   = [
+#       "ecr:GetAuthorizationToken",
+#       "ecr:BatchCheckLayerAvailability",
+#       "ecr:GetDownloadUrlForLayer",
+#       "ecr:GetRepositoryPolicy",
+#       "ecr:DescribeRepositories",
+#       "ecr:ListImages",
+#       "ecr:DescribeImages",
+#       "ecr:BatchGetImage"
+#     ]
+    
+#     resources = [
+#       "*"
+#     ]
+
+#   }
+# }
+
+# resource "aws_iam_policy" "ecr_policy_pop" {
+#   name   = "ecr_policy_pop"
+#   path   = "/"
+#   policy = "${data.aws_iam_policy_document.ecr_policy_pop.json}"
+# }
+
+# resource "aws_iam_user" "pop_service_account" {
+#   name = "svc-mdrpop"
+#   path = "/service/"
+# }
+
+# resource "aws_iam_user_policy_attachment" "pop_service_account_1" {
+#   user       = "${aws_iam_user.pop_service_account.name}"
+#   policy_arn = "${aws_iam_policy.ecr_policy_pop.arn}"
+# }
+
+
+# resource "aws_iam_access_key" "pop_service_account" {
+#   user    = "${aws_iam_user.pop_service_account.name}"
+#   pgp_key = "${file("../00-organizations-and-iam/duane_waddle.pgp")}"
+# }
+
+# output "pop_service_account_key_id" {
+#   value = "${aws_iam_access_key.pop_service_account.id}"
+# }
+
+# output "pop_service_account_secret" {
+#   value = "${aws_iam_access_key.pop_service_account.encrypted_secret}"
+# }
+
+# !!!!!    END OF RETAINED FOR FUTURE USE   !!!!!

base/codebuild_splunk_apps/kms.tf

@@ -0,0 +1,148 @@
+#Codebuild artifacts by rule must be encrypted by a KMS key
+# using the default aws/s3 key doesn't work with cross-account access
+resource "aws_kms_key" "s3_codebuild_splunk_apps_artifacts" {
+  description             = "Codebuild Artifacts S3 bucket"
+  enable_key_rotation     = true
+  policy                  = data.aws_iam_policy_document.codebuild_splunk_apps_kms_key_encryption_policy.json
+}
+
+resource "aws_kms_alias" "codebuilt-artifacts" {
+  name          = "alias/codebuild-splunk-apps"
+  target_key_id = aws_kms_key.s3_codebuild_splunk_apps_artifacts.key_id
+}
+
+
+data "aws_iam_policy_document" "codebuild_splunk_apps_kms_key_encryption_policy" {
+  #policy_id = "key-consolepolicy-3"
+  statement {
+    sid = "Enable IAM User Permissions"
+    effect = "Allow"
+    principals {
+      type = "AWS"
+      identifiers = [ 
+        "arn:${var.aws_partition}:iam::${var.aws_account_id}:role/user/mdr_terraformer",
+        "arn:${var.aws_partition}:iam::${var.aws_account_id}:user/MDRAdmin"
+        ]
+    }
+    actions   = [ "kms:*" ]
+    resources = [ "*" ]
+  }
+
+  statement {
+    sid = "Allow access for Key Administrators"
+    effect = "Allow"
+    principals {
+      type = "AWS"
+      identifiers = [
+        "arn:${var.aws_partition}:iam::${var.aws_account_id}:role/user/mdr_terraformer",
+      ]
+    }
+
+    actions = [
+      "kms:Create*",
+      "kms:Describe*",
+      "kms:Enable*",
+      "kms:List*",
+      "kms:Put*",
+      "kms:Update*",
+      "kms:Revoke*",
+      "kms:Disable*",
+      "kms:Get*",
+      "kms:Delete*",
+      "kms:TagResource",
+      "kms:UntagResource",
+      "kms:ScheduleKeyDeletion",
+      "kms:CancelKeyDeletion"
+    ]
+    resources = [ "*" ]
+  }
+
+  statement {
+    sid =  "Allow use of the key"
+    effect = "Allow"
+    principals {
+      type = "AWS"
+      identifiers = [
+        "arn:${var.aws_partition}:iam::${var.aws_account_id}:role/msoc-default-instance-role"
+      ]
+    }
+    actions = [
+      "kms:Encrypt",
+      "kms:Decrypt",
+      "kms:ReEncrypt*",
+      "kms:GenerateDataKey*",
+      "kms:DescribeKey"
+    ]
+    resources = [ "*" ]
+  }
+
+  statement  {
+    sid = "Allow access through Amazon S3 for all principals in the account that are authorized to use Amazon S3"
+    effect = "Allow"
+    principals {
+      type = "AWS"
+      identifiers = [ "*" ]
+    }
+    actions = [
+        "kms:Encrypt",
+        "kms:Decrypt",
+        "kms:ReEncrypt*",
+        "kms:GenerateDataKey*",
+        "kms:DescribeKey"
+    ]
+    resources = [ "*" ]
+
+    condition {
+      test = "StringEquals"
+      variable = "kms.ViaService"
+      values = [ "s3.${var.aws_region}.amazonaws.com" ]
+    }
+
+    condition {
+      test = "StringEquals"
+      variable = "kms.CallerAccount"
+      values = [ var.aws_account_id ]
+    }
+  }
+
+  statement  {
+    sid = "Allow access from the codebuild role"
+    effect = "Allow"
+    principals {
+      type = "AWS"
+    
+      # FIXME this needs to be a better role by far
+      identifiers = [ aws_iam_role.codebuild_splunk_apps_role.arn ]
+    }
+    actions = [
+      "kms:Encrypt",
+      "kms:Decrypt",
+      "kms:ReEncrypt*",
+      "kms:GenerateDataKey*",
+      "kms:DescribeKey"
+    ]
+    resources = [ "*" ]
+  }
+  
+  statement {
+    sid = "Allow attachment of persistent resources"
+    effect = "Allow"
+    principals {
+      type = "AWS"
+      identifiers = [
+        "arn:${var.aws_partition}:iam::${var.aws_account_id}:role/msoc-default-instance-role"
+      ]
+    }
+    actions = [
+      "kms:CreateGrant",
+      "kms:ListGrants",
+      "kms:RevokeGrant"
+    ]
+    resources = [ "*" ]
+    condition {
+      test = "Bool"
+      variable =  "kms:GrantIsForAWSResource"
+      values = [ "true" ]
+      }
+    }
+}

base/codebuild_splunk_apps/main.tf

@@ -0,0 +1,76 @@
+data "github_repository" "this" {
+    name    = "content_source"
+}
+
+#resource "aws_codebuild_source_credential" "github_token" {
+#  auth_type   = "PERSONAL_ACCESS_TOKEN"
+#  server_type = "GITHUB_ENTERPRISE"
+#  token       = "" # This could be used to make life easier, but it would be stored in the state in plaintext.
+#}
+output "Codebuild_AWS_Key_Reminder" {
+  value = "REMINDER: If this is a fresh deployment, you must manually enter the GITHUB token for 'mdr-aws-codebuild' (found in the vault) into one of the codebuild jobs."
+}
+
+resource "aws_codebuild_project" "this" {
+  for_each               = local.splunk_server_types
+
+  name                  = "splunk_apps_${var.splunk_prefix}_${each.value}"
+  description           = "Splunk Application build for ${each.value}"
+  service_role          = aws_iam_role.codebuild_splunk_apps_role.arn
+  encryption_key        = aws_kms_key.s3_codebuild_splunk_apps_artifacts.arn
+  badge_enabled         = var.badge_enabled
+  concurrent_build_limit = 1
+  build_timeout          = 60
+
+  source {
+    type                = "GITHUB_ENTERPRISE"
+    location            = data.github_repository.this.http_clone_url
+    report_build_status = true
+    git_clone_depth     = 1
+  }
+
+  source_version = var.source_version
+
+  environment {
+    compute_type        = "BUILD_GENERAL1_SMALL"
+    image               = "${var.common_services_account}.dkr.ecr.us-gov-east-1.amazonaws.com/content_generator:latest"
+    image_pull_credentials_type = "SERVICE_ROLE"
+    type                = "LINUX_CONTAINER"
+    environment_variable {
+      name = "TAG"
+      type = "PLAINTEXT"
+      value = "${var.splunk_prefix}:${each.value}"
+    }
+  }
+
+  artifacts {
+    type                = "S3"
+    location            = "xdr-${var.splunk_prefix}-${var.environment}-splunk-apps"
+    name                = each.value
+    #path                = each.value
+    namespace_type      = "NONE"
+    packaging           = "NONE"
+  }
+
+  tags = merge(var.standard_tags, var.tags)
+}
+
+#resource "aws_codebuild_webhook" "this" {
+#  project_name  = var.name
+#  branch_filter = var.webhook_branch_filter
+#
+#  depends_on = [ aws_codebuild_project.this  ]
+#}
+
+#resource "github_repository_webhook" "this" {
+#  active     = true
+#  events     = ["push"]
+#  repository = data.github_repository.this.name
+#
+#  configuration {
+#    url          = aws_codebuild_webhook.this.payload_url
+#    secret       = aws_codebuild_webhook.this.secret
+#    content_type = "json"
+#    insecure_ssl = false
+#  }
+#}

base/codebuild_splunk_apps/vars.tf

@@ -0,0 +1,39 @@
+locals {
+  # creates a job for each of these types, using <splunk_prefix>:<server type> as the tag
+  splunk_server_types = toset([
+    #"hf",
+    #"idx",
+    "sh-es",
+    #"sh-cx",
+  ])
+}
+
+variable "source_version" {
+  description = "Tag or branch for the git repository."
+  type = string
+  default = "master"
+}
+
+variable "tags" {
+  description = "Tags to add to the resource (in addition to global standard tags)"
+  type        = map
+  default     = { }
+}
+variable "standard_tags" { type = map }
+variable "environment" { type = string }
+variable "aws_region" { type = string }
+variable "aws_partition" { type = string }
+variable "aws_partition_alias" { type = string }
+variable "aws_account_id" { type = string }
+variable "common_services_account" { type = string }
+variable "splunk_prefix" { type = string }
+
+variable "badge_enabled" {
+    type = string
+    default = "false"
+}
+
+variable "webhook_branch_filter" {
+    type = string
+    default = "^(master|develop)$"
+}

base/splunk_servers/app_s3_bucket/iam_splunk_apps_s3_writer_role.tf → base/splunk_servers/app_s3_bucket/iam_splunk_apps_s3_writer_role.tf.disabled

@@ -11,7 +11,8 @@ resource "aws_iam_role" "splunk_apps_s3_writer_role" {
       "Effect": "Allow",
       "Principal": {
         "AWS": [
-          "arn:${var.aws_partition}:iam::${var.c2_accounts[var.aws_partition]}:role/salt-master-instance-role"
+          "arn:${var.aws_partition}:iam::${var.c2_accounts[var.aws_partition]}:role/salt-master-instance-role",
+          "arn:${var.aws_partition}:iam::${var.c2_accounts[var.aws_partition]}:role/lambda/lambda-content-generator"
         ]
       },
       "Action": "sts:AssumeRole"

base/splunk_servers/app_s3_bucket/main.tf

@@ -57,30 +57,29 @@ resource "aws_s3_bucket_public_access_block" "public_access_block" {
   depends_on = [aws_s3_bucket_policy.policy]
 }
 
-resource "aws_s3_bucket_policy" "policy" {
-  bucket = aws_s3_bucket.bucket.id
+data "aws_iam_policy_document" "policy" {
+  statement {
+    sid       = "AccountAllow"
+    effect    = "Allow"
+    resources = [
+      "${aws_s3_bucket.bucket.arn}",
+      "${aws_s3_bucket.bucket.arn}/*"
+    ]
 
-  policy = <<POLICY
-{
-  "Version": "2012-10-17",
-  "Id": "AllowThisAccount",
-  "Statement": [
-    {
-      "Sid": "AccountAllow",
-      "Effect": "Allow",
-      "Principal": {
-        "AWS": ${jsonencode(local.account_arns)}
-      },
-      "Action": [
-        "s3:GetObject",
-        "s3:ListBucket"
-      ],
-      "Resource": [
-        "${aws_s3_bucket.bucket.arn}",
-        "${aws_s3_bucket.bucket.arn}/*"
-      ]
+    actions = [
+      "s3:GetObject",
+      "s3:ListBucket",
+    ]
+
+    principals {
+      type        = "AWS"
+      identifiers = local.account_arns
     }
-  ]
+  }
 }
-POLICY
+
+resource "aws_s3_bucket_policy" "policy" {
+  bucket = aws_s3_bucket.bucket.id
+
+  policy = data.aws_iam_policy_document.policy.json
 }

base/splunk_servers/app_s3_bucket/outputs.tf

@@ -6,6 +6,6 @@ output "Splunk_Role_ARN" {
   value = aws_iam_role.splunk_apps_s3_role.arn
 }
 
-output "Writer_Role_ARN" {
-  value = aws_iam_role.splunk_apps_s3_writer_role.arn
-}
+#output "Writer_Role_ARN" {
+#  value = aws_iam_role.splunk_apps_s3_writer_role.arn
+#}