MSOCI-2063 codebuild for RPM building container images

base/codebuild_ecr_base/README.md

@@ -1,9 +1,20 @@
 # CodeBuild ECR Base
 
-This module is the building blocks for the AWS CodeBuild projects. No CodeBuild projects should be included in this module. This module is for AWS objects that all or many CodeBuild projects require. 
+This module is the building blocks for the AWS CodeBuild projects.  It has the
+foundational pieces for using codebuild to build Linux software / RPMs. You may
+need to compile an RPM for a specific release of an OS, and CodeBuild by default only
+has Docker images for Amazon Linux 2 and Ubuntu.  So part of this builds RHEL Docker
+images so that you can compile software in a RHEL userspace.
 
-When starting from scratch, the RHEL image needs to be manually uploaded to ECR. The TF creates the ECR repo.
+It builds these resources:
 
-- Run the TF
-- Upload the RHEL docker images needed for the CodeBuild projects
+* IAM role for Codebuild jobs to run and be launched by Eventbridge (codebuild-role)
+* S3 bucket for holding Codebuild artifacts (xdr-codebuild-artifacts)
+* ECR Repos for OS-specific containers
+* Codebuild jobs to `docker build` containers for the OS-specific containers
+  * RHEL 7
+  * RHEL 8
+  * RHEL 9 beta (not working right now)
 
+The ECR repo and related codebuild jobs use a submodule.  The OS-specific containers are
+rebuilt daily to keep them up to date with the latest RHEL patches.

base/codebuild_ecr_base/iam.tf

@@ -9,7 +9,8 @@ resource "aws_iam_role" "codebuild_role" {
         "Effect": "Allow",
         "Principal": {
           "Service": [
-            "codebuild.amazonaws.com"
+            "codebuild.amazonaws.com",
+            "events.amazonaws.com"
             ]
         },
         "Action": "sts:AssumeRole"
@@ -24,11 +25,12 @@ resource "aws_iam_role_policy_attachment" "codebuild_role_policy_attach" {
   policy_arn = aws_iam_policy.codebuild_policy.arn
 }
 
-# Some things about this policy I'm not perfectly sure about, like
-# should the account number be hardcoded?  Also, it reads like we'll have to
-# update it each time we have a new repository added to codecommit - that
-# or we'll need to authorize the codebuild role to be able to pull from any 
-# codecommit repo.  Which may be fine?
+# FIXME: Not sure about this policy
+#   1.  Lets codebuild (apparently) write to ANY s3 bucket
+#   2.  Lets codebuild (apparently) write to ANY ECR repo
+#   3.  It's in JSON instead of a terraform data source so these comments
+#           have to be at the top instead of inline where they would make sense.
+#   4.  Latest codebuild policies (from AWS console) have report-group resources and actions
 resource "aws_iam_policy" "codebuild_policy" {
   name        = "codebuild_policy"
   description = "Policy for AWS codebuild to build and store artifacts"
@@ -94,6 +96,17 @@ resource "aws_iam_policy" "codebuild_policy" {
               "ecr:PutImage",
               "ecr:UploadLayerPart"
             ]
+        },
+        {
+            "Action": [
+              "codebuild:StartBuild",
+              "codebuild:StopBuild",
+              "codebuild:BatchGet*",
+              "codebuild:Get*",
+              "codebuild:List*"
+            ],
+            "Effect": "Allow",
+            "Resource": "*"
         }
     ]
 }
@@ -159,4 +172,4 @@ EOF
 #   value = "${aws_iam_access_key.pop_service_account.encrypted_secret}"
 # }
 
-# !!!!!    END OF RETAINED FOR FUTURE USE   !!!!!
+# !!!!!    END OF RETAINED FOR FUTURE USE   !!!!!

base/codebuild_ecr_base/main.tf

@@ -1,67 +0,0 @@
-
-#Base RHEL repository used for building XDR RPMs like syslog-ng in CodeBuild
-resource "aws_ecr_repository" "codebuild-rhel7" {
-  name  = "codebuild-rhel7"
-  tags = merge(var.standard_tags, var.tags)
-
-  image_scanning_configuration {
-    scan_on_push = true
-  }
-}
-
-#Allow codebuild to access the ECR Repository
-resource "aws_ecr_repository_policy" "codebuild-rhel7" {
-  repository = aws_ecr_repository.codebuild-rhel7.name
-
-  policy = <<EOF
-{
-    "Version": "2008-10-17",
-    "Statement": [
-        {
-            "Sid": "new statement",
-            "Effect": "Allow",
-            "Principal": {
-                "Service": "codebuild.amazonaws.com"
-            },
-            "Action": [
-                "ecr:GetDownloadUrlForLayer",
-                "ecr:BatchGetImage",
-                "ecr:BatchCheckLayerAvailability"
-            ]
-        }
-    ]
-}
-EOF
-}
-
-# not needed, but leaving the code for possible future use. 
-# #base centos7 image used for building portal
-# resource "aws_ecr_repository" "codebuild-centos7" {
-#   name     = "codebuild-centos7"
-#   tags = merge(var.standard_tags, var.tags)
-# }
-
-# #Allow codebuild to access the ECR Repository
-# resource "aws_ecr_repository_policy" "codebuild-centos7" {
-#   repository = aws_ecr_repository.codebuild-centos7.name
-
-#   policy = <<EOF
-# {
-#     "Version": "2008-10-17",
-#     "Statement": [
-#         {
-#             "Sid": "new statement",
-#             "Effect": "Allow",
-#             "Principal": {
-#                 "Service": "codebuild.amazonaws.com"
-#             },
-#             "Action": [
-#                 "ecr:GetDownloadUrlForLayer",
-#                 "ecr:BatchGetImage",
-#                 "ecr:BatchCheckLayerAvailability"
-#             ]
-#         }
-#     ]
-# }
-# EOF
-# }

base/codebuild_ecr_base/outputs.tf

@@ -14,6 +14,6 @@ output artifact_s3_bucket {
 #   value = aws_ecr_repository.codebuild-centos7.repository_url
 # }
 
-output codebuild_image_rhel7 {
-  value = aws_ecr_repository.codebuild-rhel7.repository_url
-}
+#output codebuild_image_rhel7 {
+#  value = aws_ecr_repository.codebuild-rhel7.repository_url
+#}

base/codebuild_ecr_base/repos.tf

@@ -0,0 +1,84 @@
+
+module "codebuild-rhel7" {
+  source = "../../submodules/codebuild/codebuild-ecr-image"
+
+  name             = "codebuild-rhel7"
+  github_clone_url = "https://github.xdr.accenturefederalcyber.com/mdr-engineering/mdr-codebuild-images.git"
+  source_version   = "master"
+  standard_tags    = var.standard_tags
+  tags             = var.tags
+
+  codebuild_assume_role_arn = aws_iam_role.codebuild_role.arn
+  schedule_expression       = "cron(05 16 * * ? *)"
+
+  env_vars = {
+    "IMAGE_REPO_NAME" = {
+      value = "codebuild-rhel7"
+    }
+    "IMAGE_TAG" = {
+      value = "latest"
+    }
+    "AWS_ACCOUNT_ID" = {
+      value = var.aws_account_id
+    }
+    "DOCKERBUILDPATH" = {
+      value = "rhel7"
+    }
+  }
+}
+
+module "codebuild-rhel8" {
+  source = "../../submodules/codebuild/codebuild-ecr-image"
+
+  name             = "codebuild-rhel8"
+  github_clone_url = "https://github.xdr.accenturefederalcyber.com/mdr-engineering/mdr-codebuild-images.git"
+  source_version   = "master"
+  standard_tags    = var.standard_tags
+  tags             = var.tags
+
+  codebuild_assume_role_arn = aws_iam_role.codebuild_role.arn
+  schedule_expression       = "cron(05 16 * * ? *)"
+
+  env_vars = {
+    "IMAGE_REPO_NAME" = {
+      value = "codebuild-rhel8"
+    }
+    "IMAGE_TAG" = {
+      value = "latest"
+    }
+    "AWS_ACCOUNT_ID" = {
+      value = var.aws_account_id
+    }
+    "DOCKERBUILDPATH" = {
+      value = "rhel8"
+    }
+  }
+}
+
+module "codebuild-rhel9beta" {
+  source = "../../submodules/codebuild/codebuild-ecr-image"
+
+  name             = "codebuild-rhel9beta"
+  github_clone_url = "https://github.xdr.accenturefederalcyber.com/mdr-engineering/mdr-codebuild-images.git"
+  source_version   = "master"
+  standard_tags    = var.standard_tags
+  tags             = var.tags
+
+  codebuild_assume_role_arn = aws_iam_role.codebuild_role.arn
+  schedule_expression       = "cron(05 16 * * ? *)"
+
+  env_vars = {
+    "IMAGE_REPO_NAME" = {
+      value = "codebuild-rhel9beta"
+    }
+    "IMAGE_TAG" = {
+      value = "latest"
+    }
+    "AWS_ACCOUNT_ID" = {
+      value = var.aws_account_id
+    }
+    "DOCKERBUILDPATH" = {
+      value = "rhel9-beta"
+    }
+  }
+}

base/codebuild_ecr_base/s3.tf

@@ -27,7 +27,7 @@ data "aws_iam_policy_document" "artifacts" {
     resources = [ "${aws_s3_bucket.artifacts.arn}/*" ]
     principals {
       type = "AWS"
-      identifiers = [ for a in var.responsible_accounts[var.environment]: "arn:${var.aws_partition}:iam::${a}:root" ]
+      identifiers = sort([ for a in var.responsible_accounts[var.environment]: "arn:${var.aws_partition}:iam::${a}:root" ])
     }
   }
 }

base/codebuild_ecr_project/main.tf

@@ -64,7 +64,7 @@ data "aws_iam_policy_document" "ecr_cross_account_policy" {
     ]
     principals {
       type = "AWS"
-      identifiers = [ for a in var.responsible_accounts[var.environment]: "arn:${var.aws_partition}:iam::${a}:root" ]
+      identifiers = sort([ for a in var.responsible_accounts[var.environment]: "arn:${var.aws_partition}:iam::${a}:root" ])
     }
   }
   # Allow codebuild access

submodules/codebuild/codebuild-ecr-image/codebuild.tf

@@ -0,0 +1,59 @@
+resource "aws_codebuild_project" "this" {
+  name                  = var.name
+  description           = "Container for ${var.name}"
+  service_role          = var.codebuild_assume_role_arn
+  #encryption_key        = var.kms_key
+  #badge_enabled         = var.badge_enabled
+
+  source {
+    type                = "GITHUB_ENTERPRISE"
+    #location           = data.github_repository.this.http_clone_url
+    location            = var.github_clone_url
+    report_build_status = true
+    git_clone_depth     = 1
+    buildspec           = var.buildspec
+  }
+
+  source_version = var.source_version
+
+  environment {
+    compute_type        = "BUILD_GENERAL1_SMALL"
+    image               = "aws/codebuild/amazonlinux2-x86_64-standard:3.0"
+    type                = "LINUX_CONTAINER"
+    privileged_mode     = true
+
+
+    dynamic "environment_variable" {
+      for_each = var.env_vars
+      iterator = each
+      content {
+        name = each.key
+        value = each.value["value"]
+        type = try(each.value["type"],"PLAINTEXT")
+
+      }
+
+    }
+
+  }
+
+  artifacts {
+    type                = "NO_ARTIFACTS"
+  }
+
+  tags = merge(var.standard_tags, var.tags)
+}
+
+resource "aws_cloudwatch_event_rule" "schedule_rule" {
+  count               = var.schedule_expression=="" ? 0 : 1
+  name                = "scheduled_build-${var.name}"
+  schedule_expression = var.schedule_expression
+}
+
+resource "aws_cloudwatch_event_target" "trigger_build" {
+  count     = var.schedule_expression=="" ? 0 : 1
+  target_id = "trigger_build"
+  rule      = aws_cloudwatch_event_rule.schedule_rule[count.index].name
+  arn       = aws_codebuild_project.this.id
+  role_arn  = var.codebuild_assume_role_arn
+}

submodules/codebuild/codebuild-ecr-image/ecr_repo.tf

@@ -0,0 +1,55 @@
+
+resource "aws_ecr_repository" "this" {
+  name  = var.name
+  tags = merge(var.standard_tags, var.tags)
+
+  image_scanning_configuration {
+    scan_on_push = true
+  }
+}
+
+data "aws_iam_policy_document" "ecr_repository_policy" {
+  statement {
+    sid    = "LetCodebuildServiceUseTheseImages"
+    effect = "Allow"
+    principals {
+      type        = "Service"
+      identifiers = [ "codebuild.amazonaws.com" ]
+    }
+    actions = [
+      "ecr:GetDownloadUrlForLayer",
+      "ecr:BatchGetImage",
+      "ecr:BatchCheckLayerAvailability"
+    ]
+  }
+
+  statement {
+    sid    = "LetCodebuildIAMRolePushImagesHere"
+    effect = "Allow"
+    principals {
+      type        = "AWS"
+      identifiers = [ var.codebuild_assume_role_arn ]
+    }
+    actions = [
+      "ecr:BatchCheckLayerAvailability",
+      "ecr:BatchGetImage",
+      "ecr:CompleteLayerUpload",
+      "ecr:DescribeImages",
+      "ecr:DescribeRepositories",
+      "ecr:GetAuthorizationToken",
+      "ecr:GetDownloadUrlForLayer",
+      "ecr:InitiateLayerUpload",
+      "ecr:ListImages",
+      "ecr:PutImage",
+      "ecr:UploadLayerPart",
+    ]
+  }
+
+}
+
+#Allow codebuild to access the ECR Repository to use the images
+resource "aws_ecr_repository_policy" "this" {
+  repository = aws_ecr_repository.this.name
+  policy     = data.aws_iam_policy_document.ecr_repository_policy.json
+}
+

submodules/codebuild/codebuild-ecr-image/vars.tf

@@ -0,0 +1,29 @@
+variable "tags" {
+  description = "Tags to add to the resource (in addition to global standard tags)"
+  type        = map
+  default     = { }
+}
+variable "standard_tags" { type = map }
+
+variable "name" { type=string }
+variable "codebuild_assume_role_arn" { type=string }
+variable "github_clone_url" { type=string }
+variable "source_version" { type=string }
+variable "env_vars" { type = map }
+variable "buildspec" {
+  type    = string
+  default = "buildspec.yml"
+}
+
+variable "schedule_expression" {
+  type    = string
+  default = ""
+}
+
+#variable "environment" { type = string }
+#variable "aws_partition" { type = string }
+#variable "aws_region" { type = string }
+#variable "aws_partition_alias" { type = string }
+#variable "aws_account_id" { type = string }
+#variable "common_services_account" { type = string }
+#variable "responsible_accounts" { type = map(list(string)) }