Getting closer

.gitignore

@@ -1,5 +1,6 @@
 .terraform
 
+.swp
 *.bak
 .*.swp
 

.tfsec.yaml

@@ -1,4 +1,5 @@
 ---
 exclude:
-  - aws-dynamodb-table-customer-key # We don't care about default keys, encryption is fine
+  - aws-dynamodb-table-customer-key # We don't care about default keys, any encryption is fine
+  - aws-s3-encryption-customer-key  # We don't care about default keys, any encryption is fine
   - aws-iam-no-policy-wildcards # We allow wildcards for sensible iam policies

config.tf

@@ -7,7 +7,8 @@ locals {
   region  = "us-east-2"
   tags = {
     "tf_module" : basename(abspath(".")),
-    "project" : "monkeybox_emr_lab"
+    "project" : "monkeybox_emr_lab",
+    "for-use-with-amazon-emr-managed-policies" : true,
   }
 }
 

emr.tf

@@ -19,11 +19,11 @@ resource "aws_emr_cluster" "cluster" {
     target_on_demand_capacity = 0
     target_spot_capacity      = 1
     instance_type_configs {
-      instance_type                              = "m5.xlarge"
+      instance_type                              = "m5.2xlarge"
       bid_price_as_percentage_of_on_demand_price = 50
     }
     instance_type_configs {
-      instance_type                              = "m5.large"
+      instance_type                              = "m5.xlarge"
       bid_price_as_percentage_of_on_demand_price = 70
     }
     launch_specifications {
@@ -50,7 +50,7 @@ resource "aws_emr_cluster" "cluster" {
         type                 = "gp3"
         volumes_per_instance = 1
       }
-      instance_type     = "m5.xlarge"
+      instance_type     = "m5.2xlarge"
       weighted_capacity = 1
     }
     instance_type_configs {
@@ -60,7 +60,7 @@ resource "aws_emr_cluster" "cluster" {
         type                 = "gp3"
         volumes_per_instance = 1
       }
-      instance_type     = "m5.large"
+      instance_type     = "m5.xlarge"
       weighted_capacity = 2
     }
     launch_specifications {
@@ -152,13 +152,13 @@ resource "aws_emr_cluster" "cluster" {
 #EOF
 
 
-resource "aws_emr_managed_scaling_policy" "samplepolicy" {
+resource "aws_emr_managed_scaling_policy" "cluster-asg-policy" {
   cluster_id = aws_emr_cluster.cluster.id
   compute_limits {
     unit_type                       = "Instances"
     minimum_capacity_units          = 1
-    maximum_capacity_units          = 6
+    maximum_capacity_units          = 10
     maximum_ondemand_capacity_units = 2
-    maximum_core_capacity_units     = 6
+    maximum_core_capacity_units     = 10
   }
 }

iam.tf

@@ -4,7 +4,7 @@
 
 # IAM role for EMR Service
 resource "aws_iam_role" "iam_emr_service_role" {
-  name_prefix = local.unique_id
+  name_prefix = substr("svc-role-${local.unique_id}-", 0, 38)
 
   assume_role_policy = <<EOF
 {
@@ -86,15 +86,20 @@ data "aws_iam_policy_document" "iam_emr_service_policy" {
 }
 
 resource "aws_iam_role_policy" "iam_emr_service_policy" {
-  name_prefix = local.unique_id
+  name_prefix = substr("svc-pol-${local.unique_id}-", 0, 38)
   role        = aws_iam_role.iam_emr_service_role.id
 
   policy = data.aws_iam_policy_document.iam_emr_service_policy.json
 }
 
+resource "aws_iam_role_policy_attachment" "iam_emr_service_policy" {
+  role       = aws_iam_role.iam_emr_service_role.name
+  policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonEMRServicePolicy_v2"
+}
+
 # IAM Role for EC2 Instance Profile
 resource "aws_iam_role" "iam_emr_profile_role" {
-  name_prefix = local.unique_id
+  name_prefix = substr("emr-prof-${local.unique_id}-", 0, 38)
 
   assume_role_policy = <<EOF
 {
@@ -114,7 +119,7 @@ EOF
 }
 
 resource "aws_iam_instance_profile" "emr_profile" {
-  name_prefix = local.unique_id
+  name_prefix = substr("emr-prof-${local.unique_id}-", 0, 38)
   role        = aws_iam_role.iam_emr_profile_role.name
 }
 
@@ -147,12 +152,36 @@ data "aws_iam_policy_document" "iam_emr_profile_policy" {
       "sdb:*",
       "sns:*",
       "sqs:*",
+      "glue:CreateDatabase",
+      "glue:UpdateDatabase",
+      "glue:DeleteDatabase",
+      "glue:GetDatabase",
+      "glue:GetDatabases",
+      "glue:CreateTable",
+      "glue:UpdateTable",
+      "glue:DeleteTable",
+      "glue:GetTable",
+      "glue:GetTables",
+      "glue:GetTableVersions",
+      "glue:CreatePartition",
+      "glue:BatchCreatePartition",
+      "glue:UpdatePartition",
+      "glue:DeletePartition",
+      "glue:BatchDeletePartition",
+      "glue:GetPartition",
+      "glue:GetPartitions",
+      "glue:BatchGetPartition",
+      "glue:CreateUserDefinedFunction",
+      "glue:UpdateUserDefinedFunction",
+      "glue:DeleteUserDefinedFunction",
+      "glue:GetUserDefinedFunction",
+      "glue:GetUserDefinedFunctions"
     ]
   }
 }
 
 resource "aws_iam_role_policy" "iam_emr_profile_policy" {
-  name_prefix = local.unique_id
+  name_prefix = substr("emr-prof-${local.unique_id}-", 0, 38)
   role        = aws_iam_role.iam_emr_profile_role.id
 
   policy = data.aws_iam_policy_document.iam_emr_profile_policy.json

s3-emr.tf

@@ -23,8 +23,8 @@ resource "aws_s3_bucket_server_side_encryption_configuration" "s3_emr" {
 
   rule {
     apply_server_side_encryption_by_default {
-      kms_master_key_id = "aws/s3" # Better to use a cmk
-      sse_algorithm     = "aws:kms"
+      #kms_master_key_id = "aws/s3" # Better to use a cmk
+      sse_algorithm = "aws:kms"
     }
   }
 }
@@ -65,11 +65,10 @@ resource "aws_s3_bucket_lifecycle_configuration" "s3_emr" {
 }
 
 resource "aws_s3_object" "food-services" {
-  bucket = aws_s3_bucket.s3_emr.id
-  key    = "aws_examples/food_establishment_data.csv"
-  source = "data/food_establishment_data.csv"
-
-  etag = filemd5("data/food_establishment_data.csv")
+  bucket      = aws_s3_bucket.s3_emr.id
+  key         = "aws_examples/food_establishment_data.csv"
+  source      = "data/food_establishment_data.csv"
+  source_hash = filemd5("data/food_establishment_data.csv") # Update if this changes
 
   tags = local.tags
 }