Final Updates for Portal

Switches from LC to LT
Sets up user-data for ASG

base/customer_portal/README.md

@@ -1,3 +1,30 @@
 # xdr customer portal instances
 
-Builds and configures the instances that host the customer portal website.
+Builds and configures the instances that host the customer portal website. NOTE: the grain ec2_tags:Name and the pillar aws_registry_account are required for portal salt state to complete successfully. 
+
+New Portal Server setup steps 
+- test.version   				# are we on the correct salt version?
+- saltutil.sync_all             
+- saltutil.refresh_modules      # refresh grains
+- saltutil.refresh_pillar       # refresh pillars
+- pillar.get aws_registry_account  # This one is needed
+- slsutil.renderer salt://docker/portal.sls # Does this render properly?
+- grains.get environment 		# make sure "test" is present
+- state.sls os_modifications 	# get some base stuff out of the way
+- grains.get ec2_tags:Name 			# make sure customer-portal is present for highstate to work
+- state.highstate               # push everything including docker and docker images
+
+## Vault Auth Issues
+
+HELP! I destroyed then recreated the AWS IAM Portal Role and now Vault will not let me log in!!
+
+In Vault disable the auth method
+`vault auth disable aws`
+
+Then in terraform reapply the config. 
+`terragrunt-local apply -target=vault_auth_backend.aws -target=vault_aws_auth_backend_client.aws -target=vault_aws_auth_backend_role.portal`
+
+Vault apparently caches the AWS response for the portal IAM role. 
+
+https://blog.gruntwork.io/a-guide-to-automating-hashicorp-vault-3-authenticating-with-an-iam-user-or-role-a3203a3ee088 
+It is important to note that although the Vault Role is configured with the IAM principal ARN, what Vault actually checks against is a unique internal ID from AWS. So if you destroy and recreate your IAM Role, Vault will reject the login attempt.

base/customer_portal/cloud-init/cloud-init.tpl

@@ -63,6 +63,29 @@ runcmd:
  - /bin/systemctl enable amazon-ssm-agent
  - /usr/sbin/aide --update --verbose=0
  - /bin/cp /var/lib/aide/aide.db.new.gz /var/lib/aide/aide.db.gz
+ # Allow Salt to setup Portal for autoscaling group
+ - "/bin/echo MARKER: START SALT SYNC"
+ - /bin/salt-call saltutil.sync_all refresh=True
+ # Chicken/egg problem. We need pillars to get correct grains, and grains to get correct pillars.
+ # Sleep needs to be this long due to the magical forces inside of the salt master that govern pillars. 
+ # It takes 30 minutes to fully start portal docker container. 
+ - /bin/sleep 420
+ - /bin/salt-call --refresh-grains-cache saltutil.refresh_modules
+ - /bin/sleep 60
+ - /bin/salt-call --refresh-grains-cache saltutil.refresh_grains
+ - /bin/sleep 60
+ - /bin/salt-call --refresh-grains-cache saltutil.refresh_pillar
+ - /bin/sleep 60
+  # Recording our initial values is useful for troubleshooting
+ - /bin/salt-call pillar.get aws_registry_account --out=text > /root/pillar.aws_registry_account.yml
+ - /bin/salt-call pillar.items > /root/pillars.initial_highstate.yml
+ - /bin/salt-call grains.items > /root/grains.initial_highstate.yml
+ - "/bin/echo MARKER: START FIRST HIGHSTATE"
+ - /bin/salt-call state.highstate
+ - "/bin/echo MARKER: END FIRST HIGHSTATE"
+ - "/bin/echo MARKER: START SECOND HIGHSTATE"
+ - /bin/salt-call state.highstate
+ - "/bin/echo MARKER: END SECOND HIGHSTATE"
 
 
 # Either final message or power state, but probably not both

base/customer_portal/ecr.tf

@@ -51,7 +51,7 @@ data "aws_iam_policy_document" "portal_server_ecr_policy" {
   }
 
   statement {
-    sid    = "AllowCluCommunicationECR"
+    sid    = "AllowCommunicationECR"
     effect = "Allow"
 
     actions = [

base/customer_portal/elb.tf

@@ -69,6 +69,12 @@ resource "aws_lb_listener" "portal_https_redirect" {
   }
 }
 
+# Attach the instances to the ELB
+resource "aws_autoscaling_attachment" "customer_portal_asg_attachment" {
+  alb_target_group_arn = aws_alb_target_group.portal.arn
+  autoscaling_group_name = aws_autoscaling_group.customer_portal.name
+}
+
 #----------
 # DNS Entry
 #----------

base/customer_portal/main.tf

@@ -20,113 +20,279 @@ data "aws_kms_key" "ebs-key" {
 #------------------------------------
 # TODO: switch this to Launch Template for gp3 volume usage. 
 # https://github.com/terraform-community-modules/tf_aws_asg_elb/issues/11
-module "customer_portal_asg" {
-  source  = "terraform-aws-modules/autoscaling/aws"
-  version = "3.9.0"
-  name = "customer-portal"
-
-  lc_name = "customer-portal-lc"
-
-  iam_instance_profile = aws_iam_instance_profile.portal_server_instance_profile.name
-  image_id             = local.ami_map[local.ami_selection]
-  instance_type        = var.instance_type
-  security_groups      = [ data.aws_security_group.typical-host.id, aws_security_group.customer_portal.id ]
-  user_data            = data.template_cloudinit_config.cloud-init.rendered
-  key_name             = "msoc-build"
-  ebs_optimized        = true
-  target_group_arns    = [ aws_alb_target_group.portal.arn, ]
-
-  root_block_device = [
-      {
-        volume_type = "gp2"
-        volume_size = "100"
-        delete_on_termination = true
-        encrypted = true
-        kms_key_id = data.aws_kms_key.ebs-key.arn
-      },
-  ]
-
-    ebs_block_device = [
-      {
-        # swap
-        device_name = "/dev/xvdm"
-        #volume_size = xx
-        delete_on_termination = true
-        encrypted = true
-        kms_key_id = data.aws_kms_key.ebs-key.arn
-        # Snapshot IDs need to be grabbed from the ami, or it will replace every time. It's ugly.
-        # This may prompt replacement when the AMI is updated.
-        # See:
-        #   https://github.com/hashicorp/terraform/issues/19958
-        #   https://github.com/terraform-providers/terraform-provider-aws/issues/13118
-        snapshot_id = local.block_device_mappings[local.ami_selection]["/dev/xvdm"].ebs.snapshot_id
-      },
-      {
-        # /home
-        device_name = "/dev/xvdn"
-        # volume_size = xx
-        delete_on_termination = true
-        encrypted = true
-        kms_key_id = data.aws_kms_key.ebs-key.arn
-        snapshot_id = local.block_device_mappings[local.ami_selection]["/dev/xvdn"].ebs.snapshot_id
-      },
-      {
-        # /var
-        device_name = "/dev/xvdo"
-        # volume_size = xx
-        delete_on_termination = true
-        encrypted = true
-        kms_key_id = data.aws_kms_key.ebs-key.arn
-        snapshot_id = local.block_device_mappings[local.ami_selection]["/dev/xvdo"].ebs.snapshot_id
-      },
-      {
-        # /var/tmp
-        device_name = "/dev/xvdp"
-        # volume_size = xx
-        delete_on_termination = true
-        encrypted = true
-        kms_key_id = data.aws_kms_key.ebs-key.arn
-        snapshot_id = local.block_device_mappings[local.ami_selection]["/dev/xvdp"].ebs.snapshot_id
-      },
-      {
-        # /var/log
-        device_name = "/dev/xvdq"
-        # volume_size = xx
-        delete_on_termination = true
-        encrypted = true
-        kms_key_id = data.aws_kms_key.ebs-key.arn
-        snapshot_id = local.block_device_mappings[local.ami_selection]["/dev/xvdq"].ebs.snapshot_id
-      },
-      {
-        # /var/log/audit
-        device_name = "/dev/xvdr"
-        # volume_size = xx
-        delete_on_termination = true
-        encrypted = true
-        kms_key_id = data.aws_kms_key.ebs-key.arn
-        snapshot_id = local.block_device_mappings[local.ami_selection]["/dev/xvdr"].ebs.snapshot_id
-      },
-      {
-        # /tmp
-        device_name = "/dev/xvds"
-        # volume_size = xx
-        delete_on_termination = true
-        encrypted = true
-        kms_key_id = data.aws_kms_key.ebs-key.arn
-        snapshot_id = local.block_device_mappings[local.ami_selection]["/dev/xvds"].ebs.snapshot_id
-      },
-    ]
-
-
-  # Auto scaling group
-  asg_name                  = "customer-portal-asg"
-  vpc_zone_identifier       = var.private_subnets
-  health_check_type         = "EC2"
+# module "customer_portal_asg" {
+#   source  = "terraform-aws-modules/autoscaling/aws"
+#   version = "3.9.0"
+#   name = "customer-portal"
+
+#   lc_name = "customer-portal-lc"
+
+#   iam_instance_profile = aws_iam_instance_profile.portal_server_instance_profile.name
+#   image_id             = local.ami_map[local.ami_selection]
+#   instance_type        = var.instance_type
+#   security_groups      = [ data.aws_security_group.typical-host.id, aws_security_group.customer_portal.id ]
+#   user_data            = data.template_cloudinit_config.cloud-init.rendered
+#   key_name             = "msoc-build"
+#   ebs_optimized        = true
+#   target_group_arns    = [ aws_alb_target_group.portal.arn, ]
+
+#   root_block_device = [
+#       {
+#         volume_type = "gp2"
+#         volume_size = "100"
+#         delete_on_termination = true
+#         encrypted = true
+#         kms_key_id = data.aws_kms_key.ebs-key.arn
+#       },
+#   ]
+
+#     ebs_block_device = [
+#       {
+#         # swap
+#         device_name = "/dev/xvdm"
+#         #volume_size = xx
+#         delete_on_termination = true
+#         encrypted = true
+#         kms_key_id = data.aws_kms_key.ebs-key.arn
+#         # Snapshot IDs need to be grabbed from the ami, or it will replace every time. It's ugly.
+#         # This may prompt replacement when the AMI is updated.
+#         # See:
+#         #   https://github.com/hashicorp/terraform/issues/19958
+#         #   https://github.com/terraform-providers/terraform-provider-aws/issues/13118
+#         snapshot_id = local.block_device_mappings[local.ami_selection]["/dev/xvdm"].ebs.snapshot_id
+#       },
+#       {
+#         # /home
+#         device_name = "/dev/xvdn"
+#         # volume_size = xx
+#         delete_on_termination = true
+#         encrypted = true
+#         kms_key_id = data.aws_kms_key.ebs-key.arn
+#         snapshot_id = local.block_device_mappings[local.ami_selection]["/dev/xvdn"].ebs.snapshot_id
+#       },
+#       {
+#         # /var
+#         device_name = "/dev/xvdo"
+#         # volume_size = xx
+#         delete_on_termination = true
+#         encrypted = true
+#         kms_key_id = data.aws_kms_key.ebs-key.arn
+#         snapshot_id = local.block_device_mappings[local.ami_selection]["/dev/xvdo"].ebs.snapshot_id
+#       },
+#       {
+#         # /var/tmp
+#         device_name = "/dev/xvdp"
+#         # volume_size = xx
+#         delete_on_termination = true
+#         encrypted = true
+#         kms_key_id = data.aws_kms_key.ebs-key.arn
+#         snapshot_id = local.block_device_mappings[local.ami_selection]["/dev/xvdp"].ebs.snapshot_id
+#       },
+#       {
+#         # /var/log
+#         device_name = "/dev/xvdq"
+#         # volume_size = xx
+#         delete_on_termination = true
+#         encrypted = true
+#         kms_key_id = data.aws_kms_key.ebs-key.arn
+#         snapshot_id = local.block_device_mappings[local.ami_selection]["/dev/xvdq"].ebs.snapshot_id
+#       },
+#       {
+#         # /var/log/audit
+#         device_name = "/dev/xvdr"
+#         # volume_size = xx
+#         delete_on_termination = true
+#         encrypted = true
+#         kms_key_id = data.aws_kms_key.ebs-key.arn
+#         snapshot_id = local.block_device_mappings[local.ami_selection]["/dev/xvdr"].ebs.snapshot_id
+#       },
+#       {
+#         # /tmp
+#         device_name = "/dev/xvds"
+#         # volume_size = xx
+#         delete_on_termination = true
+#         encrypted = true
+#         kms_key_id = data.aws_kms_key.ebs-key.arn
+#         snapshot_id = local.block_device_mappings[local.ami_selection]["/dev/xvds"].ebs.snapshot_id
+#       },
+#     ]
+
+
+#   # Auto scaling group
+#   asg_name                  = "customer-portal-asg"
+#   vpc_zone_identifier       = var.private_subnets
+#   health_check_type         = "EC2"
+#   min_size                  = 1
+#   max_size                  = 2
+#   desired_capacity          = 2
+#   wait_for_capacity_timeout = 0
+#   tags_as_map = merge(var.standard_tags, var.tags)
+# }
+
+resource "aws_launch_template" "customer_portal" {
+  name          = "customer-portal-lt"
+  instance_type = var.instance_type
+  image_id      = local.ami_map[local.ami_selection]
+  user_data     = data.template_cloudinit_config.cloud-init.rendered
+  ebs_optimized = true
+  tags          = merge(var.standard_tags, var.tags)
+  key_name      = "msoc-build"
+
+  iam_instance_profile {
+    name = aws_iam_instance_profile.portal_server_instance_profile.name
+  }
+
+  network_interfaces {
+    delete_on_termination       = true
+    associate_public_ip_address = false
+    security_groups             = [ data.aws_security_group.typical-host.id, aws_security_group.customer_portal.id ]
+  }
+
+  block_device_mappings {
+    device_name = "/dev/sda1"
+
+    ebs {
+      volume_type = "gp3"
+      volume_size = "100"
+      delete_on_termination = true
+      encrypted = true
+      kms_key_id = data.aws_kms_key.ebs-key.arn
+    }
+  }
+  block_device_mappings {
+    # swap
+    device_name = "/dev/xvdm"
+
+    ebs {
+      volume_type = "gp3"
+      volume_size = "8"
+      delete_on_termination = true
+      encrypted = true
+      kms_key_id = data.aws_kms_key.ebs-key.arn
+      # Snapshot IDs need to be grabbed from the ami, or it will replace every time. It's ugly.
+      # This may prompt replacement when the AMI is updated.
+      # See:
+      #   https://github.com/hashicorp/terraform/issues/19958
+      #   https://github.com/terraform-providers/terraform-provider-aws/issues/13118
+      #snapshot_id = local.block_device_mappings[local.ami_selection]["/dev/xvdm"].ebs.snapshot_id
+    }
+  }
+  block_device_mappings {
+    # /home
+    device_name = "/dev/xvdn"
+
+    ebs {
+      volume_type = "gp3"
+      volume_size = "4"
+      delete_on_termination = true
+      encrypted = true
+      kms_key_id = data.aws_kms_key.ebs-key.arn
+      #snapshot_id = local.block_device_mappings[local.ami_selection]["/dev/xvdn"].ebs.snapshot_id
+    }
+  }
+  block_device_mappings {
+    # /var
+    device_name = "/dev/xvdo"
+
+    ebs {
+      volume_type = "gp3"
+      volume_size = "15"
+      delete_on_termination = true
+      encrypted = true
+      kms_key_id = data.aws_kms_key.ebs-key.arn
+      #snapshot_id = local.block_device_mappings[local.ami_selection]["/dev/xvdo"].ebs.snapshot_id
+    }
+  }
+  block_device_mappings {
+    # /var/tmp
+    device_name = "/dev/xvdp"
+
+    ebs {
+      volume_type = "gp3"
+      volume_size = "4"
+      delete_on_termination = true
+      encrypted = true
+      kms_key_id = data.aws_kms_key.ebs-key.arn
+      #snapshot_id = local.block_device_mappings[local.ami_selection]["/dev/xvdp"].ebs.snapshot_id
+    }
+  }
+  block_device_mappings {
+    # /var/log
+    device_name = "/dev/xvdq"
+
+    ebs {
+      volume_type = "gp3"
+      volume_size = "8"
+      delete_on_termination = true
+      encrypted = true
+      kms_key_id = data.aws_kms_key.ebs-key.arn
+      #snapshot_id = local.block_device_mappings[local.ami_selection]["/dev/xvdq"].ebs.snapshot_id
+    }
+  }
+  block_device_mappings {
+    # /var/log/audit
+    device_name = "/dev/xvdr"
+
+    ebs {
+      volume_type = "gp3"
+      volume_size = "8"
+      delete_on_termination = true
+      encrypted = true
+      kms_key_id = data.aws_kms_key.ebs-key.arn
+      #snapshot_id = local.block_device_mappings[local.ami_selection]["/dev/xvdr"].ebs.snapshot_id
+    }
+  }
+  block_device_mappings {
+    # /tmp
+    device_name = "/dev/xvds"
+
+    ebs {
+      volume_type = "gp3"
+      volume_size = "4"
+      delete_on_termination = true
+      encrypted = true
+      kms_key_id = data.aws_kms_key.ebs-key.arn
+      #snapshot_id = local.block_device_mappings[local.ami_selection]["/dev/xvds"].ebs.snapshot_id
+    }
+  }
+
+  tag_specifications {
+    resource_type = "instance"
+    tags = merge(var.tags, { "Name": var.instance_name }) # This may have no effect?
+  }
+
+  tag_specifications {
+    resource_type = "volume"
+    tags = merge(var.tags, { "Name": var.instance_name }) # This may have no effect
+  }
+
+  lifecycle {
+    create_before_destroy = true
+  }
+}
+
+resource "aws_autoscaling_group" "customer_portal" {
+  name = "customer-portal-asg"
+  launch_template {
+    id = aws_launch_template.customer_portal.id
+    version = "$Latest"
+  }
+  vpc_zone_identifier = var.private_subnets
   min_size                  = 1
   max_size                  = 2
   desired_capacity          = 2
   wait_for_capacity_timeout = 0
-  tags_as_map = merge(var.standard_tags, var.tags)
+  health_check_type         = "EC2"
+  tag {
+    key = "Name"
+    value = var.instance_name
+    propagate_at_launch = true
+    }
+  
+  # Must ignore changes to attachments, or tf will flip flop
+  lifecycle {
+    ignore_changes = [ load_balancers, target_group_arns ]
+  }
 }