Merge pull request #117 from mdr-engineering/feature/ftd_MSOCI-1445_FixHECForMooseMigration

Adds Private ALB for Moose Internal Use

base/splunk_servers/indexer_cluster/cloud-init/cloud-init.tpl

@@ -87,7 +87,12 @@ runcmd:
  # Recording our initial values is useful for troubleshooting
  - /bin/salt-call pillar.items > /root/pillars.initial_highstate.yml
  - /bin/salt-call grains.items > /root/grains.initial_highstate.yml
+ - "/bin/echo MARKER: START FIRST HIGHSTATE"
  - /bin/salt-call state.highstate
+ - "/bin/echo MARKER: END FIRST HIGHSTATE"
+ - "/bin/echo MARKER: START SECOND HIGHSTATE"
+ - /bin/salt-call state.highstate
+ - "/bin/echo MARKER: END SECOND HIGHSTATE"
 
 # Either final message or power state, but probably not both
 final_message: "The system is up after $UPTIME seconds"

base/splunk_servers/indexer_cluster/elb-private-for-moose-only.tf

@@ -0,0 +1,117 @@
+#------------------------------------------------------------------------------
+# An external ALB for the indexers for HEC
+#------------------------------------------------------------------------------
+
+#########################
+# DNS Entry
+module "private_dns_record_hec_pvt" {
+  source = "../../../submodules/dns/private_CNAME_record"
+  enabled = local.is_moose ? true : false
+
+  name = "${var.prefix}-hec"
+  target_dns_names = local.is_moose ? [ aws_lb.hec_pvt[0].dns_name ] : [ "na" ]
+  dns_info = var.dns_info
+
+  providers = {
+    aws.c2 = aws.c2
+  }
+}
+
+#########################
+# Certificate - NOTE: Public certificate for a private HEC
+resource "aws_acm_certificate" "hec_pvt_cert" {
+  count = local.is_moose ? 1 : 0
+
+  domain_name       = "${var.prefix}-hec.${var.dns_info["private"]["zone"]}"
+  validation_method = "DNS"
+
+  tags = merge(var.standard_tags, var.tags)
+}
+
+resource "aws_acm_certificate_validation" "hec_pvt_cert_validation" {
+  certificate_arn         = local.is_moose ? aws_acm_certificate.hec_pvt_cert[0].arn : ""
+  validation_record_fqdns = [for record in aws_route53_record.hec_pvt_cert_validation: record.fqdn] # will be empty if not moose
+}
+
+resource "aws_route53_record" "hec_pvt_cert_validation" {
+  provider = aws.mdr-common-services-commercial
+
+  for_each = ( local.is_moose ? {
+      for dvo in aws_acm_certificate.hec_pvt_cert[0].domain_validation_options: dvo.domain_name => {
+        name   = dvo.resource_record_name
+        record = dvo.resource_record_value
+        type   = dvo.resource_record_type
+      } 
+    } : {} 
+  )# Empty map if not moose
+
+  allow_overwrite = true
+  name            = each.value.name
+  records         = [each.value.record]
+  ttl             = 60
+  type            = each.value.type
+  zone_id         = var.dns_info["public"]["zone_id"]
+}
+
+#########################
+# ELB
+resource "aws_lb" "hec_pvt" {
+  count = local.is_moose ? 1 : 0
+
+  tags               = merge(var.standard_tags, var.tags)
+  name               = "${var.prefix}-hec-private"
+  load_balancer_type = "application"
+  security_groups    = [ aws_security_group.hec_pvt_elb_security_group[0].id ]
+  subnets            = var.private_subnets
+  internal           = true
+}
+
+resource "aws_lb_listener" "hec_pvt_443" {
+  count             = local.is_moose ? 1 : 0
+
+  load_balancer_arn = aws_lb.hec_pvt[0].arn
+  port              = 443
+  protocol          = "HTTPS"
+  ssl_policy        = "ELBSecurityPolicy-TLS-1-2-2017-01"
+  certificate_arn   = aws_acm_certificate.hec_pvt_cert[0].arn
+  default_action {
+    type = "forward"
+    target_group_arn = aws_lb_target_group.hec_pvt_8088[0].arn
+  }
+}
+
+resource "aws_lb_listener" "hec_pvt_8088" {
+  count = local.is_moose ? 1 : 0
+
+  load_balancer_arn = aws_lb.hec_pvt[0].arn
+  port              = 8088
+  protocol          = "HTTPS"
+  ssl_policy        = "ELBSecurityPolicy-TLS-1-2-2017-01"
+  certificate_arn   = aws_acm_certificate.hec_pvt_cert[0].arn
+  default_action {
+    type = "forward"
+    target_group_arn = aws_lb_target_group.hec_pvt_8088[0].arn
+  }
+}
+
+resource "aws_lb_target_group" "hec_pvt_8088" {
+  count = local.is_moose ? 1 : 0
+
+  name         = "${var.prefix}-hec-pvt-targets"
+  port         = 8088
+  protocol     = "HTTPS"
+  target_type  = "instance"
+  vpc_id       = var.vpc_id
+
+  health_check {
+    path     = "/services/collector/health/1.0"
+    protocol = "HTTPS"
+  }
+}
+
+# Attach the instnaces to the ELB
+resource "aws_autoscaling_attachment" "hec_pvt_asg_attachments" {
+  for_each = local.is_moose ? toset([ module.indexer0.asg_name[0], module.indexer1.asg_name[0], module.indexer2.asg_name[0] ]) : []
+  alb_target_group_arn = aws_lb_target_group.hec_pvt_8088[0].arn
+  autoscaling_group_name = each.key
+}

base/splunk_servers/indexer_cluster/security-group-elb-pvt.tf

@@ -0,0 +1,42 @@
+resource "aws_security_group" "hec_pvt_elb_security_group" {
+  count = local.is_moose ? 1 : 0
+  name = "hec_pvt_elb_security_group"
+  description = "Security Group for the private moose HEC ELBs"
+  vpc_id = var.vpc_id
+  tags = merge(var.standard_tags, var.tags, { "Name" = "hec_pvt_elb_security_group" })
+}
+
+## Ingress
+resource "aws_security_group_rule" "hec-pvt-https-in" {
+  count             = local.is_moose ? 1 : 0
+  description       = "HEC port - HTTPS for moose only"
+  type              = "ingress"
+  from_port         = 443
+  to_port           = 443
+  protocol          = "tcp"
+  cidr_blocks       = [ "10.0.0.0/8" ]
+  security_group_id = aws_security_group.hec_pvt_elb_security_group[0].id
+}
+
+resource "aws_security_group_rule" "hec-pvt-in" {
+  count             = local.is_moose ? 1 : 0
+  description       = "HEC port in"
+  type              = "ingress"
+  from_port         = 8088
+  to_port           = 8088
+  protocol          = "tcp"
+  cidr_blocks       = [ "10.0.0.0/8" ]
+  security_group_id = aws_security_group.hec_pvt_elb_security_group[0].id
+}
+
+## Egress
+resource "aws_security_group_rule" "hec-pvt-out" {
+  count             = local.is_moose ? 1 : 0
+  description       = "HEC to the indexers"
+  type              = "egress"
+  from_port         = 8088
+  to_port           = 8088
+  protocol          = "tcp"
+  cidr_blocks       = local.splunk_vpc_cidrs
+  security_group_id = aws_security_group.hec_pvt_elb_security_group[0].id
+}

submodules/dns/private_CNAME_record/main.tf

@@ -0,0 +1,10 @@
+resource "aws_route53_record" "dns" {
+  count = var.enabled ? 1 : 0
+
+  name = var.name
+  zone_id = var.dns_info["private"]["zone_id"]
+  type = "CNAME"
+  ttl  = 300
+  records = var.target_dns_names
+  provider = aws.c2
+}

submodules/dns/private_CNAME_record/outputs.tf

@@ -0,0 +1,4 @@
+output "forward" { 
+  # Parenthesis required to resolve ambiguity
+  value = { for entry in aws_route53_record.dns: entry.fqdn => entry.records }
+}

submodules/dns/private_CNAME_record/provider.tf

@@ -0,0 +1,3 @@
+provider "aws" {
+  alias = "c2"
+}

submodules/dns/private_CNAME_record/vars.tf

@@ -0,0 +1,15 @@
+variable "enabled" { 
+  description = "Set to false to do nothing"
+  type = bool 
+  default = true
+}
+
+variable "reverse_enabled" { 
+  description = "Set to false to skip creation of reverse DNS. This can be useful when first creating resources when the private IP cannot be determined. 'enabled' must be true as well."
+  type = bool 
+  default = true
+}
+
+variable "name" { type = string }
+variable "target_dns_names" { type = list(string) }
+variable "dns_info" { type = map }

submodules/splunk/splunk_indexer_asg/main.tf

@@ -6,18 +6,20 @@ data "aws_kms_key" "ebs-key" {
   key_id = "alias/ebs_root_encrypt_decrypt"
 }
 
-resource "aws_placement_group" "cluster-placement" {
-  name     = "splunk-indexer-cluster-placement-${var.asg_number}"
-  # Cluster placement is an interesting question for us.
-  # Since we're multisite, we're going to make each site use a 'cluster' strategy to keep indexers
-  # close together.
-  #
-  # 'spread' would be more appropriate if we were worried about individual site
-  # failures.
-  #
-  # And it doesn't really matter atm since all of ours are count=1
-  strategy = "cluster"
-}
+# Placement groups are a good idea if we get bigger. This code works, but disabling for now because
+# t3* instance types don't support them.
+#resource "aws_placement_group" "cluster-placement" {
+#  name     = "splunk-indexer-cluster-placement-${var.asg_number}"
+#  # Cluster placement is an interesting question for us.
+#  # Since we're multisite, we're going to make each site use a 'cluster' strategy to keep indexers
+#  # close together.
+#  #
+#  # 'spread' would be more appropriate if we were worried about individual site
+#  # failures.
+#  #
+#  # And it doesn't really matter atm since all of ours are count=1
+#  strategy = "cluster"
+#}
 
 #better solutions are to upgrade to TF .12 or maybe this...
 #https://github.com/mavin/terraform-aws-tags-to-asg-tags/blob/master/vars.tf
@@ -180,7 +182,9 @@ resource "aws_autoscaling_group" "splunk_indexer_asg" {
         id      = aws_launch_template.splunk_indexer.id
         version = "$Latest"
     }
-    placement_group      = aws_placement_group.cluster-placement.id
+    # Placement groups are a good idea if we get bigger. This code works, but disabling for now because
+    # t3* instance types don't support them.
+    #placement_group      = aws_placement_group.cluster-placement.id
     vpc_zone_identifier  = var.vpc_zone_identifier
     min_size             = var.min_size
     max_size             = var.max_size