Merge pull request #440 from mdr-engineering/feature/ftd_MSOCI-2165_VMRayUpdates

Updates VMRay for Ubuntu 20.04 Final Release Compatibility

base/vmray_instances/README.md

@@ -1,3 +1,8 @@
 # vmray instances
 
 Builds and configures the instances required for VMRay
+
+## WAIT AFTER BUILDING
+
+After building, give the system a _long_ time (an hour?) to fully process the cloud-init.
+

base/vmray_instances/alb.tf

@@ -8,6 +8,8 @@ resource "aws_alb" "vmray_internal" {
   subnets            = var.public_subnets
   load_balancer_type = "application"
 
+  drop_invalid_header_fields = true
+
   access_logs {
     bucket  = "xdr-elb-${var.environment}"
     enabled = true

base/vmray_instances/cloud-init/cloud-init.tpl

@@ -75,6 +75,7 @@ write_files:
 
 packages:
  - vim
+ - ubuntu-advantage-tools
 
 package_update: true # Always patch
 
@@ -89,10 +90,13 @@ runcmd:
  - export https_proxy=http://${proxy}:80
  - export no_proxy=localhost,127.0.0.1,169.254.169.254
  - ua auto-attach
- - ua enable --assume-yes usg fips fips-updates
- - /usr/share/ubuntu-scap-security-guides/cis-hardening/Canonical_Ubuntu_20.04_CIS-harden.sh lvl2_server
+ - ua enable --assume-yes usg
+# - ua enable --assume-yes usg fips fips-updates
+# - /usr/share/ubuntu-scap-security-guides/cis-hardening/Canonical_Ubuntu_20.04_CIS-harden.sh lvl2_server
  - apt update 
  - apt upgrade -y
+ - apt install -y usg
+ - usg fix cis_level2_server 
  # VMRay is incompatible with firewalld, but this should be enabled for other ubuntu systems
  #- apt install -y firewalld
  #- /bin/systemctl start firewalld

base/vmray_instances/s3.tf

@@ -40,6 +40,15 @@ resource "aws_s3_bucket_lifecycle_configuration" "s3_lifecyle_storage" {
     expiration {
       days = 90
     }
+
+    noncurrent_version_expiration {
+      noncurrent_days = 93 # If they've expired and not been synced back, be able to rescue them
+    }
+
+    noncurrent_version_transition {
+      noncurrent_days = 30
+      storage_class   = "STANDARD_IA"
+    }
   }
 }
 

base/vmray_instances/security-groups.tf

@@ -12,6 +12,7 @@ resource "aws_security_group" "vmray_server_sg" {
 }
 
 resource "aws_security_group_rule" "vmray_server_http_in_from_workers" {
+  description              = "Allow inbound port 80 for redirect from other vmray servers"
   type                     = "ingress"
   from_port                = 80
   to_port                  = 80
@@ -21,6 +22,7 @@ resource "aws_security_group_rule" "vmray_server_http_in_from_workers" {
 }
 
 resource "aws_security_group_rule" "vmray_server_https_in_from_workers" {
+  description              = "Allow inbound https for interserver communication from other vmray servers"
   type                     = "ingress"
   from_port                = 443
   to_port                  = 443
@@ -30,10 +32,11 @@ resource "aws_security_group_rule" "vmray_server_https_in_from_workers" {
 }
 
 resource "aws_security_group_rule" "vmray_server_https_in" {
-  type      = "ingress"
-  from_port = 443
-  to_port   = 443
-  protocol  = "tcp"
+  description = "Allow https ingress from the VPN"
+  type        = "ingress"
+  from_port   = 443
+  to_port     = 443
+  protocol    = "tcp"
   #cidr_blocks       = var.cidr_map["vpc-access"]
   source_security_group_id = aws_security_group.vmray_alb_internal.id
   security_group_id        = aws_security_group.vmray_server_sg.id
@@ -41,6 +44,7 @@ resource "aws_security_group_rule" "vmray_server_https_in" {
 
 ## VMRay Does DNS Lookups to the Local Network
 resource "aws_security_group_rule" "vmray_server_tcpdns_out" {
+  description       = "Allow DNS lookups to the local DNS server"
   type              = "egress"
   from_port         = 53
   to_port           = 53
@@ -50,6 +54,7 @@ resource "aws_security_group_rule" "vmray_server_tcpdns_out" {
 }
 
 resource "aws_security_group_rule" "vmray_server_udpdns_out" {
+  description       = "Allow DNS lookups to the local DNS server"
   type              = "egress"
   from_port         = 53
   to_port           = 53
@@ -58,8 +63,9 @@ resource "aws_security_group_rule" "vmray_server_udpdns_out" {
   security_group_id = aws_security_group.vmray_server_sg.id
 }
 
-## VMRay Requires Direct Internet Access
+# tfsec:ignore:aws-vpc-no-public-egress-sgr VMRay Requires Direct Internet Access
 resource "aws_security_group_rule" "vmray_server_http_out" {
+  description       = "VMRay requires direct HTTP outbound"
   type              = "egress"
   from_port         = 80
   to_port           = 80
@@ -68,7 +74,9 @@ resource "aws_security_group_rule" "vmray_server_http_out" {
   security_group_id = aws_security_group.vmray_server_sg.id
 }
 
+# tfsec:ignore:aws-vpc-no-public-egress-sgr VMRay Requires Direct Internet Access
 resource "aws_security_group_rule" "vmray_server_https_out" {
+  description       = "VMRay requires direct HTTPS outbound"
   type              = "egress"
   from_port         = 443
   to_port           = 443
@@ -78,6 +86,7 @@ resource "aws_security_group_rule" "vmray_server_https_out" {
 }
 
 resource "aws_security_group_rule" "vmray_server_vnc_to_workers" {
+  description              = "VMRay uses VNC for client machine access."
   type                     = "egress"
   from_port                = 5900
   to_port                  = 5999
@@ -95,6 +104,7 @@ resource "aws_security_group" "vmray_worker_sg" {
 }
 
 resource "aws_security_group_rule" "vmwary_worker_vnc_from_server" {
+  description              = "VMRay uses VNC for client machine access."
   type                     = "ingress"
   from_port                = 5900
   to_port                  = 5999
@@ -104,6 +114,7 @@ resource "aws_security_group_rule" "vmwary_worker_vnc_from_server" {
 }
 
 resource "aws_security_group_rule" "vmwary_worker_vnc_from_access" {
+  description       = "VMRay uses VNC for client machine access."
   type              = "ingress"
   from_port         = 5900
   to_port           = 5999
@@ -112,8 +123,8 @@ resource "aws_security_group_rule" "vmwary_worker_vnc_from_access" {
   security_group_id = aws_security_group.vmray_worker_sg.id
 }
 
-## VMRay Does DNS Lookups to the Local Network
 resource "aws_security_group_rule" "vmray_worker_tcpdns_out" {
+  description       = "VMRay does DNS lookups to an instance in  the local vpc."
   type              = "egress"
   from_port         = 53
   to_port           = 53
@@ -123,6 +134,7 @@ resource "aws_security_group_rule" "vmray_worker_tcpdns_out" {
 }
 
 resource "aws_security_group_rule" "vmray_worker_udpdns_out" {
+  description       = "VMRay does DNS lookups to an instance in  the local vpc."
   type              = "egress"
   from_port         = 53
   to_port           = 53
@@ -131,7 +143,9 @@ resource "aws_security_group_rule" "vmray_worker_udpdns_out" {
   security_group_id = aws_security_group.vmray_worker_sg.id
 }
 
+# tfsec:ignore:aws-vpc-no-public-egress-sgr VMRay Requires Direct Internet Access
 resource "aws_security_group_rule" "vmray_worker_http_out" {
+  description       = "VMRay requires direct HTTP access."
   type              = "egress"
   from_port         = 80
   to_port           = 80
@@ -140,7 +154,9 @@ resource "aws_security_group_rule" "vmray_worker_http_out" {
   security_group_id = aws_security_group.vmray_worker_sg.id
 }
 
+# tfsec:ignore:aws-vpc-no-public-egress-sgr VMRay Requires Direct Internet Access
 resource "aws_security_group_rule" "vmray_worker_https_out" {
+  description       = "VMRay requires direct HTTPS access."
   type              = "egress"
   from_port         = 443
   to_port           = 443
@@ -150,6 +166,7 @@ resource "aws_security_group_rule" "vmray_worker_https_out" {
 }
 
 resource "aws_security_group_rule" "vmray_worker_http_to_server" {
+  description              = "VMRay worker communicates with the server."
   type                     = "egress"
   from_port                = 80
   to_port                  = 80
@@ -159,6 +176,7 @@ resource "aws_security_group_rule" "vmray_worker_http_to_server" {
 }
 
 resource "aws_security_group_rule" "vmray_worker_https_to_server" {
+  description              = "VMRay worker communicates with the server."
   type                     = "egress"
   from_port                = 443
   to_port                  = 443

base/vmray_instances/server.tf

@@ -46,22 +46,23 @@ resource "aws_network_interface" "vmray-server-interface" {
   tags            = merge(var.standard_tags, var.tags, { Name = "vmray-server" })
 }
 
-# Make /opt/vmray separate from the instance for greater margin of safety
-resource "aws_ebs_volume" "server_opt_vmray" {
-  availability_zone = var.azs[0]
-  size              = var.vmray_server_opt_vmray_size
-  type              = "gp3"
-  encrypted         = true
-  kms_key_id        = data.aws_kms_key.ebs-key.arn
-
-  tags = merge(var.standard_tags, var.tags, { Name = "vmray-server" })
-}
-
-resource "aws_volume_attachment" "server_opt_vmray" {
-  device_name = "/dev/xvdf"
-  volume_id   = aws_ebs_volume.server_opt_vmray.id
-  instance_id = aws_instance.vmray-server-instance.id
-}
+## Make /opt/vmray separate from the instance for greater margin of safety
+#resource "aws_ebs_volume" "server_opt_vmray" {
+#  availability_zone = var.azs[0]
+#  size              = var.vmray_server_opt_vmray_size
+#  type              = "gp3"
+#  encrypted         = true
+#  kms_key_id        = data.aws_kms_key.ebs-key.arn
+#
+#  tags = merge(var.standard_tags, var.tags, { Name = "vmray-server" })
+#}
+#
+# Has to be on the same partition as /tmp
+#resource "aws_volume_attachment" "server_opt_vmray" {
+#  device_name = "/dev/xvdf"
+#  volume_id   = aws_ebs_volume.server_opt_vmray.id
+#  instance_id = aws_instance.vmray-server-instance.id
+#}
 
 resource "aws_instance" "vmray-server-instance" {
   tenancy                              = "default"
@@ -79,9 +80,15 @@ resource "aws_instance" "vmray-server-instance" {
   # that could be removed.
   lifecycle { ignore_changes = [ami, key_name, user_data, ebs_block_device] }
 
+  metadata_options {
+    http_endpoint = "enabled"
+    # tfsec:ignore:aws-ec2-enforce-http-token-imds salt requirement?
+    http_tokens = "optional"
+  }
+
   root_block_device {
     volume_type           = "gp3"
-    volume_size           = "60"
+    volume_size           = var.vmray_server_drive_size
     delete_on_termination = true
     encrypted             = true
     kms_key_id            = data.aws_kms_key.ebs-key.arn
@@ -112,26 +119,26 @@ resource "aws_instance" "vmray-server-instance" {
     kms_key_id            = data.aws_kms_key.ebs-key.arn
     snapshot_id           = local.block_device_mappings["/dev/xvdn"].ebs.snapshot_id
   }
-  ebs_block_device {
-    # /var
-    device_name = "/dev/xvdo"
-    # volume_size = xx
-    volume_type           = "gp3"
-    delete_on_termination = true
-    encrypted             = true
-    kms_key_id            = data.aws_kms_key.ebs-key.arn
-    snapshot_id           = local.block_device_mappings["/dev/xvdo"].ebs.snapshot_id
-  }
-  ebs_block_device {
-    # /var/tmp
-    device_name = "/dev/xvdp"
-    # volume_size = xx
-    volume_type           = "gp3"
-    delete_on_termination = true
-    encrypted             = true
-    kms_key_id            = data.aws_kms_key.ebs-key.arn
-    snapshot_id           = local.block_device_mappings["/dev/xvdp"].ebs.snapshot_id
-  }
+  #  ebs_block_device {
+  #    # /var
+  #    device_name = "/dev/xvdo"
+  #    # volume_size = xx
+  #    volume_type           = "gp3"
+  #    delete_on_termination = true
+  #    encrypted             = true
+  #    kms_key_id            = data.aws_kms_key.ebs-key.arn
+  #    snapshot_id           = local.block_device_mappings["/dev/xvdo"].ebs.snapshot_id
+  #  }
+  #  ebs_block_device {
+  #    # /var/tmp
+  #    device_name = "/dev/xvdp"
+  #    # volume_size = xx
+  #    volume_type           = "gp3"
+  #    delete_on_termination = true
+  #    encrypted             = true
+  #    kms_key_id            = data.aws_kms_key.ebs-key.arn
+  #    snapshot_id           = local.block_device_mappings["/dev/xvdp"].ebs.snapshot_id
+  #  }
   ebs_block_device {
     # /var/log
     device_name = "/dev/xvdq"
@@ -152,16 +159,16 @@ resource "aws_instance" "vmray-server-instance" {
     kms_key_id            = data.aws_kms_key.ebs-key.arn
     snapshot_id           = local.block_device_mappings["/dev/xvdr"].ebs.snapshot_id
   }
-  ebs_block_device {
-    # /tmp
-    device_name           = "/dev/xvds"
-    volume_size           = 100 # VMray extracts isos to /tmp
-    volume_type           = "gp3"
-    delete_on_termination = true
-    encrypted             = true
-    kms_key_id            = data.aws_kms_key.ebs-key.arn
-    snapshot_id           = local.block_device_mappings["/dev/xvds"].ebs.snapshot_id
-  }
+  #  ebs_block_device {
+  #    # /tmp
+  #    device_name           = "/dev/xvds"
+  #    volume_size           = 100 # VMray extracts isos to /tmp
+  #    volume_type           = "gp3"
+  #    delete_on_termination = true
+  #    encrypted             = true
+  #    kms_key_id            = data.aws_kms_key.ebs-key.arn
+  #    snapshot_id           = local.block_device_mappings["/dev/xvds"].ebs.snapshot_id
+  #  }
 
   network_interface {
     device_index         = 0
@@ -219,10 +226,10 @@ data "template_cloudinit_config" "cloud-init-vmray-server" {
   }
 
   # mount /dev/xvdf at /opt/vmray
-  part {
-    content_type = "text/cloud-boothook"
-    content      = file("${path.module}/cloud-init/opt_vmray.boothook")
-  }
+  #part {
+  #  content_type = "text/cloud-boothook"
+  #  content      = file("${path.module}/cloud-init/opt_vmray.boothook")
+  #}
 
 }
 

base/vmray_instances/vars.tf

@@ -3,14 +3,14 @@ variable "vmray_worker_instance_count" {
   type        = number
 }
 
-variable "vmray_server_opt_vmray_size" {
-  description = "Size of /opt/vmray directory on the server."
+variable "vmray_server_drive_size" {
+  description = "Size of the main partition on the server."
   type        = number
   default     = 30 # I don't expect images here, so not much space should be required
 }
 
-variable "vmray_worker_opt_vmray_size" {
-  description = "Size of /opt/vmray directory on the worker nodes"
+variable "vmray_worker_drive_size" {
+  description = "Size of the main partition on the worker nodes"
   type        = number
   default     = 300 # Stores images, which are ~ 25GB each
 }

base/vmray_instances/worker.tf

@@ -7,23 +7,23 @@ resource "aws_network_interface" "vmray-worker-interface" {
 }
 
 # Make /opt/vmray separate from the instance for greater margin of safety
-resource "aws_ebs_volume" "worker_opt_vmray" {
-  count             = var.vmray_worker_instance_count
-  availability_zone = var.azs[count.index % 3]
-  size              = var.vmray_worker_opt_vmray_size
-  type              = "gp3"
-  encrypted         = true
-  kms_key_id        = data.aws_kms_key.ebs-key.arn
-
-  tags = merge(var.standard_tags, var.tags, { Name = "vmray-worker-${count.index}" })
-}
-
-resource "aws_volume_attachment" "worker_opt_vmray" {
-  count       = var.vmray_worker_instance_count
-  device_name = "/dev/xvdf"
-  volume_id   = aws_ebs_volume.worker_opt_vmray[count.index].id
-  instance_id = aws_instance.vmray-worker-instance[count.index].id
-}
+#resource "aws_ebs_volume" "worker_opt_vmray" {
+#  count             = var.vmray_worker_instance_count
+#  availability_zone = var.azs[count.index % 3]
+#  size              = var.vmray_worker_opt_vmray_size
+#  type              = "gp3"
+#  encrypted         = true
+#  kms_key_id        = data.aws_kms_key.ebs-key.arn
+#
+#  tags = merge(var.standard_tags, var.tags, { Name = "vmray-worker-${count.index}" })
+#}
+
+#resource "aws_volume_attachment" "worker_opt_vmray" {
+#  count       = var.vmray_worker_instance_count
+#  device_name = "/dev/xvdf"
+#  volume_id   = aws_ebs_volume.worker_opt_vmray[count.index].id
+#  instance_id = aws_instance.vmray-worker-instance[count.index].id
+#}
 
 resource "aws_instance" "vmray-worker-instance" {
   count                                = var.vmray_worker_instance_count
@@ -42,9 +42,15 @@ resource "aws_instance" "vmray-worker-instance" {
   # that could be removed.
   lifecycle { ignore_changes = [ami, key_name, user_data, ebs_block_device] }
 
+  metadata_options {
+    http_endpoint = "enabled"
+    # tfsec:ignore:aws-ec2-enforce-http-token-imds Saltstack doesn't use s3 sources appropriately
+    http_tokens = "optional"
+  }
+
   root_block_device {
     volume_type           = "gp3"
-    volume_size           = "60"
+    volume_size           = var.vmray_worker_drive_size
     delete_on_termination = true
     encrypted             = true
     kms_key_id            = data.aws_kms_key.ebs-key.arn
@@ -86,26 +92,26 @@ resource "aws_instance" "vmray-worker-instance" {
     kms_key_id            = data.aws_kms_key.ebs-key.arn
     snapshot_id           = local.block_device_mappings["/dev/xvdn"].ebs.snapshot_id
   }
-  ebs_block_device {
-    # /var
-    device_name = "/dev/xvdo"
-    # volume_size = xx
-    volume_type           = "gp3"
-    delete_on_termination = true
-    encrypted             = true
-    kms_key_id            = data.aws_kms_key.ebs-key.arn
-    snapshot_id           = local.block_device_mappings["/dev/xvdo"].ebs.snapshot_id
-  }
-  ebs_block_device {
-    # /var/tmp
-    device_name = "/dev/xvdp"
-    # volume_size = xx
-    volume_type           = "gp3"
-    delete_on_termination = true
-    encrypted             = true
-    kms_key_id            = data.aws_kms_key.ebs-key.arn
-    snapshot_id           = local.block_device_mappings["/dev/xvdp"].ebs.snapshot_id
-  }
+  #  ebs_block_device {
+  #    # /var
+  #    device_name = "/dev/xvdo"
+  #    # volume_size = xx
+  #    volume_type           = "gp3"
+  #    delete_on_termination = true
+  #    encrypted             = true
+  #    kms_key_id            = data.aws_kms_key.ebs-key.arn
+  #    snapshot_id           = local.block_device_mappings["/dev/xvdo"].ebs.snapshot_id
+  #  }
+  #  ebs_block_device {
+  #    # /var/tmp
+  #    device_name = "/dev/xvdp"
+  #    # volume_size = xx
+  #    volume_type           = "gp3"
+  #    delete_on_termination = true
+  #    encrypted             = true
+  #    kms_key_id            = data.aws_kms_key.ebs-key.arn
+  #    snapshot_id           = local.block_device_mappings["/dev/xvdp"].ebs.snapshot_id
+  #  }
   ebs_block_device {
     # /var/log
     device_name = "/dev/xvdq"
@@ -126,16 +132,16 @@ resource "aws_instance" "vmray-worker-instance" {
     kms_key_id            = data.aws_kms_key.ebs-key.arn
     snapshot_id           = local.block_device_mappings["/dev/xvdr"].ebs.snapshot_id
   }
-  ebs_block_device {
-    # /tmp
-    device_name           = "/dev/xvds"
-    volume_size           = 100
-    volume_type           = "gp3"
-    delete_on_termination = true
-    encrypted             = true
-    kms_key_id            = data.aws_kms_key.ebs-key.arn
-    snapshot_id           = local.block_device_mappings["/dev/xvds"].ebs.snapshot_id
-  }
+  #  ebs_block_device {
+  #    # /tmp
+  #    device_name           = "/dev/xvds"
+  #    volume_size           = 100
+  #    volume_type           = "gp3"
+  #    delete_on_termination = true
+  #    encrypted             = true
+  #    kms_key_id            = data.aws_kms_key.ebs-key.arn
+  #    snapshot_id           = local.block_device_mappings["/dev/xvds"].ebs.snapshot_id
+  #  }
 
   network_interface {
     device_index         = 0