Merge pull request #311 from mdr-engineering/feature/dw_MSOCI-1623_more_tq_fixes

Adds backup volume + boothook, bigger /var volume

base/threatquotient/backup_ebs.tf

@@ -0,0 +1,18 @@
+# Make /opt/tqbackup separate from the instance for greater margin of safety
+resource "aws_ebs_volume" "tqbackup" {
+  count = local.instance_count
+  availability_zone = var.azs[count.index % 3]
+  size = var.tqbackup_size
+  type = "gp3"
+  encrypted = true
+  kms_key_id = data.aws_kms_key.ebs-key.arn
+
+  tags = merge(var.standard_tags, var.tags, { Name = "${local.server_name_stem}-${count.index}", Path = "/opt/tqbackup", Device = "/dev/xvdf" })
+}
+
+resource "aws_volume_attachment" "tqbackup" {
+  count = local.instance_count
+  device_name = "/dev/xvdf"
+  volume_id   = aws_ebs_volume.tqbackup[count.index].id
+  instance_id = aws_instance.instance[count.index].id
+}

base/threatquotient/cloud-init/opt_tqbackup.boothook

@@ -0,0 +1,80 @@
+#!/bin/bash
+#
+#
+exec > /dev/console
+exec 2>&1
+
+declare -A EBSMAP
+
+# Build a map of EBS NVMe disks from their AWS-API-name to their NVMe name
+# this makes an associative array (like a python hash) of the
+# sdX/xvdX name you'd set in AWS API to the corresponding nvmeX name
+# Thanks Fred for the awesome id-ctrl stuff I'd never seen before
+#
+# One interesting side effect observed:  the id-ctrl output is different when
+# volumes are attached at boot time (no /dev/) versus attached after the OS
+# is started (includes /dev/)
+function make_nve_ebs_map {
+        for DEVICE in $( lsblk -d -o NAME,MODEL -n | egrep "Elastic Block Store" | awk '{ print $1 }' ); do
+                UNDERLYING=$( nvme id-ctrl --raw-binary /dev/${DEVICE} 2>/dev/null | cut -c 3073-3104 | tr -d ' ' | sed "s#/dev/##" )
+
+                EBSMAP[$UNDERLYING]=$DEVICE
+                UNDERLYING2=$( echo $UNDERLYING | sed "s/sd/xvd/" )
+                EBSMAP[$UNDERLYING2]=$DEVICE
+        done
+}
+
+function do_the_mount
+{
+	VOL_LABEL=$1
+	VOLUME=$2
+	MOUNTPOINT=$3
+
+
+	DONE=0
+	TRIES=0
+	while [[ $DONE -ne 1 ]] && [[ $TRIES -lt 20 ]]; do
+		echo "Looking for $VOLUME to come attached"
+		make_nve_ebs_map
+
+		#echo "------- current nvme/ebs map -------"
+		#for K in "${!EBSMAP[@]}"; do echo $K  = ${EBSMAP[$K]} ; done
+		#echo "------- end current nvme/ebs map -------"
+
+		if [[ -b /dev/$VOLUME ]]; then
+			DEV="/dev/$VOLUME"
+			DONE=1
+		elif [[ -b /dev/${EBSMAP[$VOLUME]} ]]; then
+			DEV="/dev/${EBSMAP[$VOLUME]}"
+			DONE=1
+		else
+			sleep 10
+			TRIES=$(( $TRIES + 1 ))
+		fi
+
+		echo "Volume $VOLUME available at $DEV"
+	done
+
+	if ! [[ -d ${MOUNTPOINT} ]]; then
+		echo "Creating mount directory ${MOUNTPOINT}"
+		mkdir -p ${MOUNTPOINT}
+	fi
+
+	if ! blkid -l -t LABEL=${VOL_LABEL}; then
+		echo "Making filesystem for LABEL=${VOL_LABEL} on ${DEV}"
+		mkfs.xfs -L ${VOL_LABEL} ${DEV}
+	fi
+
+	if ! egrep -q "LABEL=${VOL_LABEL}" /etc/fstab; then
+		echo "Adding LABEL=${VOL_LABEL} to /etc/fstab"
+		echo "LABEL=${VOL_LABEL}       ${MOUNTPOINT}    xfs    noatime,nofail  0 2" >> /etc/fstab
+	fi
+
+	if ! mountpoint ${MOUNTPOINT} >/dev/null 2>&1; then
+		echo "Mounting ${MOUNTPOINT}"
+		mount ${MOUNTPOINT}
+	fi
+
+}
+
+do_the_mount opt_tqbackup xvdf /opt/tqbackup

base/threatquotient/main.tf

@@ -70,7 +70,7 @@ resource "aws_instance" "instance" {
     # /var
     device_name = "/dev/xvdo"
     volume_type = local.ebs_volume_type
-    # volume_size = xx
+    volume_size = 80
     delete_on_termination = true
     encrypted = true
     kms_key_id = data.aws_kms_key.ebs-key.arn
@@ -174,4 +174,11 @@ data "template_cloudinit_config" "cloud_init_config" {
       }
     )
   }
+
+  # mount /dev/xvdf at /opt/tqbackup
+  part {
+    content_type = "text/cloud-boothook"
+    content      = file("${path.module}/cloud-init/opt_tqbackup.boothook")
+  }
+
 }

base/threatquotient/vars.tf

@@ -1,3 +1,8 @@
+variable tqbackup_size {
+  description = "Size of the tqbackup EBS volume"
+  default = 15
+}
+
 variable extra_key_users {
   description = "Extra encryption key users."
   type = list