MSOCI-1853 Sensu changes for Threatquotient

Tag v3.2.12

* Adds a daily SSL certificate check with direct-to-email alert (no
  victorops)
* Checking for TQ health API on :443
* Removes victorops alerting for anything in test

base/sensu-configuration/assets.tf

@@ -153,3 +153,46 @@ resource "sensu_asset" "asset_victorops" {
     ]
   }
 }
+
+#https://bonsai.sensu.io/assets/sensu-plugins/sensu-plugins-ssl
+resource "sensu_asset" "asset_ssl" {
+  name = "sensu-plugins-ssl"
+
+  build {
+    sha512 = "27667460a0e21170264bd9482bd524bc7f97705452eaac207fcf374f814935cf98442ef6fcd52e354ca8917ca36f538ebe839875fdee16c58ecf9a1dc5f93795"
+    url = "https://assets.bonsai.sensu.io/2e3663222d4558e3aae7df6e56b229cd89b3e4ae/sensu-plugins-ssl_3.0.2_centos7_linux_amd64.tar.gz"
+    filters = [
+      "entity.system.os=='linux'",
+      "entity.system.arch=='amd64'",
+      "entity.system.platform_family=='rhel'",
+      "entity.system.platform_version.split('.')[0] == '7'",
+    ]
+  }
+
+  build {
+    sha512 = "fe5cfc1261a863982cae42154270feaffd2326222b93daf05276907560f83c5e0eb63c6f175359364d981fcaeff60c441da3787930e710b825d3951b92f7a5d5"
+    url = "https://assets.bonsai.sensu.io/2e3663222d4558e3aae7df6e56b229cd89b3e4ae/sensu-plugins-ssl_3.0.2_debian_linux_amd64.tar.gz"
+    filters = [
+      "entity.system.os=='linux'",
+      "entity.system.arch=='amd64'",
+      "entity.system.platform_family=='debian'"
+    ]
+  }
+}
+
+
+# https://bonsai.sensu.io/assets/sensu/sensu-email-handler
+# I guess this one is not RHEL vs debian specific?
+#
+resource "sensu_asset" "asset_email_handler" {
+  name = "sensu/sensu-email-handler"
+
+  build {
+    sha512 = "81ffd8095c1a6b489b6eca2eae76b84f9943c59877c28e1191c0acc89055bf58d53b4fba3356ddac6572d5fe5e6a62ff018f15b7cd3ecbc37263d3fdb5a0660d"
+    url = "https://assets.bonsai.sensu.io/034de7872a4c9b59a5c575e80e62f0473365f796/sensu-email-handler_1.2.2_linux_amd64.tar.gz"
+    filters = [
+      "entity.system.os=='linux'",
+      "entity.system.arch=='amd64'",
+    ]
+  }
+}

base/sensu-configuration/checks-threatq.tf

@@ -0,0 +1,21 @@
+resource "sensu_check" "check_cert_expire_443" {
+  name           = "check_cert_expire_443"
+  command        = "check-ssl-cert.rb -h 127.0.0.1 -p 443 -c 7 -w 30"
+  namespace      = "default"
+  subscriptions  = [ "threatq", ]
+  handlers       = [ "email-xdr-eng" ]
+  cron           = "00 16 * * *"
+  publish        = "true"
+  runtime_assets = [ "sensu-plugins-ssl", "sensu-ruby-runtime" ]
+}
+
+resource "sensu_check" "check_threatq_health" {
+  name           = "check_threatq_health"
+  command        = "check-http.rb -k -u https://localhost/health/ -q OK"
+  namespace      = "default"
+  subscriptions  = [ "threatq", ]
+  handlers       = local.default_handlers
+  interval       = 60
+  publish        = "true"
+  runtime_assets = [ "sensu-plugins-http", "sensu-ruby-runtime" ]
+}

base/sensu-configuration/checks.tf

@@ -17,6 +17,8 @@ locals {
     "10.10.10.10": "dnstest.pvt.xdr.accenturefederalcyber.com.",
   }
 
+  default_handlers = var.environment == "test" ? ["logfile"] : [ "victorops", "logfile" ]
+
   # To ensure this list is complete:
   #   salt \* grains.get partitions | grep "^    /" | sort | uniq | grep -v snap
   disk_thresholds = {
@@ -51,7 +53,7 @@ resource "sensu_check" "check_dns_all" {
   command        = "check_dns --hostname=${ each.key } --expected-address=${ each.value } --warning=${ var.sensu_checks["dns"]["warning"] }  --critical=${ var.sensu_checks["dns"]["critical"] }"
   namespace      = "default"
   subscriptions  = [ "linux", ]
-  handlers       = [ "victorops", "logfile", ]
+  handlers       = local.default_handlers
   interval       = 60
   publish        = "true"
   runtime_assets = [ "sensu-plugins-monitoring", ]
@@ -63,7 +65,7 @@ resource "sensu_check" "check_dns_private" {
   command        = "check_dns --hostname=${ each.key } --expected-address=${ each.value } --warning=${ var.sensu_checks["dns"]["warning"] }  --critical=${ var.sensu_checks["dns"]["critical"] }"
   namespace      = "default"
   subscriptions  = [ "private_dns_client", ]
-  handlers       = [ "victorops", "logfile", ]
+  handlers       = local.default_handlers
   interval       = 60
   publish        = "true"
   runtime_assets = [ "sensu-plugins-monitoring", ]
@@ -77,7 +79,7 @@ resource "sensu_check" "check_disks_all" {
   command        = "check_disk -c 20% -A --ignore-eregi-path '/snap/' --ignore-eregi-path '/opt/docker/.*' --ignore-eregi-path '/(sys|proc)/'"
   namespace      = "default"
   subscriptions  = [ "check_disk_all" ]
-  handlers       = [ "victorops", "logfile", ]
+  handlers       = local.default_handlers
   interval       = 300
   publish        = "true"
   runtime_assets = [ "sensu-plugins-monitoring", ]
@@ -91,7 +93,7 @@ resource "sensu_check" "check_disks" {
   command        = "check_disk -c ${ each.value } -p ${ replace(each.key, "root", "/") }"
   namespace      = "default"
   subscriptions  = [ "check_disk${ replace(each.key, "/", "_") }" ]
-  handlers       = [ "victorops", "logfile", ]
+  handlers       = local.default_handlers
   interval       = 300
   publish        = "true"
   runtime_assets = [ "sensu-plugins-monitoring", ]
@@ -103,7 +105,7 @@ resource "sensu_check" "check_disks" {
 #  command        = "check_ping -H ${each.value} -w 500,80% -c 500,80% -4"
 #  namespace      = "default"
 #  subscriptions  = [ "interconnect-0", ]
-#  handlers       = [ "victorops", "logfile", ]
+#  handlers       = local.default_handlers
 #  interval       = 60
 #  publish        = "true"
 #  runtime_assets = [ "sensu-plugins-monitoring", ]
@@ -115,7 +117,7 @@ resource "sensu_check" "check_disks" {
 #  command        = "check_ping -H ${each.value} -w 500,80% -c 500,80% -4"
 #  namespace      = "default"
 #  subscriptions  = [ "interconnect-1", ]
-#  handlers       = [ "victorops", "logfile", ]
+#  handlers       = local.default_handlers
 #  interval       = 60
 #  publish        = "true"
 #  runtime_assets = [ "sensu-plugins-monitoring", ]
@@ -126,7 +128,7 @@ resource "sensu_check" "check_phantom_ports" {
   command        = "check-ports.rb -h 0.0.0.0 -p 443"
   namespace      = "default"
   subscriptions  = [ "phantom_ports", ]
-  handlers       = [ "victorops", "logfile", ]
+  handlers       = local.default_handlers
   interval       = 60
   publish        = "true"
   runtime_assets = [ "sensu-plugins-network-checks", "sensu-ruby-runtime", ]
@@ -137,7 +139,7 @@ resource "sensu_check" "check_portal_http" {
   command        = "check-http.rb -u https://portal.${var.dns_info["public"]["zone"]}/api/health/ --response-code 200"
   namespace      = "default"
   subscriptions  = [ "portal", ]
-  handlers       = [ "victorops", "logfile", ]
+  handlers       = local.default_handlers
   interval       = 60
   publish        = "true"
   runtime_assets = [ "sensu-plugins-http", "sensu-ruby-runtime", ]
@@ -148,7 +150,7 @@ resource "sensu_check" "check_salt_master_ports" {
   command        = "check-ports.rb -h 0.0.0.0 -p 4505,4506"
   namespace      = "default"
   subscriptions  = [ "salt_master_ports", ]
-  handlers       = [ "victorops", "logfile", ]
+  handlers       = local.default_handlers
   interval       = 60
   publish        = "true"
   runtime_assets = [ "sensu-plugins-network-checks", "sensu-ruby-runtime", ]
@@ -159,7 +161,7 @@ resource "sensu_check" "check_splunk_cm_ports" {
   command        = "check-ports.rb -h 0.0.0.0 -p 8089"
   namespace      = "default"
   subscriptions  = [ "splunk_cm_ports", ]
-  handlers       = [ "victorops", "logfile", ]
+  handlers       = local.default_handlers
   interval       = 60
   publish        = "true"
   runtime_assets = [ "sensu-plugins-network-checks", "sensu-ruby-runtime", ]
@@ -170,7 +172,7 @@ resource "sensu_check" "check_splunk_ds_ports" {
   command        = "check-ports.rb -h 0.0.0.0 -p 8089"
   namespace      = "default"
   subscriptions  = [ "splunk_ds_ports", ]
-  handlers       = [ "victorops", "logfile", ]
+  handlers       = local.default_handlers
   interval       = 60
   publish        = "true"
   runtime_assets = [ "sensu-plugins-network-checks", "sensu-ruby-runtime", ]
@@ -181,7 +183,7 @@ resource "sensu_check" "check_splunk_hf_ports" {
   command        = "check-ports.rb -h 0.0.0.0 -p 8089,8000"
   namespace      = "default"
   subscriptions  = [ "splunk_hf_ports", ]
-  handlers       = [ "victorops", "logfile", ]
+  handlers       = local.default_handlers
   interval       = 60
   publish        = "true"
   runtime_assets = [ "sensu-plugins-network-checks", "sensu-ruby-runtime", ]
@@ -192,7 +194,7 @@ resource "sensu_check" "check_splunk_indexer_ports_moose" {
   command        = "check-ports.rb -h 0.0.0.0 -p 8089,9998,9887,8088"
   namespace      = "default"
   subscriptions  = [ "splunk_indexer_ports_moose", ]
-  handlers       = [ "victorops", "logfile", ]
+  handlers       = local.default_handlers
   interval       = 60
   publish        = "true"
   runtime_assets = [ "sensu-plugins-network-checks", "sensu-ruby-runtime", ]
@@ -203,7 +205,7 @@ resource "sensu_check" "check_splunk_indexer_ports" {
   command        = "check-ports.rb -h 0.0.0.0 -p 8089,9998,9887"
   namespace      = "default"
   subscriptions  = [ "splunk_indexer_ports", ]
-  handlers       = [ "victorops", "logfile", ]
+  handlers       = local.default_handlers
   interval       = 60
   publish        = "true"
   runtime_assets = [ "sensu-plugins-network-checks", "sensu-ruby-runtime", ]
@@ -214,7 +216,7 @@ resource "sensu_check" "check_splunk_sh_ports" {
   command        = "check-ports.rb -h 0.0.0.0 -p 8089,8000"
   namespace      = "default"
   subscriptions  = [ "splunk_sh_ports", ]
-  handlers       = [ "victorops", "logfile", ]
+  handlers       = local.default_handlers
   interval       = 60
   publish        = "true"
   runtime_assets = [ "sensu-plugins-network-checks", "sensu-ruby-runtime", ]
@@ -225,7 +227,7 @@ resource "sensu_check" "check_splunk_uf_ports" {
   command        = "check-ports.rb -h 0.0.0.0 -p 8089"
   namespace      = "default"
   subscriptions  = [ "splunk", ]
-  handlers       = [ "victorops", "logfile", ]
+  handlers       = local.default_handlers
   interval       = 60
   publish        = "true"
   runtime_assets = [ "sensu-plugins-network-checks", "sensu-ruby-runtime", ]
@@ -236,7 +238,7 @@ resource "sensu_check" "check_syslog-ng_service" {
   command        = "check-process.rb -p syslog-ng"
   namespace      = "default"
   subscriptions  = [ "syslog-ng_service", ]
-  handlers       = [ "victorops", "logfile", ]
+  handlers       = local.default_handlers
   interval       = 60
   publish        = "true"
   runtime_assets = [ "sensu-plugins-process-checks", "sensu-ruby-runtime", ]
@@ -247,7 +249,7 @@ resource "sensu_check" "check_salt-minion_service" {
   command        = "check-process.rb -p salt-minion --critical-under 1 --critical-over 5"
   namespace      = "default"
   subscriptions  = [ "salt-minion_service", ]
-  handlers       = [ "victorops", "logfile", ]
+  handlers       = local.default_handlers
   interval       = 60
   publish        = "true"
   runtime_assets = [ "sensu-plugins-process-checks", "sensu-ruby-runtime", ]
@@ -260,7 +262,7 @@ resource "sensu_check" "check_splunk_service" {
   command        = "check-process.rb -p 'splunkd -p' --user splunk -C 2"
   namespace      = "default"
   subscriptions  = [ "splunk_service", ]
-  handlers       = [ "victorops", "logfile", ]
+  handlers       = local.default_handlers
   interval       = 60
   publish        = "true"
   runtime_assets = [ "sensu-plugins-process-checks", "sensu-ruby-runtime", ]

base/sensu-configuration/datasources.tf

@@ -0,0 +1,13 @@
+locals {
+  smtp_secrets = jsondecode(data.aws_secretsmanager_secret_version.smtp_creds.secret_string)
+}
+
+data "aws_secretsmanager_secret" "smtp_creds" {
+  name = "smtp/smtp_creds"
+  provider = aws.c2
+}
+
+data "aws_secretsmanager_secret_version" "smtp_creds" {
+  secret_id = data.aws_secretsmanager_secret.smtp_creds.id
+  provider = aws.c2
+}

base/sensu-configuration/handlers.tf

@@ -3,6 +3,8 @@ locals {
 }
 
 resource "sensu_handler" "handler_keepalive" {
+  # No victorops in test at all pl0x
+  count     = var.environment == "test" ? 0 : 1
   name      = "keepalive"
   type      = "set"
   namespace = "default"
@@ -28,3 +30,14 @@ resource "sensu_handler" "handler_logfile" {
   runtime_assets = [ "sensu-ruby-runtime", ]
   command        = "/usr/local/bin/stdin_to_log.py"
 }
+
+resource "sensu_handler" "handler_email" {
+  name           = "email-xdr-eng"
+  type           = "pipe"
+  namespace      = "default"
+  handlers       = [ ]
+  filters        = [ "is_incident", "not_silenced" ]
+  runtime_assets = [ "sensu/sensu-email-handler", ]
+  command        = "sensu-email-handler -f sensu@${local.smtp_secrets["mail_from_domain"]} -t duane.e.waddle@accenturefederal.com -T /etc/sensu/email-template -s ${local.smtp_secrets["mail_host"]} -u ${local.smtp_secrets["mail_user"]} -p '${local.smtp_secrets["mail_password"]}'"
+  timeout        = 10
+}