Merge pull request #286 from mdr-engineering/feature/ftd_na_DiskCheckCompleteness

Disk Checks are now per-partition and subscribed to by the entities

base/sensu-configuration/checks.tf

@@ -17,25 +17,29 @@ locals {
     "10.10.10.10": "dnstest.pvt.xdr.accenturefederalcyber.com.",
   }
 
-  standard_paths = {
-    "root": "1000",
-    "/tmp": "20%",
+  # To ensure this list is complete:
+  #   salt \* grains.get partitions | grep "^    /" | sort | uniq | grep -v snap
+  disk_thresholds = {
+    "/root": "1000", # This is the / partition
+    "/boot": "20%",
     "/home": "20%",
+    "/opt": "20%",
+    "/tmp": "20%",
     "/var": "20%",
-    "/var/tmp": "20%",
     "/var/log": "20%"
     "/var/log/audit": "20%",
-    "/opt": "20%",
-    "/boot": "20%",
-  }
-
-  indexer_paths = {
+    "/var/tmp": "20%",
+    "/var/www/html": "20%",
+    # Indexers:
     "/opt/splunk": 5000,
     "/opt/splunkdata/hot": local.splunk_hot,
-  }
-
-  syslog_paths = {
-    "/opt/syslog-ng": 7000
+    # Syslog Servers
+    "/mnt/resource": "20%",
+    "/opt/syslog-ng": 7000,
+    # Docker Hosts
+    "/opt/docker/containers": "20%",
+    "/opt/docker/overlay": "20%",
+    "/opt/docker/overlay2": "20%"
   }
 }
 
@@ -63,36 +67,28 @@ resource "sensu_check" "check_dns_private" {
   runtime_assets = [ "sensu-plugins-monitoring", ]
 }
 
-resource "sensu_check" "check_disk_standard_paths" {
-  for_each       = local.standard_paths
-  name           = "check_disk_base_${ replace(each.key, "/", "_") }"
-  command        = "check_disk -c ${ each.value } -p ${ replace(each.key, "root", "/") }"
-  namespace      = "default"
-  subscriptions  = [ "linux", ]
-  handlers       = [ "victorops", "logfile", ]
-  cron           = "* * * * *"
-  publish        = "true"
-  runtime_assets = [ "sensu-plugins-monitoring", ]
-}
-
-resource "sensu_check" "check_disk_indexer_paths" {
-  for_each       = local.indexer_paths
-  name           = "check_disk_indexer_${ replace(each.key, "/", "_") }"
-  command        = "check_disk -c ${ each.value } -p ${ replace(each.key, "root", "/") }"
+# Systems that can't enumerate their mount points get
+# a single 'check all' disk space option
+resource "sensu_check" "check_disks_all" {
+  name           = "check_disk_all"
+  # Checks all partitions except /snap/*
+  command        = "check_disk -c 20% -A --ignore-eregi-path '/snap/' --ignore-eregi-path '/opt/docker/.*' --ignore-eregi-path '/(sys|proc)/'"
   namespace      = "default"
-  subscriptions  = [ "check_disk_indexer", ]
+  subscriptions  = [ "check_disk_all" ]
   handlers       = [ "victorops", "logfile", ]
   cron           = "* * * * *"
   publish        = "true"
   runtime_assets = [ "sensu-plugins-monitoring", ]
 }
 
-resource "sensu_check" "check_disk_syslog_paths" {
-  for_each       = local.syslog_paths
-  name           = "check_disk_syslog_${ replace(each.key, "/", "_") }"
+# Systems that enumerate their mount points subscribe to
+# particular checks with thresholds defined above.
+resource "sensu_check" "check_disks" {
+  for_each       = local.disk_thresholds
+  name           = "check_disk${ replace(each.key, "/", "_") }"
   command        = "check_disk -c ${ each.value } -p ${ replace(each.key, "root", "/") }"
   namespace      = "default"
-  subscriptions  = [ "check_disk_syslog", ]
+  subscriptions  = [ "check_disk${ replace(each.key, "/", "_") }" ]
   handlers       = [ "victorops", "logfile", ]
   cron           = "* * * * *"
   publish        = "true"