xdr-terraform-modules/base/sensu-configuration/checks.tf

locals {
  splunk_hot = var.environment == "test" ? "10000" : "50000"
  #interconnect-0 = var.environment == "test" ? ["169.254.230.197", "169.254.142.233", "169.254.221.229", "169.254.145.141"] : ["169.254.152.217", "169.254.88.105", "169.254.253.45", "169.254.91.129"]
  #interconnect-1 = var.environment == "test" ? ["169.254.186.189", "169.254.119.73", "169.254.20.161", "169.254.128.189"] : ["169.254.247.157", "169.254.246.157", "169.254.22.21", "169.254.38.13"]
  dns_checks_all = var.environment == "test" ? {
    "dnstest.accenturefederalcyber.com": "203.0.113.1",
    "dnstest.xdrtest.accenturefederalcyber.com": "203.0.113.1",
  } : { 
    "dnstest.accenturefederalcyber.com": "203.0.113.1",
    "dnstest.xdr.accenturefederalcyber.com": "203.0.113.1",
  }
  dns_checks_private = var.environment == "test" ? {
    "dnstest.pvt.xdrtest.accenturefederalcyber.com": "10.10.10.10",
    "10.10.10.10": "dnstest.pvt.xdrtest.accenturefederalcyber.com.",
  } : {
    "dnstest.pvt.xdr.accenturefederalcyber.com": "10.10.10.10",
    "10.10.10.10": "dnstest.pvt.xdr.accenturefederalcyber.com.",
  }

  # To ensure this list is complete:
  #   salt \* grains.get partitions | grep "^    /" | sort | uniq | grep -v snap
  disk_thresholds = {
    "/root": "20%", # This is the / partition
    "/boot": "20%",
    "/home": "20%",
    "/opt": "20%",
    "/tmp": "20%",
    "/var": "20%",
    "/var/log": "20%"
    "/var/log/audit": "20%",
    "/var/tmp": "20%",
    "/var/www/html": "20%",
    # Indexers:
    "/opt/splunk": 5500,
    "/opt/splunkdata/hot": local.splunk_hot,
    # Syslog Servers
    "/mnt/resource": "20%",
    "/opt/syslog-ng": 7000,
    # Docker Hosts
    "/opt/docker/containers": "20%",
    "/opt/docker/overlay": "20%",
    "/opt/docker/overlay2": "20%",
    # VMRay
    "/opt/vmray": "20%"
  }
}

resource "sensu_check" "check_dns_all" {
  for_each       = local.dns_checks_all
  name           = "check_dns_${ each.key }"
  command        = "check_dns --hostname=${ each.key } --expected-address=${ each.value } --warning=${ var.sensu_checks["dns"]["warning"] }  --critical=${ var.sensu_checks["dns"]["critical"] }"
  namespace      = "default"
  subscriptions  = [ "linux", ]
  handlers       = [ "victorops", "logfile", ]
  interval       = 60
  publish        = "true"
  runtime_assets = [ "sensu-plugins-monitoring", ]
}

resource "sensu_check" "check_dns_private" {
  for_each       = local.dns_checks_private
  name           = "check_dns_${ each.key }"
  command        = "check_dns --hostname=${ each.key } --expected-address=${ each.value } --warning=${ var.sensu_checks["dns"]["warning"] }  --critical=${ var.sensu_checks["dns"]["critical"] }"
  namespace      = "default"
  subscriptions  = [ "private_dns_client", ]
  handlers       = [ "victorops", "logfile", ]
  interval       = 60
  publish        = "true"
  runtime_assets = [ "sensu-plugins-monitoring", ]
}

# Systems that can't enumerate their mount points get
# a single 'check all' disk space option
resource "sensu_check" "check_disks_all" {
  name           = "check_disk_all"
  # Checks all partitions except /snap/*
  command        = "check_disk -c 20% -A --ignore-eregi-path '/snap/' --ignore-eregi-path '/opt/docker/.*' --ignore-eregi-path '/(sys|proc)/'"
  namespace      = "default"
  subscriptions  = [ "check_disk_all" ]
  handlers       = [ "victorops", "logfile", ]
  interval       = 300
  publish        = "true"
  runtime_assets = [ "sensu-plugins-monitoring", ]
}

# Systems that enumerate their mount points subscribe to
# particular checks with thresholds defined above.
resource "sensu_check" "check_disks" {
  for_each       = local.disk_thresholds
  name           = "check_disk${ replace(each.key, "/", "_") }"
  command        = "check_disk -c ${ each.value } -p ${ replace(each.key, "root", "/") }"
  namespace      = "default"
  subscriptions  = [ "check_disk${ replace(each.key, "/", "_") }" ]
  handlers       = [ "victorops", "logfile", ]
  interval       = 300
  publish        = "true"
  runtime_assets = [ "sensu-plugins-monitoring", ]
}

#resource "sensu_check" "check_ping_interconnect-0" {
#  for_each       = toset(local.interconnect-0)
#  name           = "ping_interconnect-0-${index(local.interconnect-0, each.value) +1}"
#  command        = "check_ping -H ${each.value} -w 500,80% -c 500,80% -4"
#  namespace      = "default"
#  subscriptions  = [ "interconnect-0", ]
#  handlers       = [ "victorops", "logfile", ]
#  interval       = 60
#  publish        = "true"
#  runtime_assets = [ "sensu-plugins-monitoring", ]
#}
#
#resource "sensu_check" "check_ping_interconnect-1" {
#  for_each       = toset(local.interconnect-1)
#  name           = "ping_interconnect-1-${index(local.interconnect-1, each.value) +1}"
#  command        = "check_ping -H ${each.value} -w 500,80% -c 500,80% -4"
#  namespace      = "default"
#  subscriptions  = [ "interconnect-1", ]
#  handlers       = [ "victorops", "logfile", ]
#  interval       = 60
#  publish        = "true"
#  runtime_assets = [ "sensu-plugins-monitoring", ]
#}

resource "sensu_check" "check_phantom_ports" {
  name           = "phantom_ports"
  command        = "check-ports.rb -h 0.0.0.0 -p 443"
  namespace      = "default"
  subscriptions  = [ "phantom_ports", ]
  handlers       = [ "victorops", "logfile", ]
  interval       = 60
  publish        = "true"
  runtime_assets = [ "sensu-plugins-network-checks", "sensu-ruby-runtime", ]
}

resource "sensu_check" "check_portal_http" {
  name           = "portal_http"
  command        = "check-http.rb -u https://portal.${var.dns_info["public"]["zone"]}/api/health/ --response-code 200"
  namespace      = "default"
  subscriptions  = [ "portal", ]
  handlers       = [ "victorops", "logfile", ]
  interval       = 60
  publish        = "true"
  runtime_assets = [ "sensu-plugins-http", "sensu-ruby-runtime", ]
}

resource "sensu_check" "check_salt_master_ports" {
  name           = "salt_master_ports"
  command        = "check-ports.rb -h 0.0.0.0 -p 4505,4506"
  namespace      = "default"
  subscriptions  = [ "salt_master_ports", ]
  handlers       = [ "victorops", "logfile", ]
  interval       = 60
  publish        = "true"
  runtime_assets = [ "sensu-plugins-network-checks", "sensu-ruby-runtime", ]
}

resource "sensu_check" "check_splunk_cm_ports" {
  name           = "splunk_cm_ports"
  command        = "check-ports.rb -h 0.0.0.0 -p 8089"
  namespace      = "default"
  subscriptions  = [ "splunk_cm_ports", ]
  handlers       = [ "victorops", "logfile", ]
  interval       = 60
  publish        = "true"
  runtime_assets = [ "sensu-plugins-network-checks", "sensu-ruby-runtime", ]
}

resource "sensu_check" "check_splunk_ds_ports" {
  name           = "splunk_ds_ports"
  command        = "check-ports.rb -h 0.0.0.0 -p 8089"
  namespace      = "default"
  subscriptions  = [ "splunk_ds_ports", ]
  handlers       = [ "victorops", "logfile", ]
  interval       = 60
  publish        = "true"
  runtime_assets = [ "sensu-plugins-network-checks", "sensu-ruby-runtime", ]
}

resource "sensu_check" "check_splunk_hf_ports" {
  name           = "splunk_hf_ports"
  command        = "check-ports.rb -h 0.0.0.0 -p 8089,8000"
  namespace      = "default"
  subscriptions  = [ "splunk_hf_ports", ]
  handlers       = [ "victorops", "logfile", ]
  interval       = 60
  publish        = "true"
  runtime_assets = [ "sensu-plugins-network-checks", "sensu-ruby-runtime", ]
}

resource "sensu_check" "check_splunk_indexer_ports_moose" {
  name           = "splunk_indexer_ports_moose"
  command        = "check-ports.rb -h 0.0.0.0 -p 8089,9998,9887,8088"
  namespace      = "default"
  subscriptions  = [ "splunk_indexer_ports_moose", ]
  handlers       = [ "victorops", "logfile", ]
  interval       = 60
  publish        = "true"
  runtime_assets = [ "sensu-plugins-network-checks", "sensu-ruby-runtime", ]
}

resource "sensu_check" "check_splunk_indexer_ports" {
  name           = "splunk_indexer_ports"
  command        = "check-ports.rb -h 0.0.0.0 -p 8089,9998,9887"
  namespace      = "default"
  subscriptions  = [ "splunk_indexer_ports", ]
  handlers       = [ "victorops", "logfile", ]
  interval       = 60
  publish        = "true"
  runtime_assets = [ "sensu-plugins-network-checks", "sensu-ruby-runtime", ]
}

resource "sensu_check" "check_splunk_sh_ports" {
  name           = "splunk_sh_ports"
  command        = "check-ports.rb -h 0.0.0.0 -p 8089,8000"
  namespace      = "default"
  subscriptions  = [ "splunk_sh_ports", ]
  handlers       = [ "victorops", "logfile", ]
  interval       = 60
  publish        = "true"
  runtime_assets = [ "sensu-plugins-network-checks", "sensu-ruby-runtime", ]
}

resource "sensu_check" "check_splunk_uf_ports" {
  name           = "splunk_uf_ports"
  command        = "check-ports.rb -h 0.0.0.0 -p 8089"
  namespace      = "default"
  subscriptions  = [ "splunk", ]
  handlers       = [ "victorops", "logfile", ]
  interval       = 60
  publish        = "true"
  runtime_assets = [ "sensu-plugins-network-checks", "sensu-ruby-runtime", ]
}

resource "sensu_check" "check_syslog-ng_service" {
  name           = "syslog-ng_service"
  command        = "check-process.rb -p syslog-ng"
  namespace      = "default"
  subscriptions  = [ "syslog-ng_service", ]
  handlers       = [ "victorops", "logfile", ]
  interval       = 60
  publish        = "true"
  runtime_assets = [ "sensu-plugins-process-checks", "sensu-ruby-runtime", ]
}

resource "sensu_check" "check_salt-minion_service" {
  name           = "salt-minion_service"
  command        = "check-process.rb -p salt-minion --critical-under 1 --critical-over 5"
  namespace      = "default"
  subscriptions  = [ "salt-minion_service", ]
  handlers       = [ "victorops", "logfile", ]
  interval       = 60
  publish        = "true"
  runtime_assets = [ "sensu-plugins-process-checks", "sensu-ruby-runtime", ]
}

# Don't detect the Splunk Universal Forwarder running as root. 
# Only look for Splunk Enterprise running as splunk user.  
resource "sensu_check" "check_splunk_service" {
  name           = "splunk_service"
  command        = "check-process.rb -p 'splunkd -p' --user splunk -C 2"
  namespace      = "default"
  subscriptions  = [ "splunk_service", ]
  handlers       = [ "victorops", "logfile", ]
  interval       = 60
  publish        = "true"
  runtime_assets = [ "sensu-plugins-process-checks", "sensu-ruby-runtime", ]
}