xdr-terraform-modules/base/sensu-configuration/checks.tf

locals {
  splunk_hot = var.environment == "test" ? "10000" : "50000"
  #interconnect-0 = var.environment == "test" ? ["169.254.230.197", "169.254.142.233", "169.254.221.229", "169.254.145.141"] : ["169.254.152.217", "169.254.88.105", "169.254.253.45", "169.254.91.129"]
  #interconnect-1 = var.environment == "test" ? ["169.254.186.189", "169.254.119.73", "169.254.20.161", "169.254.128.189"] : ["169.254.247.157", "169.254.246.157", "169.254.22.21", "169.254.38.13"]
  dns_checks_all = var.environment == "test" ? {
    "dnstest.accenturefederalcyber.com" : "203.0.113.1",
    "dnstest.xdrtest.accenturefederalcyber.com" : "203.0.113.1",
    } : {
    "dnstest.accenturefederalcyber.com" : "203.0.113.1",
    "dnstest.xdr.accenturefederalcyber.com" : "203.0.113.1",
  }
  dns_checks_private = var.environment == "test" ? {
    "dnstest.pvt.xdrtest.accenturefederalcyber.com" : "10.10.10.10",
    "10.10.10.10" : "dnstest.pvt.xdrtest.accenturefederalcyber.com.",
    } : {
    "dnstest.pvt.xdr.accenturefederalcyber.com" : "10.10.10.10",
    "10.10.10.10" : "dnstest.pvt.xdr.accenturefederalcyber.com.",
  }

  default_handlers = var.environment == "test" ? ["logfile"] : ["victorops", "logfile"]

  # To ensure this list is complete:
  #   salt \* grains.get partitions | grep "^    /" | sort | uniq | grep -v snap
  disk_thresholds = {
    "/root" : "20%", # This is the / partition
    "/boot" : "20%",
    "/home" : "20%",
    "/opt" : "20%",
    "/tmp" : "20%",
    "/var" : "20%",
    "/var/log" : "20%"
    "/var/log/audit" : "20%",
    "/var/tmp" : "20%",
    "/var/www/html" : "20%",
    # Indexers:
    "/opt/splunk" : 5500,
    "/opt/splunkdata/hot" : local.splunk_hot,
    # Syslog Servers
    "/mnt/resource" : "20%",
    "/opt/syslog-ng" : 7000,
    # Docker Hosts
    "/opt/docker/containers" : "20%",
    "/opt/docker/overlay" : "20%",
    "/opt/docker/overlay2" : "20%",
    # VMRay
    "/opt/vmray" : "20%"
  }
}

resource "sensu_check" "check_dns_all" {
  for_each       = local.dns_checks_all
  name           = "check_dns_${each.key}"
  command        = "check_dns --hostname=${each.key} --expected-address=${each.value} --warning=${local.sensu_checks["dns"]["warning"]}  --critical=${local.sensu_checks["dns"]["critical"]}"
  namespace      = "default"
  subscriptions  = ["linux", ]
  handlers       = local.default_handlers
  interval       = 60
  publish        = "true"
  runtime_assets = ["sensu-plugins-monitoring", ]
}

resource "sensu_check" "check_dns_private" {
  for_each       = local.dns_checks_private
  name           = "check_dns_${each.key}"
  command        = "check_dns --hostname=${each.key} --expected-address=${each.value} --warning=${local.sensu_checks["dns"]["warning"]}  --critical=${local.sensu_checks["dns"]["critical"]}"
  namespace      = "default"
  subscriptions  = ["private_dns_client", ]
  handlers       = local.default_handlers
  interval       = 60
  publish        = "true"
  runtime_assets = ["sensu-plugins-monitoring", ]
}

# Systems that can't enumerate their mount points get
# a single 'check all' disk space option
resource "sensu_check" "check_disks_all" {
  name = "check_disk_all"
  # Checks all partitions except /snap/*
  command        = "check_disk -c 20% -A --ignore-eregi-path '/snap/' --ignore-eregi-path '/opt/docker/.*' --ignore-eregi-path '/(sys|proc)/'"
  namespace      = "default"
  subscriptions  = ["check_disk_all"]
  handlers       = local.default_handlers
  interval       = 300
  publish        = "true"
  runtime_assets = ["sensu-plugins-monitoring", ]
}

# Systems that enumerate their mount points subscribe to
# particular checks with thresholds defined above.
resource "sensu_check" "check_disks" {
  for_each       = local.disk_thresholds
  name           = "check_disk${replace(each.key, "/", "_")}"
  command        = "check_disk -c ${each.value} -p ${replace(each.key, "root", "/")}"
  namespace      = "default"
  subscriptions  = ["check_disk${replace(each.key, "/", "_")}"]
  handlers       = local.default_handlers
  interval       = 300
  publish        = "true"
  runtime_assets = ["sensu-plugins-monitoring", ]
}

#resource "sensu_check" "check_ping_interconnect-0" {
#  for_each       = toset(local.interconnect-0)
#  name           = "ping_interconnect-0-${index(local.interconnect-0, each.value) +1}"
#  command        = "check_ping -H ${each.value} -w 500,80% -c 500,80% -4"
#  namespace      = "default"
#  subscriptions  = [ "interconnect-0", ]
#  handlers       = local.default_handlers
#  interval       = 60
#  publish        = "true"
#  runtime_assets = [ "sensu-plugins-monitoring", ]
#}
#
#resource "sensu_check" "check_ping_interconnect-1" {
#  for_each       = toset(local.interconnect-1)
#  name           = "ping_interconnect-1-${index(local.interconnect-1, each.value) +1}"
#  command        = "check_ping -H ${each.value} -w 500,80% -c 500,80% -4"
#  namespace      = "default"
#  subscriptions  = [ "interconnect-1", ]
#  handlers       = local.default_handlers
#  interval       = 60
#  publish        = "true"
#  runtime_assets = [ "sensu-plugins-monitoring", ]
#}

resource "sensu_check" "check_phantom_ports" {
  name           = "phantom_ports"
  command        = "check-ports.rb -h 0.0.0.0 -p 443"
  namespace      = "default"
  subscriptions  = ["phantom_ports", ]
  handlers       = local.default_handlers
  interval       = 60
  publish        = "true"
  runtime_assets = ["sensu-plugins-network-checks", "sensu-ruby-runtime", ]
}

resource "sensu_check" "check_portal_http" {
  name           = "portal_http"
  command        = "check-http.rb -u https://portal.${var.dns_info["public"]["zone"]}/api/health/ --response-code 200"
  namespace      = "default"
  subscriptions  = ["portal", ]
  handlers       = local.default_handlers
  interval       = 60
  publish        = "true"
  runtime_assets = ["sensu-plugins-http", "sensu-ruby-runtime", ]
}

resource "sensu_check" "check_salt_master_ports" {
  name           = "salt_master_ports"
  command        = "check-ports.rb -h 0.0.0.0 -p 4505,4506"
  namespace      = "default"
  subscriptions  = ["salt_master_ports", ]
  handlers       = local.default_handlers
  interval       = 60
  publish        = "true"
  runtime_assets = ["sensu-plugins-network-checks", "sensu-ruby-runtime", ]
}

resource "sensu_check" "check_splunk_cm_ports" {
  name           = "splunk_cm_ports"
  command        = "check-ports.rb -h 0.0.0.0 -p 8089"
  namespace      = "default"
  subscriptions  = ["splunk_cm_ports", ]
  handlers       = local.default_handlers
  interval       = 60
  publish        = "true"
  runtime_assets = ["sensu-plugins-network-checks", "sensu-ruby-runtime", ]
}

resource "sensu_check" "check_splunk_ds_ports" {
  name           = "splunk_ds_ports"
  command        = "check-ports.rb -h 0.0.0.0 -p 8089"
  namespace      = "default"
  subscriptions  = ["splunk_ds_ports", ]
  handlers       = local.default_handlers
  interval       = 60
  publish        = "true"
  runtime_assets = ["sensu-plugins-network-checks", "sensu-ruby-runtime", ]
}

resource "sensu_check" "check_splunk_hf_ports" {
  name           = "splunk_hf_ports"
  command        = "check-ports.rb -h 0.0.0.0 -p 8089,8000"
  namespace      = "default"
  subscriptions  = ["splunk_hf_ports", ]
  handlers       = local.default_handlers
  interval       = 60
  publish        = "true"
  runtime_assets = ["sensu-plugins-network-checks", "sensu-ruby-runtime", ]
}

resource "sensu_check" "check_splunk_indexer_ports_moose" {
  name           = "splunk_indexer_ports_moose"
  command        = "check-ports.rb -h 0.0.0.0 -p 8089,9998,9887,8088"
  namespace      = "default"
  subscriptions  = ["splunk_indexer_ports_moose", ]
  handlers       = local.default_handlers
  interval       = 60
  publish        = "true"
  runtime_assets = ["sensu-plugins-network-checks", "sensu-ruby-runtime", ]
}

resource "sensu_check" "check_splunk_indexer_ports" {
  name           = "splunk_indexer_ports"
  command        = "check-ports.rb -h 0.0.0.0 -p 8089,9998,9887"
  namespace      = "default"
  subscriptions  = ["splunk_indexer_ports", ]
  handlers       = local.default_handlers
  interval       = 60
  publish        = "true"
  runtime_assets = ["sensu-plugins-network-checks", "sensu-ruby-runtime", ]
}

resource "sensu_check" "check_splunk_sh_ports" {
  name           = "splunk_sh_ports"
  command        = "check-ports.rb -h 0.0.0.0 -p 8089,8000"
  namespace      = "default"
  subscriptions  = ["splunk_sh_ports", ]
  handlers       = local.default_handlers
  interval       = 60
  publish        = "true"
  runtime_assets = ["sensu-plugins-network-checks", "sensu-ruby-runtime", ]
}

resource "sensu_check" "check_splunk_uf_ports" {
  name           = "splunk_uf_ports"
  command        = "check-ports.rb -h 0.0.0.0 -p 8089"
  namespace      = "default"
  subscriptions  = ["splunk", ]
  handlers       = local.default_handlers
  interval       = 60
  publish        = "true"
  runtime_assets = ["sensu-plugins-network-checks", "sensu-ruby-runtime", ]
}

resource "sensu_check" "check_syslog-ng_service" {
  name           = "syslog-ng_service"
  command        = "check-process.rb -p syslog-ng"
  namespace      = "default"
  subscriptions  = ["syslog-ng_service", ]
  handlers       = local.default_handlers
  interval       = 60
  publish        = "true"
  runtime_assets = ["sensu-plugins-process-checks", "sensu-ruby-runtime", ]
}

resource "sensu_check" "check_salt-minion_service" {
  name           = "salt-minion_service"
  command        = "check-process.rb -p salt-minion --critical-under 1"
  namespace      = "default"
  subscriptions  = ["salt-minion_service", ]
  handlers       = local.default_handlers
  interval       = 60
  publish        = "true"
  runtime_assets = ["sensu-plugins-process-checks", "sensu-ruby-runtime", ]
}

# Don't detect the Splunk Universal Forwarder running as root. 
# Only look for Splunk Enterprise running as splunk user.  
resource "sensu_check" "check_splunk_service" {
  name           = "splunk_service"
  command        = "check-process.rb -p 'splunkd -p' --user splunk -C 2"
  namespace      = "default"
  subscriptions  = ["splunk_service", ]
  handlers       = local.default_handlers
  interval       = 60
  publish        = "true"
  runtime_assets = ["sensu-plugins-process-checks", "sensu-ruby-runtime", ]
}

resource "sensu_check" "check_vault_service" {
  name           = "vault_service"
  command        = "check-process.rb -p vault"
  namespace      = "default"
  subscriptions  = ["vault_service", ]
  handlers       = local.default_handlers
  interval       = 60
  publish        = "true"
  runtime_assets = ["sensu-plugins-process-checks", "sensu-ruby-runtime", ]
}

resource "sensu_check" "check_cribl_service" {
  name           = "cribl_service"
  command        = "check-process.rb -p cribl"
  namespace      = "default"
  subscriptions  = ["cribl_service", ]
  handlers       = local.default_handlers
  interval       = 60
  publish        = "true"
  runtime_assets = ["sensu-plugins-process-checks", "sensu-ruby-runtime", ]
}