Gogs 5 lat temu
commit
108c1da63f
48 zmienionych plików z 1433 dodań i 0 usunięć
  1. 21 0
      .gitignore
  2. 9 0
      1.bootstrap/keys.tf
  3. 3 0
      1.bootstrap/provider.tf
  4. 32 0
      1.bootstrap/s3.terraform_backend.tf
  5. 6 0
      2.network/README.md
  6. 12 0
      2.network/backend.tf
  7. 4 0
      2.network/caller.tf
  8. 17 0
      2.network/outputs.tf
  9. 3 0
      2.network/provider.tf
  10. 84 0
      2.network/vpc.tf
  11. 12 0
      3.infrastructure/backend.tf
  12. 4 0
      3.infrastructure/caller.tf
  13. 27 0
      3.infrastructure/keys.tf
  14. 54 0
      3.infrastructure/policies/kms_ebs_key_policy.json.tpl
  15. 38 0
      3.infrastructure/policies/s3_splunk_bucket_policy.json.tpl
  16. 25 0
      3.infrastructure/policies/splunk-ec2-standalone-permissions-policy.json.tpl
  17. 12 0
      3.infrastructure/policies/splunk-ec2-standalone-trust-policy.json.tpl
  18. 26 0
      3.infrastructure/policies/splunk-smartstore-access.policy.json
  19. 3 0
      3.infrastructure/provider.tf
  20. 13 0
      3.infrastructure/remote_state_network.tf
  21. 37 0
      3.infrastructure/role.splunk_standalone.tf
  22. 72 0
      3.infrastructure/s3.smartstore.tf
  23. 18 0
      4.images/Makefile
  24. 3 0
      4.images/README.md
  25. 5 0
      4.images/configs/README.md
  26. 95 0
      4.images/configs/indexes.conf
  27. 3 0
      4.images/configs/server.conf
  28. 1 0
      4.images/configs/splunk.secret
  29. 12 0
      4.images/ephemeral_init/ephemeral-init.service
  30. 17 0
      4.images/ephemeral_init/initialize_nvme_storage.sh
  31. 124 0
      4.images/indexer.json
  32. 2 0
      4.images/scripts/clone-prep.sh
  33. 11 0
      4.images/scripts/configure-ephemeral-init.sh
  34. 6 0
      4.images/scripts/get-splunk-ansible.sh
  35. 32 0
      4.images/scripts/install-splunk.sh
  36. 12 0
      4.images/scripts/mountpoints.sh
  37. 9 0
      4.images/scripts/system-updates-and-packages.sh
  38. 7 0
      4.images/scripts/users.sh
  39. 79 0
      4.images/specs/default.yml
  40. 337 0
      4.images/specs/default.yml.spec
  41. 1 0
      4.images/splunk-dist/README.md
  42. 0 0
      4.images/splunk-dist/version_is_8.0.0
  43. 12 0
      5.standalone/backend.tf
  44. 4 0
      5.standalone/caller.tf
  45. 3 0
      5.standalone/provider.tf
  46. 13 0
      5.standalone/remote_state_network.tf
  47. 99 0
      5.standalone/standalone.tf
  48. 14 0
      README.md

+ 21 - 0
.gitignore

@@ -0,0 +1,21 @@
+*.tmp
+*.swp
+*.log
+.terraform
+*.ami.id
+
+# Terraform State
+terraform.tfstate
+*.backup
+
+# Binaries
+*.tgz
+*.rpm
+*.deb
+
+*.images/*.txt
+
+# I just use these for scratch
+tmpy
+tmpy2
+

+ 9 - 0
1.bootstrap/keys.tf

@@ -0,0 +1,9 @@
+resource "aws_kms_key" "terraform_key" {
+  description             = "This key is used to encrypt the terraform state for Splunk in S3"
+  deletion_window_in_days = 10
+}
+
+resource "aws_kms_alias" "terraform_key" {
+  name = "alias/splunk-standalone-terraform"
+  target_key_id = "${aws_kms_key.terraform_key.key_id}"
+}

+ 3 - 0
1.bootstrap/provider.tf

@@ -0,0 +1,3 @@
+provider "aws" {
+  region = "us-east-2"
+}

+ 32 - 0
1.bootstrap/s3.terraform_backend.tf

@@ -0,0 +1,32 @@
+resource "aws_s3_bucket" "splunk_terraform_state" {
+  # This must be unique, and must match the config in other directories's "backend.tf"
+  bucket = "ftd-splunk-standalone-terraform-state"
+
+  versioning {
+    enabled = true
+  }
+
+  lifecycle {
+    prevent_destroy = false
+  }
+
+  force_destroy = true
+
+  server_side_encryption_configuration {
+    rule {
+      apply_server_side_encryption_by_default {
+        kms_master_key_id = "${aws_kms_key.terraform_key.arn}"
+        sse_algorithm     = "aws:kms"
+      }
+    }
+  }
+}
+
+resource "aws_s3_bucket_public_access_block" "keep_terraform_safe" {
+  bucket = "${aws_s3_bucket.splunk_terraform_state.id}"
+
+  block_public_acls = true
+  block_public_policy = true
+  ignore_public_acls = true
+  restrict_public_buckets = true
+}

+ 6 - 0
2.network/README.md

@@ -0,0 +1,6 @@
+# Network
+
+If you don't ahve VPCs and subnets, this will create them.
+
+If you already have assigned VPCs and subnets, make sure they have S3 endpoints.
+

+ 12 - 0
2.network/backend.tf

@@ -0,0 +1,12 @@
+# This is where terraform stores state, which allows for multiple
+# developers on teh same terraform configuration.
+terraform {
+  backend "s3" {
+    # This must match the name from the bootstrap directory
+    bucket = "ftd-splunk-standalone-terraform-state"
+    key    = "splunk/network/terraform.tfstate"
+    region = "us-east-2"
+    encrypt = true
+    kms_key_id = "alias/splunk-standalone-terraform"
+  }
+}

+ 4 - 0
2.network/caller.tf

@@ -0,0 +1,4 @@
+# callers are used to get information about the current
+# aws account and region.
+data "aws_caller_identity" "current" {}
+data "aws_region" "current" {}

+ 17 - 0
2.network/outputs.tf

@@ -0,0 +1,17 @@
+output "vpc_id" {
+  value = aws_vpc.vpc_primary.id
+}
+output "vpc_cidr" {
+  value = aws_vpc.vpc_primary.cidr_block
+}
+
+output "subnet0_id" {
+  value = aws_subnet.splunk_subnet_0.id
+}
+output "subnet1_id" {
+  value = aws_subnet.splunk_subnet_1.id
+}
+output "subnet2_id" {
+  value = aws_subnet.splunk_subnet_2.id
+}
+

+ 3 - 0
2.network/provider.tf

@@ -0,0 +1,3 @@
+provider "aws" {
+  region = "us-east-2"
+}

+ 84 - 0
2.network/vpc.tf

@@ -0,0 +1,84 @@
+###########
+# Create a VPC with an Internet gateway for everybody to play in
+resource "aws_vpc" "vpc_primary" {
+  cidr_block = "10.45.0.0/16"
+  enable_dns_support = true
+  enable_dns_hostnames = true
+  tags = {
+    Name = "Splunk Standalone"
+    Project = "Splunk"
+    Environment = "Production"
+  }
+}
+
+resource "aws_internet_gateway" "gw_primary" {
+  vpc_id = "${aws_vpc.vpc_primary.id}"
+  tags = {
+    Name = "Primary Gateway"
+  }
+}
+
+###########
+# Create 3 Subnets, one in each AZ.
+# Note that most servers will be spread between 1 and 2, but we need
+# 3 AZs for a quorum for the SearchHead Cluster.
+data "aws_availability_zones" "available" {}
+resource "aws_subnet" "splunk_subnet_0" {
+  vpc_id = "${aws_vpc.vpc_primary.id}"
+  cidr_block = "10.45.0.0/24"
+  availability_zone = "${data.aws_availability_zones.available.names[0]}"
+  map_public_ip_on_launch = true
+  tags = {
+    Name = "Splunk Subnet 0"
+  }
+}
+resource "aws_subnet" "splunk_subnet_1" {
+  vpc_id = "${aws_vpc.vpc_primary.id}"
+  cidr_block = "10.45.1.0/24"
+  availability_zone = "${data.aws_availability_zones.available.names[1]}"
+  map_public_ip_on_launch = true
+  tags = {
+    Name = "Splunk Subnet 1"
+  }
+}
+resource "aws_subnet" "splunk_subnet_2" {
+  vpc_id = "${aws_vpc.vpc_primary.id}"
+  cidr_block = "10.45.2.0/24"
+  availability_zone = "${data.aws_availability_zones.available.names[2]}"
+  map_public_ip_on_launch = true
+  tags = {
+    Name = "Splunk Subnet 2"
+  }
+}
+
+###########
+# Create a route table
+resource "aws_route_table" "r" {
+  vpc_id = "${aws_vpc.vpc_primary.id}"
+  route {
+    cidr_block = "0.0.0.0/0"
+    gateway_id = "${aws_internet_gateway.gw_primary.id}"
+  }
+  tags = {
+    Name = "Primary Route Table"
+  }
+}
+
+###########
+# Create an S3 endpoint
+resource "aws_vpc_endpoint" "s3" {
+  vpc_id = "${aws_vpc.vpc_primary.id}"
+  service_name = "com.amazonaws.us-east-2.s3"
+}
+resource "aws_vpc_endpoint_route_table_association" "s3_routing" {
+  route_table_id = "${aws_route_table.r.id}"
+  vpc_endpoint_id = "${aws_vpc_endpoint.s3.id}"
+}
+
+
+# Associate with VPC
+resource "aws_main_route_table_association" "rt_public_a" {
+  vpc_id = "${aws_vpc.vpc_primary.id}"
+  route_table_id = "${aws_route_table.r.id}"
+}
+

+ 12 - 0
3.infrastructure/backend.tf

@@ -0,0 +1,12 @@
+# This is where terraform stores state, which allows for multiple
+# developers on teh same terraform configuration.
+terraform {
+  backend "s3" {
+    # This must match the name from the bootstrap directory
+    bucket = "ftd-splunk-standalone-terraform-state"
+    key    = "splunk/infrastructure/terraform.tfstate"
+    region = "us-east-2"
+    encrypt = true
+    kms_key_id = "alias/splunk-standalone-terraform"
+  }
+}

+ 4 - 0
3.infrastructure/caller.tf

@@ -0,0 +1,4 @@
+# callers are used to get information about the current
+# aws account and region.
+data "aws_caller_identity" "current" {}
+data "aws_region" "current" {}

+ 27 - 0
3.infrastructure/keys.tf

@@ -0,0 +1,27 @@
+data "template_file" "kms_ebs_key_policy" {
+  template = "${file("policies/kms_ebs_key_policy.json.tpl")}"
+  vars = {
+    account = "${data.aws_caller_identity.current.account_id}"
+  }
+}
+
+resource "aws_kms_key" "splunk_ebs_key" {
+  description             = "This key is used to encrypt the EBS drives for splunk systems."
+  deletion_window_in_days = 10
+  policy = "${data.template_file.kms_ebs_key_policy.rendered}"
+}
+
+resource "aws_kms_alias" "splunk_ebs_key" {
+  name = "alias/splunk_standalone_ebs"
+  target_key_id = "${aws_kms_key.splunk_ebs_key.key_id}"
+}
+
+resource "aws_kms_key" "splunk_s3_key" {
+  description             = "This key is used to encrypt the S3 SmartStore backend for splunk systems."
+  deletion_window_in_days = 10
+}
+
+resource "aws_kms_alias" "splunk_s3_key" {
+  name = "alias/splunk_standalone_s3"
+  target_key_id = "${aws_kms_key.splunk_s3_key.key_id}"
+}

+ 54 - 0
3.infrastructure/policies/kms_ebs_key_policy.json.tpl

@@ -0,0 +1,54 @@
+{
+    "Id": "key-consolepolicy-3",
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Sid": "Enable IAM User Permissions",
+            "Effect": "Allow",
+            "Principal": {
+                "AWS": [ "arn:aws:iam::${account}:root" ]
+            },
+            "Action": "kms:*",
+            "Resource": "*"
+        },
+        {
+            "Sid": "Allow use of the key",
+            "Effect": "Allow",
+            "Principal": {
+                "AWS": [
+                    "arn:aws:iam::${account}:role/aws-service-role/spot.amazonaws.com/AWSServiceRoleForEC2Spot",
+                    "arn:aws:iam::${account}:role/aws-service-role/spotfleet.amazonaws.com/AWSServiceRoleForEC2SpotFleet"
+                ]
+            },
+            "Action": [
+                "kms:Encrypt",
+                "kms:Decrypt",
+                "kms:ReEncrypt*",
+                "kms:GenerateDataKey*",
+                "kms:DescribeKey"
+            ],
+            "Resource": "*"
+        },
+        {
+            "Sid": "Allow attachment of persistent resources",
+            "Effect": "Allow",
+            "Principal": {
+                "AWS": [
+                    "arn:aws:iam::${account}:role/aws-service-role/spot.amazonaws.com/AWSServiceRoleForEC2Spot",
+                    "arn:aws:iam::${account}:role/aws-service-role/spotfleet.amazonaws.com/AWSServiceRoleForEC2SpotFleet"
+                ]
+            },
+            "Action": [
+                "kms:CreateGrant",
+                "kms:ListGrants",
+                "kms:RevokeGrant"
+            ],
+            "Resource": "*",
+            "Condition": {
+                "Bool": {
+                    "kms:GrantIsForAWSResource": "true"
+                }
+            }
+        }
+    ]
+}

+ 38 - 0
3.infrastructure/policies/s3_splunk_bucket_policy.json.tpl

@@ -0,0 +1,38 @@
+{
+  "Version": "2012-10-17",
+  "Id": "SmartStoreBucketPolicy",
+  "Statement": [
+    {
+      "Sid": "RestrictedAllow",
+      "Effect": "Allow",
+      "Principal": {
+        "AWS": [
+          "${role_arn}"
+        ]
+      },
+      "Action": [
+        "s3:*"
+      ],
+      "Resource": [
+        "${bucket_arn}",
+        "${bucket_arn}/*"
+      ],
+      "Condition": {
+         "IpAddress": {"aws:SourceIp": "${vpc_cidr}"}
+      }
+    },
+    {
+      "Action": "s3:*",
+      "Effect": "Allow",
+      "Resource": [
+        "${bucket_arn}",
+        "${bucket_arn}/*"
+      ],
+      "Principal": {
+        "AWS": [
+          "arn:aws:iam::${account}:root"
+        ]
+      }
+    }
+  ]
+}

+ 25 - 0
3.infrastructure/policies/splunk-ec2-standalone-permissions-policy.json.tpl

@@ -0,0 +1,25 @@
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Sid": "S3AccessTHISISTOOBROADANDNEEDSTOBEREDUCED",
+            "Effect": "Allow",
+            "Action": [
+              "s3:*"
+            ],
+            "Resource": [
+              "${smartstore_bucket_arn}",
+              "${smartstore_bucket_arn}/*"
+            ]
+        },
+        {
+            "Sid": "KMSAccessTHISNeedsToBeRestricted",
+            "Effect": "Allow",
+            "Action": [
+              "kms:*"
+            ],
+            "Resource": [ "*"
+            ]
+        }
+    ]
+}

+ 12 - 0
3.infrastructure/policies/splunk-ec2-standalone-trust-policy.json.tpl

@@ -0,0 +1,12 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": {
+        "Service": "ec2.amazonaws.com"
+      },
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}

+ 26 - 0
3.infrastructure/policies/splunk-smartstore-access.policy.json

@@ -0,0 +1,26 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "s3:ListBucket"
+      ],
+     "Resource": [
+        "${bucket_arn}"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "s3:PutObject",
+        "s3:GetObject",
+        "s3:DeleteObject",
+        "s3:PutObjectAcl"
+      ],
+      "Resource": [
+         "${bucket_arn}"
+      ]
+    }
+  ]
+}

+ 3 - 0
3.infrastructure/provider.tf

@@ -0,0 +1,3 @@
+provider "aws" {
+  region = "us-east-2"
+}

+ 13 - 0
3.infrastructure/remote_state_network.tf

@@ -0,0 +1,13 @@
+data "terraform_remote_state" "network" {
+  backend = "s3"
+
+  config = {
+    # This must match the name from the bootstrap directory
+    bucket = "ftd-splunk-standalone-terraform-state"
+    # This must match the folder from the 2.network backend directory
+    key    = "splunk/network/terraform.tfstate"
+    region = "us-east-2"
+    encrypt = true
+    kms_key_id = "alias/splunk-standalone-terraform"
+  }
+}

+ 37 - 0
3.infrastructure/role.splunk_standalone.tf

@@ -0,0 +1,37 @@
+data "template_file" "splunk-ec2-standalone-trust-policy" {
+  template = "${file("policies/splunk-ec2-standalone-trust-policy.json.tpl")}"
+  vars = {
+    account = "${data.aws_caller_identity.current.account_id}"
+  }
+}
+
+resource "aws_iam_role" "Splunk-EC2-Standalone" {
+  name = "Splunk-EC2-Standalone"
+  assume_role_policy = "${data.template_file.splunk-ec2-standalone-trust-policy.rendered}"
+}
+
+data "template_file" "splunk-ec2-standalone-permissions-policy" {
+  template = "${file("policies/splunk-ec2-standalone-permissions-policy.json.tpl")}"
+  vars = {
+    account = "${data.aws_caller_identity.current.account_id}",
+    smartstore_bucket_arn = "${aws_s3_bucket.splunk-smartstore.arn}"
+  }
+}
+
+resource "aws_iam_policy" "Splunk-EC2-Standalone" {
+  name = "Splunk-EC2-Standalone"
+  path = "/Splunk/"
+  description = "Splunk policy for EC2 Standalone"
+  policy = "${data.template_file.splunk-ec2-standalone-permissions-policy.rendered}"
+}
+
+resource "aws_iam_role_policy_attachment" "Splunk-EC2-Standalone" {
+  role       = "${aws_iam_role.Splunk-EC2-Standalone.name}"
+  policy_arn = "${aws_iam_policy.Splunk-EC2-Standalone.arn}"
+}
+
+resource "aws_iam_instance_profile" "Splunk-EC2-Indexer" {
+  name = "Splunk-EC2-Standalone"
+  role = "${aws_iam_role.Splunk-EC2-Standalone.name}"
+}
+

+ 72 - 0
3.infrastructure/s3.smartstore.tf

@@ -0,0 +1,72 @@
+# This is the smartstore bucket. This is the data. Keep this safe.
+resource "aws_s3_bucket" "splunk-smartstore" {
+  bucket = "ftd-splunk-smartstore"
+
+  versioning {
+    enabled = true # Allows us to archive frozen data directly into glacier without splunk intervention
+  }
+
+  lifecycle {
+    # Set to TRUE for production!
+    prevent_destroy = false
+  }
+  # set to FALSE for production!
+  force_destroy = true
+
+  server_side_encryption_configuration {
+    rule {
+      apply_server_side_encryption_by_default {
+        kms_master_key_id = "${aws_kms_key.splunk_s3_key.arn}"
+        sse_algorithm     = "aws:kms"
+      }
+    }
+  }
+
+  # This will save some money by moving to "infrequently accessed" after a period of time
+  lifecycle_rule {
+    id = "InfrequentAccessAfteraYear"
+    enabled = true
+    transition {
+      # For production, probably set this to something longer, like 90 days or a year
+      days = 30
+      storage_class = "STANDARD_IA"
+    }
+
+    noncurrent_version_transition {
+      days = 0
+      storage_class = "GLACIER"
+    }
+    
+    # This would be a good place to transition to deep archive a few days/weeks/months after
+    # the transition to glacier, if you're really only rarely going to access it.
+  }
+
+  tags = {
+    Project = "Splunk"
+    Environment = "Production"
+  }
+}
+
+resource "aws_s3_bucket_public_access_block" "keep_smartstore_safe" {
+  bucket = "${aws_s3_bucket.splunk-smartstore.id}"
+
+  block_public_acls = true
+  block_public_policy = true
+  ignore_public_acls = true
+  restrict_public_buckets = true
+}
+
+data "template_file" "s3_splunk_bucket_policy" {
+  template = "${file("policies/s3_splunk_bucket_policy.json.tpl")}"
+  vars = {
+    account = "${data.aws_caller_identity.current.account_id}"
+    bucket_arn = "${aws_s3_bucket.splunk-smartstore.arn}"
+    vpc_cidr = "${data.terraform_remote_state.network.outputs.vpc_cidr}"
+    role_arn = "${aws_iam_role.Splunk-EC2-Standalone.arn}"
+  }
+}
+
+resource "aws_s3_bucket_policy" "splunk_servers_only" {
+  bucket = "${aws_s3_bucket.splunk-smartstore.id}"
+  policy = "${data.template_file.s3_splunk_bucket_policy.rendered}"
+}

+ 18 - 0
4.images/Makefile

@@ -0,0 +1,18 @@
+# AMI IDs
+AMITARGETS=indexer.ami.id
+PACKER=/usr/local/bin/packer
+
+all: $(AMITARGETS)
+
+%.ami.id: %.json scripts/* ephemeral_init/* configs/*
+	$(eval VPC_ID=$(shell cd ../2.network && terraform output vpc_id))
+	$(eval SUBNET_ID=$(shell cd ../2.network && terraform output subnet0_id))
+	$(PACKER) validate -var "vpc_id=$(VPC_ID)" -var "subnet_id=$(SUBNET_ID)" $<
+	$(PACKER) inspect $<
+	$(PACKER) build -var "vpc_id=$(VPC_ID)" -var "subnet_id=$(SUBNET_ID)" $< 2>&1 | tee $(basename $(notdir $<)).log
+	tail -2 $(basename $(notdir $<)).log | head -2 | awk 'match($$0, /ami-.*/) { print substr($$0, RSTART, RLENGTH) }' > $@
+
+clean:
+	-rm $(AMITARGETS)
+	-rm *.log
+	-rm *.tmp

+ 3 - 0
4.images/README.md

@@ -0,0 +1,3 @@
+# Images
+
+Contains the packer configs for images.

+ 5 - 0
4.images/configs/README.md

@@ -0,0 +1,5 @@
+This file contains various config files that will be copied to the splunk server.
+
+THERE ARE BETTER WAYS TO DO THIS FOR PRODUCTION.
+
+Use ansible, use salt, use a script, etc.

+ 95 - 0
4.images/configs/indexes.conf

@@ -0,0 +1,95 @@
+# Global settings
+coldToFrozenDir = /opt/splunk/var/lib/splunkfrozen
+
+# S3 Volume(s)
+[volume:s3volume]
+storageType = remote
+path = s3://ftd-splunk-smartstore/standalone/
+remote.s3.endpoint = https://s3.us-east-2.amazonaws.com
+remote.s3.supports_versioning = true
+remote.s3.encryption = sse-c
+remote.s3.kms.key_id = alias/splunk_standalone_s3
+# To test:
+#   Make sure there's a file on the smartstore storage, and run:
+#   splunk cmd splunkd rfs -- ls --starts-with volume:remote_store
+
+[volume:smartstore_cache]
+path = /opt/splunk/var/lib/smartstore_cache
+
+# Built-in Indexes
+[main]
+# Step 0: Set maxDataSize back to auto
+maxDataSize = auto
+# Step 1: Set the remote path to S3, and let the migration
+#         happen. During this time, it will continue to use
+#         the 'hot/warm' (and cold?) mount points for the
+#         cache.
+remotePath = volume:s3volume/$_index_name
+# Step 2: *WAIT FOR ALL BUCKETS TO HAVE BEEN COPIED*
+# Step 3: Change paths to the cache directory
+homePath = volume:smartstore_cache/$_index_name/db
+# SmartStore-enabled indexes do not use thawedPath or coldPath, but you must still specify them here.
+coldPath = $SPLUNK_DB/$_index_name/colddb
+thawedPath = /opt/splunk/var/lib/splunkthawed/main
+
+[_internal]
+maxDataSize = auto
+remotePath = volume:s3volume/$_index_name
+homePath = volume:smartstore_cache/$_index_name/db
+coldPath = $SPLUNK_DB/$_index_name/colddb
+thawedPath = /opt/splunk/var/lib/splunkthawed/_internal
+
+[history]
+maxDataSize = auto
+remotePath = volume:s3volume/$_index_name
+homePath = volume:smartstore_cache/$_index_name/db
+coldPath = $SPLUNK_DB/$_index_name/colddb
+thawedPath = /opt/splunk/var/lib/splunkthawed/history
+
+[summary]
+maxDataSize = auto
+remotePath = volume:s3volume/$_index_name
+homePath = volume:smartstore_cache/$_index_name/db
+coldPath = $SPLUNK_DB/$_index_name/colddb
+thawedPath = /opt/splunk/var/lib/splunkthawed/summary
+
+[_audit]
+maxDataSize = auto
+remotePath = volume:s3volume/$_index_name
+homePath = volume:smartstore_cache/$_index_name/db
+coldPath = $SPLUNK_DB/$_index_name/colddb
+thawedPath = /opt/splunk/var/lib/splunkthawed/_audit
+
+[_thefishbucket]
+maxDataSize = auto
+remotePath = volume:s3volume/$_index_name
+homePath = volume:smartstore_cache/$_index_name/db
+coldPath = $SPLUNK_DB/$_index_name/colddb
+thawedPath = /opt/splunk/var/lib/splunkthawed/_thefishbucket
+
+[_introspection]
+maxDataSize = auto
+remotePath = volume:s3volume/$_index_name
+homePath = volume:smartstore_cache/$_index_name/db
+coldPath = $SPLUNK_DB/$_index_name/colddb
+thawedPath = /opt/splunk/var/lib/splunkthawed/_introspection
+
+[_telemetry]
+maxDataSize = auto
+remotePath = volume:s3volume/$_index_name
+homePath = volume:smartstore_cache/$_index_name/db
+coldPath = $SPLUNK_DB/$_index_name/colddb
+thawedPath = /opt/splunk/var/lib/splunkthawed/_telemetry
+
+[_metrics]
+maxDataSize = auto
+remotePath = volume:s3volume/$_index_name
+homePath = volume:smartstore_cache/$_index_name/db
+coldPath = $SPLUNK_DB/$_index_name/colddb
+thawedPath = /opt/splunk/var/lib/splunkthawed/_metrics
+
+[fred]
+remotePath = volume:s3volume/$_index_name
+homePath = volume:smartstore_cache/$_index_name/db
+coldPath = $SPLUNK_DB/$_index_name/colddb
+thawedPath = /opt/splunk/var/lib/splunkthawed/fred

+ 3 - 0
4.images/configs/server.conf

@@ -0,0 +1,3 @@
+# For SmartStore
+[diskUsage]
+minFreeSpace = 20%

+ 1 - 0
4.images/configs/splunk.secret

@@ -0,0 +1 @@
+Te9zoRy1foOyqcyFei8omvTHAq5eKHfjcjwyYhFg0C9ly5Sq0VxBzCADqV.K.XYOroJkMRUfKoumKKoS7eCSSfu5u6ayObcHEpD5hg4ViXSgtnlEfFWQloL1m/uLW0QPlp/Zt4BUBpJhXSIuGU5kZN3Iua6M2PLMnfQrzB/8rsZ9sfb5dlDro1TUPbso1V8LvBVvj4ZIXutsGIzvKT.w1UEVCTqbzsXPEWqR.W1qwFtasO7ZaCEm4QTsXvppIc

+ 12 - 0
4.images/ephemeral_init/ephemeral-init.service

@@ -0,0 +1,12 @@
+# goes in /etc/systemd/system
+[Unit]
+#DefaultDependencies=no
+#After=sysinit.target local-fs.target
+#Before=base.target
+RequiresMountsFor=/opt/splunk
+
+[Service]
+ExecStart=/usr/local/sbin/initialize_nvme_storage.sh
+
+[Install]
+WantedBy=default.target

+ 17 - 0
4.images/ephemeral_init/initialize_nvme_storage.sh

@@ -0,0 +1,17 @@
+#! /bin/bash
+# This needs to be called every boot, before Splunk starts.
+if [ ! -b /dev/md0 ]; then
+  sudo rm -f /etc/mdadm.conf 2> /dev/null
+  DEVICES=$(ls -1 /dev/nvme*n1)
+  NUM=$(ls -1 /dev/nvme*n1 | wc -l)
+  sudo mdadm --create --force --verbose /dev/md0 --level=0 --name=SMARTSTORE_CACHE --raid-devices=${NUM} ${DEVICES}
+  sudo mkfs -t xfs /dev/md0
+  sudo mkdir -p /opt/splunk/var/lib/smartstore_cache 2> /dev/null
+  sudo chown -R splunk:splunk /opt/splunk 2> /dev/null
+  sudo mdadm --verbose --detail --scan | sudo tee -a /etc/mdadm.conf
+fi
+sudo mount /dev/md0 /opt/splunk/var/lib/smartstore_cache
+# Fix permissions each boot:
+sudo chown -R splunk:splunk /opt/splunk
+sudo find /opt/splunk -type d -exec chmod 755 {} \;
+sudo find /opt/splunk -type f -exec chmod g+r {} \;

+ 124 - 0
4.images/indexer.json

@@ -0,0 +1,124 @@
+{
+	"variables": {
+		"instance_type": "i3.large",
+		"region": "us-east-2",
+    "vpc_id": "{{vpc_id}}",
+    "subnet_id": "{{subnet_id}}",
+    "os_size": "10",
+    "splunk_size": "20",
+    "swap_size": "2"
+	},
+
+	"builders": [
+    {
+      "type": "amazon-ebs",
+      "region": "{{user `region`}}",
+      "vpc_id": "{{user `vpc_id`}}",
+      "subnet_id": "{{user `subnet_id`}}",
+      "source_ami_filter": {
+        "filters": {
+          "virtualization-type": "hvm",
+          "name": "amzn2-ami-hvm-2*",
+          "root-device-type": "ebs"
+        },
+        "owners": ["137112412989"],
+        "most_recent": true
+      },
+      "spot_instance_types": ["{{user `instance_type`}}"],
+      "spot_price": "auto",
+      "ssh_username": "ec2-user",
+      "ami_name": "FTD-Splunk-Standalone",
+      "ebs_optimized": true,
+      "iam_instance_profile": "Splunk-EC2-Standalone",
+      "associate_public_ip_address": true,
+      "ssh_interface": "public_ip",
+      "spot_tags": {
+        "Name": "FTD-Splunk-Standalone",
+        "project": "Splunk"
+      },
+      "run_tags": {
+        "Name": "FTD-Splunk-Standalone",
+        "project": "Splunk"
+      },
+      "tags": {
+        "Name": "FTD-Splunk-Standalone",
+        "project": "Splunk"
+      },
+      "launch_block_device_mappings": [
+        {
+          "device_name": "/dev/xvda",
+          "encrypted": "true",
+          "delete_on_termination": true,
+          "volume_type": "gp2",
+          "volume_size": "{{user `os_size`}}",
+          "kms_key_id": "alias/splunk_standalone_ebs"
+        },
+        {
+          "device_name": "/dev/xvdb",
+          "encrypted": "true",
+          "delete_on_termination": true,
+          "volume_type": "gp2",
+          "volume_size": "{{user `splunk_size`}}",
+          "kms_key_id": "alias/MBOX_EBS_Key"
+        },
+        {
+          "device_name": "/dev/xvdc",
+          "encrypted": "true",
+          "delete_on_termination": true,
+          "volume_type": "gp2",
+          "volume_size": "{{user `swap_size`}}",
+          "kms_key_id": "alias/MBOX_EBS_Key"
+        }
+      ],
+      "force_deregister": true,
+      "force_delete_snapshot": true
+    }
+  ],
+ 
+  "provisioners": [
+    {
+      "type": "shell",
+      "script": "scripts/mountpoints.sh"
+    },
+    {
+      "type": "shell",
+      "script": "scripts/system-updates-and-packages.sh"
+    },
+    {
+      "type": "shell",
+      "script": "scripts/users.sh"
+    },
+    {
+      "type": "file",
+      "source": "ephemeral_init/ephemeral-init.service",
+      "destination": "/tmp/ephemeral-init.service"
+    },
+    {
+      "type": "file",
+      "source": "ephemeral_init/initialize_nvme_storage.sh",
+      "destination": "/tmp/initialize_nvme_storage.sh"
+    },
+    {
+      "type": "shell",
+      "script": "scripts/configure-ephemeral-init.sh"
+    },
+    {
+      "type": "shell",
+      "inline": [ "mkdir configs/" ]
+    },
+    {
+      "type": "file",
+      "source": "configs/",
+      "destination": "configs/"
+    },
+    {
+      "type": "shell",
+      "script": "scripts/install-splunk.sh"
+    },
+    {
+      "type": "shell",
+      "script": "scripts/clone-prep.sh"
+    }
+  ]
+}
+

+ 2 - 0
4.images/scripts/clone-prep.sh

@@ -0,0 +1,2 @@
+#! /bin/bash
+sudo -u splunk /opt/splunk/bin/splunk clone-prep-clear-config

+ 11 - 0
4.images/scripts/configure-ephemeral-init.sh

@@ -0,0 +1,11 @@
+#! /bin/bash
+#
+# These are commands that are needed for ephemeral-init (initializing the local RAID-0 array on boot)
+# but aren't needed every time.
+sudo mv /tmp/initialize_nvme_storage.sh /usr/local/sbin/
+sudo chmod 755 /usr/local/sbin/initialize_nvme_storage.sh
+sudo mv /tmp/ephemeral-init.service /etc/systemd/system/ephemeral-init.service
+sudo chmod 644 /etc/systemd/system/ephemeral-init.service
+sudo systemctl daemon-reload
+sudo systemctl enable ephemeral-init.service
+exit 0

+ 6 - 0
4.images/scripts/get-splunk-ansible.sh

@@ -0,0 +1,6 @@
+#! /bin/bash
+sudo git clone https://github.com/splunk/splunk-ansible.git /usr/local/src/splunk-ansible
+cd /usr/local/src/splunk-ansible
+sleep 5
+sudo git checkout master
+sudo chown -R ansible:ansible /usr/local/src/splunk-ansible

+ 32 - 0
4.images/scripts/install-splunk.sh

@@ -0,0 +1,32 @@
+#! /bin/bash
+
+# Fetch and install splunk
+wget -O /tmp/splunk.rpm 'https://www.splunk.com/bin/splunk/DownloadActivityServlet?architecture=x86_64&platform=linux&version=8.0.0&product=splunk&filename=splunk-8.0.0-1357bef0a7f6-linux-2.6-x86_64.rpm&wget=true'
+sudo rpm -ivh /tmp/splunk.rpm
+
+# Custom configs
+sudo mv configs/splunk.secret /opt/splunk/etc/auth/
+for i in indexes.conf server.conf
+do
+  sudo mv configs/$i /opt/splunk/etc/system/local/
+done
+
+# Install Add Ons
+#pushd
+#cd /opt/splunk/etc/apps
+#sudo tar xvzf /tmp/splunk-add-on-for-amazon-web-services_460.tgz
+#sudo tar xvzf /tmp/splunk-app-for-aws_513.tgz
+#sudo tar xvzf /tmp/splunk-add-on-for-unix-and-linux_700.tgz
+#sudo tar xvzf /tmp/splunk-app-for-unix-and-linux_525.tgz
+#popd
+
+# Extra dirs
+sudo mkdir -p /opt/splunk/var/lib/splunkfrozen
+
+# Fix Permissions
+sudo chown -R splunk:splunk /opt/splunk
+sudo find /opt/splunk -type d -exec chmod 755 {} \;
+sudo find /opt/splunk -type f -exec chmod g+r {} \;
+sudo /opt/splunk/bin/splunk enable boot-start -user splunk --accept-license --answer-yes --seed-passwd badpassword
+sudo setfacl -Rm u:splunk:r-x,d:u:splunk:r-x /var/log
+sudo -u splunk --set-home /opt/splunk/bin/splunk enable web-ssl -auth admin:badpassword

+ 12 - 0
4.images/scripts/mountpoints.sh

@@ -0,0 +1,12 @@
+#! /bin/bash
+sudo mkfs -t xfs /dev/xvdb
+BLKID=$(sudo blkid | grep /dev/xvdb | awk '{ print $2 }' | sed 's/UUID="//' | sed 's/"//')
+sudo mkdir /opt/splunk
+echo "UUID=${BLKID} /opt/splunk xfs defaults,nofail 0 2" | sudo tee -a /etc/fstab
+
+sudo mkswap /dev/xvdc
+sudo swapon /dev/xvdc
+BLKID=$(sudo blkid | grep /dev/xvdc | awk '{ print $2 }' | sed 's/UUID="//' | sed 's/"//')
+echo "UUID=${BLKID} /opt/splunk swap swap,nofail 0 2" | sudo tee -a /etc/fstab
+
+sudo mount -a

+ 9 - 0
4.images/scripts/system-updates-and-packages.sh

@@ -0,0 +1,9 @@
+#! /bin/bash
+sleep 45 # Must sleep to allow boot to finish
+sudo yum -y update
+sleep 5
+sudo yum -y upgrade
+sleep 5
+sudo yum install -y vim-core git
+sleep 5
+sudo amazon-linux-extras install ansible2

+ 7 - 0
4.images/scripts/users.sh

@@ -0,0 +1,7 @@
+#! /bin/bash
+# Provision users
+sudo useradd --comment "Splunk Service Account" --home-dir /opt/splunk --inactive -1 --no-create-home --system --shell /bin/bash splunk
+sudo chown -R splunk:splunk /opt/splunk
+sudo useradd --comment "Ansible Superuser" --home-dir /opt/splunk --groups wheel --inactive -1 --no-create-home --system --shell /bin/bash ansible
+# Add ec2-user to the splunk group
+sudo usermod -a -G splunk ec2-user

+ 79 - 0
4.images/specs/default.yml

@@ -0,0 +1,79 @@
+---
+ansible_post_tasks: null
+ansible_pre_tasks: null
+hide_password: false
+retry_num: 50
+shc_bootstrap_delay: 30
+splunk_home_ownership_enforcement: true
+
+config:
+  baked: default.yml
+  defaults_dir: /tmp/defaults
+  env:
+    headers: null
+    var: SPLUNK_DEFAULTS_URL
+    verify: true
+  host:
+    headers: null
+    url: null
+    verify: true
+  max_delay: 60
+  max_retries: 3
+  max_timeout: 1200
+
+splunk:
+  role: splunk_standalone
+  upgrade: false
+  build_location: /tmp/splunk.tgz
+  build_remote_src: true
+  license_master_included: false
+  apps_location: null
+  license_uri: null
+  admin_user: admin
+  app_paths:
+    default: /opt/splunk/etc/apps
+    deployment: /opt/splunk/etc/deployment-apps
+    httpinput: /opt/splunk/etc/apps/splunk_httpinput
+    idxc: /opt/splunk/etc/master-apps
+    shc: /opt/splunk/etc/shcluster/apps
+  enable_service: false
+  exec: /opt/splunk/bin/splunk
+  group: splunk
+  hec_disabled: 0
+  hec_enableSSL: 1
+  hec_port: 8088
+  hec_token: 4a8a737d-5452-426c-a6f7-106dca4e813f
+  home: /opt/splunk
+  http_enableSSL: 0
+  http_enableSSL_cert: null
+  http_enableSSL_privKey: null
+  http_enableSSL_privKey_password: null
+  http_port: 8000
+  idxc:
+    enable: false
+    label: idxc_label
+    replication_factor: 3
+    replication_port: 9887
+    search_factor: 3
+    secret: dmwHG97SpM+GzeGPUELwr7xXowSAVmLW
+  ignore_license: false
+  license_download_dest: /tmp/splunk.lic
+  nfr_license: /tmp/nfr_enterprise.lic
+  opt: /opt
+  password: helloworld
+  pid: /opt/splunk/var/run/splunk/splunkd.pid
+  s2s_enable: true
+  s2s_port: 9997
+  search_head_cluster_url: null
+  secret: null
+  shc:
+    enable: false
+    label: shc_label
+    replication_factor: 3
+    replication_port: 9887
+    secret: EpcUlTUHMSOhdjRZb3QqPYf9Lf7L991c
+  smartstore: null
+  svc_port: 8089
+  tar_dir: splunk
+  user: splunk
+  wildcard_license: false

+ 337 - 0
4.images/specs/default.yml.spec

@@ -0,0 +1,337 @@
+ansible_post_tasks: <str>
+* Comma-separated list of paths or URLs to custom Ansible playbooks to run AFTER Splunk has been setup using the provided site.yml
+* Default: null
+
+ansible_pre_tasks: <str>
+* Comma-separated list of paths or URLs to custom Ansible playbooks to run BEFORE Splunk sets up using the provided site.yml
+* Default: null
+
+hide_password: <bool>
+* Boolean that determines whether or not to output Splunk admin passwords through Ansible
+* Default: false
+
+retry_num: <int>
+* Number of retries to make for potentially flakey/error-prone tasks
+* Default: 50
+
+shc_bootstrap_delay: <int>
+* Number of seconds of delay when verifying SHC success on the deployer
+* Default: 30
+
+splunk_home_ownership_enforcement: true
+* Boolean that to control and enable UAC on $SPLUNK_HOME (recommended to be enabled)
+* Default: true
+
+config:
+  baked: <str>
+  * Configuration filename
+  * Default: default.yml
+
+  defaults_dir: <str - filepath>
+  * Location on filesystem where the default.yml can be found
+  * Default: /tmp/defaults
+
+  env:
+    headers: <str>
+    * Define header information (in necessary) when pulling default.yml from a URL 
+    * Default: null
+
+    var: <str>
+    * Control environment variable name that determines location of default.yml 
+    * Default: SPLUNK_DEFAULTS_URL
+
+    verify: <bool>
+    * Enable/disable SSL validation
+    * Default: true
+  host:
+    headers: <str>
+    * Define header information (in necessary) when pulling default.yml from a URL 
+    * Default: null
+
+    url: <str>
+    * Define URL to pull default.yml from 
+    * Default: null
+
+    verify: <bool>
+    * Enable/disable SSL validation
+    * Default: true
+
+  max_delay: <int>
+  * Maximum duration (in seconds) between attempts to pull the default.yml from a remote source
+  * Default: 60
+
+  max_retries: <int>
+  * Maximum attempts to pull the default.yml from a remote source
+  * Default: 3
+
+  max_timeout: <int>
+  * Maximum timeout for attempts to pull the default.yml from a remote source
+  * Default: 1200
+
+splunkbase_username: <str>
+* Used for authentication when downloading apps from https://splunkbase.splunk.com/ (this is NOT required to even be specified, unless you have SplunkBase apps defined in your splunk.apps_location)
+* NOTE: Use this in combination with splunkbase_password. You will also need to run Ansible using the dynamic inventory script (environ.py) for this to register and work properly.
+* Default: null
+
+splunkbase_password: <str>
+* Used for authentication when downloading apps from https://splunkbase.splunk.com/ (this is NOT required to even be specified, unless you have SplunkBase apps defined in your splunk.apps_location)
+* NOTE: Use this in combination with splunkbase_username. You will also need to run Ansible using the dynamic inventory script (environ.py) for this to register and work properly.
+* Default: null
+
+splunk:
+  role: <str>
+  * Role to assume when setting up Splunk
+  * Default: splunk_standalone
+
+  upgrade: <bool>
+  * Determines whether or not to perform an upgrade (to the splunk.build_location)
+  * Default: false
+
+  build_location: <str>
+  * Splunk build location, either on the filesystem or a remote URL
+  * Default: /tmp/splunk.tgz
+
+  build_remote_src: <bool>
+  * Boolean to determine whether the installer is local (false) or remote (true)
+  * Default: true
+
+  license_master_included: <bool>
+  * Boolean to determine whether there exists a separate license master 
+  * Default: false
+  
+  preferred_captaincy: <bool>
+  * Boolean to determine whether splunk should set a preferred captain.  This can have an effect on day 2 operations if the search heads need to be restarted 
+  * Default: true
+
+  apps_location: <list>
+  * List of apps to install - elements can be in the form of a URL or a location in the filessytem
+  * Default: null
+
+  license_uri: <str>
+  * Path or remote URL to a valid Splunk license
+  * Default: null
+
+  ignore_license: <bool>
+  * Allow proceeding with a bad/invalid Splunk license
+  * Default: false
+
+  license_download_dest: <str - filepath>
+  * Path in filesystem where licenses will be downloaded as
+  * Default: /tmp/splunk.lic
+
+  nfr_license: <str - filepath>
+  * Path in filesystem where of special NFR licenses
+  * Default: /tmp/nfr_enterprise.lic
+
+  wildcard_license: <bool>
+  * Enable licenses to be interpreted as fileglobs, to support provisioning with multiple Splunk licenses
+  * Default: false
+
+  admin_user: <str>
+  * Default admin-level user to run provisioning commands under
+  * Default: admin
+
+  password: <str>
+  * Default Splunk admin user password. This is REQUIRED when starting Splunk
+  * Default: null
+
+  user: <str>
+  * Host user under which Splunk will run
+  * Default: splunk
+
+  group: <str>
+  * Host group under which Splunk will run
+  * Default: splunk
+
+  enable_service: <bool>
+  * Determine whether or not to enable Splunk for boot-start (start via sysinitv or systemd, etc.)
+  * Default: false
+
+  opt: <str - filepath>
+  * Path in filesystem where Splunk will be installed
+  * Default: /opt
+
+  home: <str - filepath>
+  * Path in filesystem where SPLUNK_HOME is located
+  * Default: /opt/splunk
+
+  exec: <str - filepath>
+  * Path in filesystem where splunk binary exists (this will depend on splunk.home)
+  * Default: /opt/splunk/bin/splunk
+
+  pid: <str - filepath>
+  * Path in filesystem of splunk PID file (this will depend on splunk.home)
+  * Default: /opt/splunk/var/run/splunk/splunkd.pid
+
+  app_paths:
+    default: <str - filepath>
+    * Path in filesystem of default apps (this will depend on splunk.home)
+    * Default: /opt/splunk/etc/apps
+
+    deployment: <str - filepath>
+    * Path in filesystem of deployment apps (this will depend on splunk.home)
+    * Default: /opt/splunk/etc/deployment-apps
+
+    httpinput: <str - filepath>
+    * Path in filesystem of the HTTP input apps (this will depend on splunk.home)
+    * Default: /opt/splunk/etc/apps/splunk_httpinput
+
+    idxc: <str - filepath>
+    * Path in filesystem of indexer cluster master apps (this will depend on splunk.home)
+    * Default: /opt/splunk/etc/master-apps
+
+    shc: <str - filepath>
+    * Path in filesystem of search head cluster apps (this will depend on splunk.home)
+    * Default: /opt/splunk/etc/shcluster/apps
+
+  hec_disabled: <int|bool>
+  * Determine whether or not to disable setting up the HTTP event collector (HEC)
+  * Default: 0
+
+  hec_enableSSL: <int|bool>
+  * Determine whether or not to enable SSL on the HTTP event collector (HEC) endpoint
+  * Default: 1
+
+  hec_port: <int>
+  * Determine the port used for the HTTP event collector (HEC) endpoint
+  * Default: 8088
+
+  hec_token: <str>
+  * Determine a token to use for the HTTP event collector (HEC) endpoint
+  * Default: null
+
+  http_enableSSL: <int|bool>
+  * Determine whether or not to enable SSL on SplunkWeb
+  * Default: 0
+
+  http_enableSSL_cert: <str>
+  * Path in filesystem to SplunkWeb SSL certificate
+  * Default: null
+
+  http_enableSSL_privKey: <str>
+  * Path in filesystem to SplunkWeb SSL private key
+  * Default: null
+
+  http_enableSSL_privKey_password: <str>
+  * Password used to setup SplunkWeb SSL private key
+  * Default: null
+
+  http_port: <int>
+  * Determine the port used for SplunkWeb
+  * Default: 8000
+
+  s2s_enable: <int|bool>
+  * Determine whether or not to enable Splunk-to-Splunk communication. This is REQUIRED for any distributed topologies.
+  * Default: true
+
+  s2s_port: <int>
+  * Determine the port used for Splunk-to-Splunk communication
+  * Default: 9997
+
+  svc_port: <int>
+  * Determine the port used for Splunk management/remote API calls
+  * Default: 8089
+
+  search_head_cluster_url: null
+  * URL of the Splunk search head cluster
+  * Default: null
+
+  secret: null
+  * Secret passcode used to encrypt all of Splunk's sensitive information on disk. When not set, Splunk will autogenerate a unique secret local to each installation. This is NOT required for any standalone or distributed Splunk topology
+  * NOTE: This may be set once at the start of provisioning any deployment. Any changes made to this splunk.secret after the deployment has been created must be resolved manually, otherwise there is a severe risk of bricking the capabilities of your Splunk environment.
+  * Default: null
+
+  idxc:
+    enable: <bool>
+    * Enable indexer clustering
+    * Default: false
+
+    label: <str>
+    * Provide a label for indexer clustering configuration
+    * Default: idxc_label
+
+    replication_factor: <int>
+    * Determine knowledge object replication factor
+    * Default: 3
+
+    replication_port: <int>
+    * Determine the port used for replication of artifacts
+    * Default: 9887
+
+    search_factor: <int>
+    * Determine the search factor used by indexer clustering
+    * Default: 3
+
+    secret: <str>
+    * Determine the secret used to configure indexer clustering. This is REQUIRED when setting up indexer clustering
+    * Default: null
+
+  shc:
+    enable: <bool>
+    * Enable search head clustering
+    * Default: false
+
+    label: <str>
+    * Provide a label for search head clustering configuration
+    * Default: shc_label
+
+    replication_factor: <int>
+    * Determine knowledge object replication factor
+    * Default: 3
+
+    replication_port: <int>
+    * Determine the port used for replication of artifacts
+    * Default: 9887
+
+    secret: <str>
+    * Determine the secret used to configure search head clustering. This is REQUIRED when setting up search head clustering
+    * Default: null
+
+  dfs:
+    enable: <bool>
+    * Enable Data Fabric Search (DFS)
+    * Default: false
+
+    port: <int>
+    * Identifies the port on which the DFSMaster Java process runs.
+    * Default: 9000
+
+    dfc_num_slots: <int>
+    * Maximum number of concurrent DFS searches that run on each search head
+    * Default: 4
+    
+    dfw_num_slots: <int>
+    * Maximum number of concurrent DFS searches that run on a search head cluster
+    * Default: 10
+    
+    dfw_num_slots_enabled: <bool>
+    * Enables you to set the value of the field dfw_num_slots.
+    * Default: false
+
+    spark_master_host: <str>
+    * This setting identifies the Spark master.
+    * Default: 127.0.0.1
+
+    spark_master_webui_port: <int>
+    * Identifies the port for the Spark master web UI.
+    * Default: 8080
+
+  smartstore: <dict>
+  * Nested dict obj to enable automatic SmartStore provisioning
+  * Default: null
+
+  tar_dir: <str>
+  * Name of directory for the Splunk tar
+  * Default: splunk
+  
+  conf: <dict>
+    (filename):
+      directory: <str - filepath>
+      * Path in filesystem to create `.conf` file
+      * Default: /opt/splunk/etc/system/local
+
+      content: <dict>
+        (section name): <dict>
+          (name) : (value)
+            * Key-value pairs in configuration file
+

+ 1 - 0
4.images/splunk-dist/README.md

@@ -0,0 +1 @@
+Download the Splunk rpm file from their website and store in this directory as `splunk.rpm`

+ 0 - 0
4.images/splunk-dist/version_is_8.0.0


+ 12 - 0
5.standalone/backend.tf

@@ -0,0 +1,12 @@
+# This is where terraform stores state, which allows for multiple
+# developers on teh same terraform configuration.
+terraform {
+  backend "s3" {
+    # This must match the name from the bootstrap directory
+    bucket = "ftd-splunk-standalone-terraform-state"
+    key    = "splunk/standalone/terraform.tfstate"
+    region = "us-east-2"
+    encrypt = true
+    kms_key_id = "alias/splunk-standalone-terraform"
+  }
+}

+ 4 - 0
5.standalone/caller.tf

@@ -0,0 +1,4 @@
+# callers are used to get information about the current
+# aws account and region.
+data "aws_caller_identity" "current" {}
+data "aws_region" "current" {}

+ 3 - 0
5.standalone/provider.tf

@@ -0,0 +1,3 @@
+provider "aws" {
+  region = "us-east-2"
+}

+ 13 - 0
5.standalone/remote_state_network.tf

@@ -0,0 +1,13 @@
+data "terraform_remote_state" "network" {
+  backend = "s3"
+
+  config = {
+    # This must match the name from the bootstrap directory
+    bucket = "ftd-splunk-standalone-terraform-state"
+    # This must match the folder from the 2.network backend directory
+    key    = "splunk/network/terraform.tfstate"
+    region = "us-east-2"
+    encrypt = true
+    kms_key_id = "alias/splunk-standalone-terraform"
+  }
+}

+ 99 - 0
5.standalone/standalone.tf

@@ -0,0 +1,99 @@
+data "aws_ami" "standalone_ami" {
+  most_recent = true
+
+  filter {
+    name = "name"
+    values = ["FTD-Splunk-Standalone"]
+  }
+
+  owners = ["${data.aws_caller_identity.current.account_id}"]
+}
+
+resource "aws_instance" "splunk_standalone" {
+  ami = "${data.aws_ami.standalone_ami.id}"
+  instance_type = "i3.large"
+  key_name = "Fred-IO"
+  
+  vpc_security_group_ids = ["${aws_security_group.splunk_standalone.id}"]
+  subnet_id = "${data.terraform_remote_state.network.outputs.subnet0_id}"
+
+  associate_public_ip_address = true
+  iam_instance_profile = "Splunk-EC2-Standalone"
+
+  ebs_optimized = true
+
+  root_block_device {
+    volume_type = "gp2"
+    volume_size = "10"
+    encrypted = true
+    kms_key_id = "alias/splunk_standalone_ebs"
+  }
+  ebs_block_device {
+    device_name = "/dev/xvdb"
+    volume_type = "gp2"
+    volume_size = 20
+    delete_on_termination = true
+    encrypted = true
+    kms_key_id = "alias/splunk_standalone_ebs"
+  }
+  ebs_block_device {
+    device_name = "/dev/xvdc"
+    volume_type = "gp2"
+    volume_size = 2
+    delete_on_termination = true
+    encrypted = true
+    kms_key_id = "alias/splunk_standalone_ebs"
+  }
+
+  tags = {
+    Name = "Splunk Standalone"
+  }
+}
+
+resource "aws_security_group" "splunk_standalone" {
+  name = "splunk_standalone"
+  description = "Basic Splunk Ports"
+  vpc_id = "${data.terraform_remote_state.network.outputs.vpc_id}"
+
+  ingress {
+    from_port = 22
+    to_port = 22
+    protocol = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+    description = "SSH from any"
+  }
+  ingress {
+    from_port = 80
+    to_port = 80
+    protocol = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+    description = "HTTP from any"
+  }
+  ingress {
+    from_port = 443
+    to_port = 443
+    protocol = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+    description = "HTTPS from any"
+  }
+  ingress {
+    from_port = 8000
+    to_port = 8000
+    protocol = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+    description = "Splunk from any"
+  }
+
+  egress {
+    from_port = 0
+    to_port = 0
+    protocol = -1
+    cidr_blocks = ["0.0.0.0/0"]
+    description = "To Any"
+  }
+}
+
+output "standalone_public_ip" {
+  value = aws_instance.splunk_standalone.public_ip
+}
+

+ 14 - 0
README.md

@@ -0,0 +1,14 @@
+# Freds AWS Standalone with SmartStore
+
+Tools and methods to set up and maintain a standalone splunk instance with SmartStore
+
+## Features
+* SmartStore for cheap storage and stateless indexes.
+* Packer AMI for Easy Updates and Quick Scaling
+
+## Requirements
+* Packer (>= 1.4.4)
+* Terraform (>= v0.12.9)
+* An AWS account
+* Credentials set up in ~/.aws
+