diff --git a/bootstrap-cluster.sh b/bootstrap-cluster.sh index 801448b9..c48e0866 100755 --- a/bootstrap-cluster.sh +++ b/bootstrap-cluster.sh @@ -1,7 +1,7 @@ #!/bin/bash set -euo pipefail -KEY="$HOME/.ssh/simplyblock-ohio.pem" +KEY="~/.ssh/simplyblock-xata.pem" print_help() { echo "Usage: $0 [options]" @@ -173,31 +173,31 @@ while [[ $# -gt 0 ]]; do shift done -SECRET_VALUE=$(terraform output -raw secret_value) +#SECRET_VALUE=$(terraform output -raw secret_value) KEY_NAME=$(terraform output -raw key_name) BASTION_IP=$(terraform output -raw bastion_public_ip) GRAFANA_ENDPOINT=$(terraform output -raw grafana_invoke_url) -ssh_dir="$HOME/.ssh" - -if [ ! -d "$ssh_dir" ]; then - mkdir -p "$ssh_dir" - echo "Directory $ssh_dir created." -else - echo "Directory $ssh_dir already exists." -fi - -if [[ -n "$SECRET_VALUE" ]]; then - KEY="$HOME/.ssh/$KEY_NAME" - if [ -f "$HOME/.ssh/$KEY_NAME" ]; then - echo "the ssh key: ${KEY} already exits on local" - else - echo "$SECRET_VALUE" >"$KEY" - chmod 400 "$KEY" - fi -else - echo "Failed to retrieve secret value. Falling back to default key." -fi +# ssh_dir="$HOME/.ssh" + +# if [ ! -d "$ssh_dir" ]; then +# mkdir -p "$ssh_dir" +# echo "Directory $ssh_dir created." +# else +# echo "Directory $ssh_dir already exists." +# fi + +# if [[ -n "$SECRET_VALUE" ]]; then +# KEY="$HOME/.ssh/$KEY_NAME" +# if [ -f "$HOME/.ssh/$KEY_NAME" ]; then +# echo "the ssh key: ${KEY} already exits on local" +# else +# echo "$SECRET_VALUE" >"$KEY" +# chmod 400 "$KEY" +# fi +# else +# echo "Failed to retrieve secret value. Falling back to default key." +# fi mnodes=$(terraform output -raw mgmt_private_ips) echo "mgmt_private_ips: ${mnodes}" @@ -316,7 +316,7 @@ for ((i = 1; i < ${#mnodes[@]}; i++)); do -o ProxyCommand="ssh -o StrictHostKeyChecking=no -i \"$KEY\" -W %h:%p ec2-user@${BASTION_IP}" \ ec2-user@${mnodes[${i}]} " MANGEMENT_NODE_IP=${mnodes[0]} - ${SBCLI_CMD} mgmt add \${MANGEMENT_NODE_IP} ${CLUSTER_ID} eth0 + ${SBCLI_CMD} mgmt add \${MANGEMENT_NODE_IP} ${CLUSTER_ID} ${CLUSTER_SECRET} eth0 " done @@ -391,7 +391,7 @@ else -o ProxyCommand="ssh -o StrictHostKeyChecking=no -i \"$KEY\" -W %h:%p ec2-user@${BASTION_IP}" \ ec2-user@${mnodes[0]} " MANGEMENT_NODE_IP=${mnodes[0]} - ${SBCLI_CMD} cluster activate ${CLUSTER_ID} + ${SBCLI_CMD} -d cluster activate ${CLUSTER_ID} " fi diff --git a/bootstrap-k3s.sh b/bootstrap-k3s.sh index 01b89bd1..b18be688 100755 --- a/bootstrap-k3s.sh +++ b/bootstrap-k3s.sh @@ -1,6 +1,6 @@ #!/bin/bash -KEY="$HOME/.ssh/simplyblock-ohio.pem" +KEY="~/.ssh/simplyblock-xata.pem" print_help() { echo "Usage: $0 [options]" @@ -29,29 +29,29 @@ while [[ $# -gt 0 ]]; do shift done -SECRET_VALUE=$(terraform output -raw secret_value) +#SECRET_VALUE=$(terraform output -raw secret_value) KEY_NAME=$(terraform output -raw key_name) -ssh_dir="$HOME/.ssh" - -if [ ! -d "$ssh_dir" ]; then - mkdir -p "$ssh_dir" - echo "Directory $ssh_dir created." -else - echo "Directory $ssh_dir already exists." -fi - -if [[ -n "$SECRET_VALUE" ]]; then - KEY="$HOME/.ssh/$KEY_NAME" - if [ -f "$HOME/.ssh/$KEY_NAME" ]; then - echo "the ssh key: ${KEY} already exits on local" - else - echo "$SECRET_VALUE" >"$KEY" - chmod 400 "$KEY" - fi -else - echo "Failed to retrieve secret value. Falling back to default key." -fi +# ssh_dir="$HOME/.ssh" + +# if [ ! -d "$ssh_dir" ]; then +# mkdir -p "$ssh_dir" +# echo "Directory $ssh_dir created." +# else +# echo "Directory $ssh_dir already exists." +# fi + +# if [[ -n "$SECRET_VALUE" ]]; then +# KEY="$HOME/.ssh/$KEY_NAME" +# if [ -f "$HOME/.ssh/$KEY_NAME" ]; then +# echo "the ssh key: ${KEY} already exits on local" +# else +# echo "$SECRET_VALUE" >"$KEY" +# chmod 400 "$KEY" +# fi +# else +# echo "Failed to retrieve secret value. Falling back to default key." +# fi BASTION_IP=$(terraform output -raw bastion_public_ip) mnodes=($(terraform output -raw extra_nodes_public_ips)) diff --git a/create_cluster.sh b/create_cluster.sh new file mode 100644 index 00000000..5a29efa2 --- /dev/null +++ b/create_cluster.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# CHANGE THE NAMESPACE NAME! +namespace="change_me" +sbcli_cmd="sbcli-dev" +CONTACT_POINT= + + +export TFSTATE_BUCKET=xata-simplyblock-staging-infra +export TFSTATE_KEY=staging/controlplane +export TFSTATE_REGION=us-east-2 + +terraform init -reconfigure \ + -backend-config="bucket=${TFSTATE_BUCKET}" \ + -backend-config="key=${TFSTATE_KEY}" \ + -backend-config="region=${TFSTATE_REGION}" \ + -backend-config="encrypt=true" + +### switch to workspace +terraform workspace select -or-create "$namespace" + +# terraform apply -var mgmt_nodes=1 -var storage_nodes=0 -var extra_nodes=0 --auto-approve + +# Specifying the instance types to use +terraform apply -var mgmt_nodes=1 -var storage_nodes=4 -var extra_nodes=0 -var "storage_nodes_arch=arm64" \ + -var mgmt_nodes_instance_type="m6i.xlarge" -var storage_nodes_instance_type="c6gd.2xlarge" \ + -var extra_nodes_instance_type="m6i.large" -var sbcli_cmd="$sbcli_cmd" \ + -var volumes_per_storage_nodes=0 --auto-approve + +# Save terraform output to a file +terraform output -json > tf_outputs.json + +# The boostrap-cluster.sh creates the KEY in `.ssh` directory in the home directory + +chmod +x ./bootstrap-cluster.sh +# specifying cluster argument to use +./bootstrap-cluster.sh --sbcli-cmd "$sbcli_cmd" --disable-ha-jm \ + --distr-ndcs 2 --distr-npcs 1 --cap-crit 99 --cap-warn 94 --prov-cap-crit 500 \ + --prov-cap-warn 200 --distr-bs 4096 --distr-chunk-bs 4096 \ + --spdk-debug --max-lvol 200 --max-snap 200 --max-prov 10T --number-of-devices 1 \ + --partitions 1 --log-del-interval 300m --metrics-retention-period 2h \ + --number-of-distribs 2 \ + --contact-point $CONTACT_POINT diff --git a/data.tf b/data.tf index 8cebbe3e..b9820a09 100644 --- a/data.tf +++ b/data.tf @@ -2,6 +2,6 @@ data "aws_availability_zones" "available" { state = "available" } -data "aws_secretsmanager_secret_version" "simply" { - secret_id = local.selected_key_name -} +# data "aws_secretsmanager_secret_version" "simply" { +# secret_id = local.selected_key_name +# } diff --git a/locals.tf b/locals.tf index 38c2aa8f..c2e54219 100644 --- a/locals.tf +++ b/locals.tf @@ -12,7 +12,7 @@ locals { key_name = { "us-east-1" = "simplyblock-us-east-1.pem" - "us-east-2" = "simplyblock-us-east-2.pem" + "us-east-2" = "simplyblock-xata.pem" "eu-north-1" = "simplyblock-eu-north-1.pem" "eu-west-1" = "simplyblock-eu-west-1.pem" } @@ -27,14 +27,14 @@ locals { # $sudo yum install -y yum-utils xorg-x11-xauth nvme-cli fio region_ami_map = { "us-east-1" = "ami-0ff9547ee3e11637a" - "us-east-2" = "ami-00b0bb86a4287f38f" + "us-east-2" = "ami-0aa8fc2422063977a" "eu-north-1" = "ami-01997ffb7707167a4" "eu-west-1" = "ami-0a3bac9371ffc12f8" } region_ami_map_arm = { "us-east-1" = "ami-0990e7074b32986af" - "us-east-2" = "ami-0e71db082192a9cf7" + "us-east-2" = "ami-08f9f3bb075432791" "eu-north-1" = "ami-006af066a79f5190f" "eu-west-1" = "ami-06028a225ee106d6f" } diff --git a/main.tf b/main.tf index fd80eaf3..d340da8b 100644 --- a/main.tf +++ b/main.tf @@ -3,11 +3,11 @@ module "vpc" { source = "terraform-aws-modules/vpc/aws" name = "${terraform.workspace}-storage-vpc-sb" - cidr = "10.0.0.0/16" + cidr = "10.0.8.0/21" azs = [data.aws_availability_zones.available.names[0], data.aws_availability_zones.available.names[1], ] - private_subnets = ["10.0.1.0/24", "10.0.3.0/24"] - public_subnets = ["10.0.2.0/24", "10.0.4.0/24"] + private_subnets = ["10.0.9.0/24", "10.0.11.0/24"] + public_subnets = ["10.0.10.0/24", "10.0.12.0/24"] map_public_ip_on_launch = true enable_nat_gateway = true @@ -682,7 +682,7 @@ resource "aws_instance" "storage_nodes" { subnet_id = module.vpc.private_subnets[local.az_index] iam_instance_profile = aws_iam_instance_profile.inst_profile.name root_block_device { - volume_size = 45 + volume_size = 80 } tags = { Name = "${terraform.workspace}-storage-${each.value + 1}" diff --git a/outputs.tf b/outputs.tf index 472eee8d..a668865b 100644 --- a/outputs.tf +++ b/outputs.tf @@ -22,10 +22,10 @@ output "key_name" { value = local.selected_key_name } -output "secret_value" { - sensitive = true - value = data.aws_secretsmanager_secret_version.simply.secret_string -} +# output "secret_value" { +# sensitive = true +# value = data.aws_secretsmanager_secret_version.simply.secret_string +# } output "mgmt_node_details" { value = { for i, instance in aws_instance.mgmt_nodes : diff --git a/test/create_cluster_ebs.sh b/test/create_cluster_ebs.sh new file mode 100644 index 00000000..e5baa2f2 --- /dev/null +++ b/test/create_cluster_ebs.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# CHANGE THE NAMESPACE NAME! +namespace="changeme" +sbcli_cmd="sbcli-dev" + + +export TFSTATE_BUCKET=xata-simplyblock-staging-infra +export TFSTATE_KEY=staging/controlplane +export TFSTATE_REGION=us-east-2 + +terraform init -reconfigure \ + -backend-config="bucket=${TFSTATE_BUCKET}" \ + -backend-config="key=${TFSTATE_KEY}" \ + -backend-config="region=${TFSTATE_REGION}" \ + -backend-config="encrypt=true" + +# uncomment if you need to destroy existing cluster with the same name +# terraform destroy --auto-approve + +### switch to workspace +terraform workspace select -or-create "$namespace" + +# terraform apply -var mgmt_nodes=1 -var storage_nodes=0 -var extra_nodes=0 --auto-approve + +# Specifying the instance types to use +terraform apply -var mgmt_nodes=1 -var storage_nodes=3 -var extra_nodes=0 \ + -var mgmt_nodes_instance_type="m6i.xlarge" -var storage_nodes_instance_type="m6i.2xlarge" \ + -var extra_nodes_instance_type="m6i.large" -var sbcli_cmd="$sbcli_cmd" \ + -var volumes_per_storage_nodes=3 -var storage_nodes_ebs_size2=100 --auto-approve + +# Save terraform output to a file +terraform output -json > tf_outputs.json + +# The boostrap-cluster.sh creates the KEY in `.ssh` directory in the home directory + +chmod +x ./bootstrap-cluster.sh +# specifying cluster argument to use +./bootstrap-cluster.sh --sbcli-cmd "$sbcli_cmd" --spdk-debug \ + --max-lvol 10 --max-snap 10 --max-prov 1200G \ + --number-of-devices 3 --log-del-interval 900m --metrics-retention-period 2h \ + --distr-ndcs 2 --distr-npcs 1 --distr-bs 4096 --distr-chunk-bs 4096 --partitions 0 diff --git a/tf_outputs.json b/tf_outputs.json new file mode 100644 index 00000000..e2b66125 --- /dev/null +++ b/tf_outputs.json @@ -0,0 +1,160 @@ +{ + "api_invoke_url": { + "sensitive": false, + "type": "string", + "value": "https://w98m1cxy4a.execute-api.us-east-2.amazonaws.com/" + }, + "bastion_public_ip": { + "sensitive": false, + "type": "string", + "value": "18.219.59.202" + }, + "extra_nodes_private_ips": { + "sensitive": false, + "type": "string", + "value": "10.0.4.183" + }, + "extra_nodes_public_ips": { + "sensitive": false, + "type": "string", + "value": "18.116.64.159" + }, + "grafana_invoke_url": { + "sensitive": false, + "type": "string", + "value": "https://w98m1cxy4a.execute-api.us-east-2.amazonaws.com/grafana" + }, + "graylog_invoke_url": { + "sensitive": false, + "type": "string", + "value": "https://w98m1cxy4a.execute-api.us-east-2.amazonaws.com/graylog" + }, + "key_name": { + "sensitive": false, + "type": "string", + "value": "simplyblock-us-east-2.pem" + }, + "mgmt_node_details": { + "sensitive": false, + "type": [ + "object", + { + "demo-k8s-mgmt-1": [ + "object", + { + "private_ip": "string", + "public_ip": "string", + "type": "string" + } + ] + } + ], + "value": { + "demo-k8s-mgmt-1": { + "private_ip": "10.0.3.132", + "public_ip": "", + "type": "m6i.xlarge" + } + } + }, + "mgmt_private_ips": { + "sensitive": false, + "type": "string", + "value": "10.0.3.132" + }, + "mgmt_public_ips": { + "sensitive": false, + "type": "string", + "value": "" + }, + "secret_value": { + "sensitive": true, + "type": "string", + "value": "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtz\nc2gtZWQyNTUxOQAAACA3Vn/Aq5mZiP9gsWHIROz4SfTqMIZxvlbTJusvgjLMowAA\nAIjs64g17OuINQAAAAtzc2gtZWQyNTUxOQAAACA3Vn/Aq5mZiP9gsWHIROz4SfTq\nMIZxvlbTJusvgjLMowAAAEAwUQIBATAFBgMrZXAEIgQgy3b2RGthAnDiTucFNF4s\npzdWf8CrmZmI/2CxYchE7PhJ9OowhnG+VtMm6y+CMsyjAAAAAAECAwQF\n-----END OPENSSH PRIVATE KEY-----" + }, + "storage_node_details": { + "sensitive": false, + "type": [ + "object", + { + "demo-k8s-storage-1": [ + "object", + { + "availability_zone": "string", + "private_ip": "string", + "public_ip": "string", + "type": "string" + } + ], + "demo-k8s-storage-2": [ + "object", + { + "availability_zone": "string", + "private_ip": "string", + "public_ip": "string", + "type": "string" + } + ], + "demo-k8s-storage-3": [ + "object", + { + "availability_zone": "string", + "private_ip": "string", + "public_ip": "string", + "type": "string" + } + ], + "demo-k8s-storage-4": [ + "object", + { + "availability_zone": "string", + "private_ip": "string", + "public_ip": "string", + "type": "string" + } + ] + } + ], + "value": { + "demo-k8s-storage-1": { + "availability_zone": "us-east-2b", + "private_ip": "10.0.3.239", + "public_ip": "", + "type": "m6gd.2xlarge" + }, + "demo-k8s-storage-2": { + "availability_zone": "us-east-2b", + "private_ip": "10.0.3.206", + "public_ip": "", + "type": "m6gd.2xlarge" + }, + "demo-k8s-storage-3": { + "availability_zone": "us-east-2b", + "private_ip": "10.0.3.80", + "public_ip": "", + "type": "m6gd.2xlarge" + }, + "demo-k8s-storage-4": { + "availability_zone": "us-east-2b", + "private_ip": "10.0.3.219", + "public_ip": "", + "type": "m6gd.2xlarge" + } + } + }, + "storage_private_ips": { + "sensitive": false, + "type": "string", + "value": "10.0.3.239 10.0.3.206 10.0.3.80 10.0.3.219" + }, + "storage_public_ips": { + "sensitive": false, + "type": "string", + "value": " " + }, + "tfengine_logs": { + "sensitive": false, + "type": "string", + "value": "simplyblock-tfengine-logs-f27c05713a" + } +} diff --git a/tfengine.tf b/tfengine.tf index a19bf1ba..cf6aaa2b 100644 --- a/tfengine.tf +++ b/tfengine.tf @@ -15,29 +15,29 @@ data "aws_ami" "this" { } } -resource "aws_autoscaling_group" "tfengine_asg" { - min_size = 1 - max_size = 1 - desired_capacity = 1 - vpc_zone_identifier = [module.vpc.private_subnets[0]] - tag { - key = "Name" - value = "${terraform.workspace}-tfengine" - propagate_at_launch = true - } - tag { - key = "long-term-test" - value = "true" - propagate_at_launch = true - } - lifecycle { - create_before_destroy = true - } - launch_template { - id = aws_launch_template.tfengine_lc.id - version = "$Latest" - } -} +# resource "aws_autoscaling_group" "tfengine_asg" { +# min_size = 1 +# max_size = 1 +# desired_capacity = 1 +# vpc_zone_identifier = [module.vpc.private_subnets[0]] +# tag { +# key = "Name" +# value = "${terraform.workspace}-tfengine" +# propagate_at_launch = true +# } +# tag { +# key = "long-term-test" +# value = "true" +# propagate_at_launch = true +# } +# lifecycle { +# create_before_destroy = true +# } +# launch_template { +# id = aws_launch_template.tfengine_lc.id +# version = "$Latest" +# } +# } resource "aws_launch_template" "tfengine_lc" { name_prefix = "tfengine" @@ -208,10 +208,10 @@ resource "aws_iam_role_policy_attachment" "AmazonSSMManagedInstanceCore" { } # NOTE: Terraform uses the same role that we use to deploy the cluster to the customer's account -resource "aws_iam_role_policy_attachment" "sbdeployPolicy" { - policy_arn = "arn:aws:iam::${local.account_id}:policy/sbdeployPolicy" - role = aws_iam_role.tfengine.name -} +# resource "aws_iam_role_policy_attachment" "sbdeployPolicy" { +# policy_arn = "arn:aws:iam::${local.account_id}:policy/sbdeployPolicy" +# role = aws_iam_role.tfengine.name +# } # attach policy resource "aws_iam_role_policy_attachment" "s3policy" { diff --git a/variables.tf b/variables.tf index 65d5afb6..9eea67b3 100644 --- a/variables.tf +++ b/variables.tf @@ -15,7 +15,7 @@ variable "az" { } variable "env" { - default = "dev" + default = "staging" type = string }