main.tf

terraform {
required_version = ">= 0.14.0"
  required_providers {
    openstack = {
      source  = "terraform-provider-openstack/openstack"
      version = "~> 1.42.0"
    }
  }
}

provider "openstack" {
  endpoint_overrides = {
      "network"  = "https://neutron-api.cloud.rc.uab.edu:9696/v2.0/"
    }
}

# runs the internal-network module
module "cluster-network" {
    internal_net = var.internal_network
    source = "./internal-network"
    # Default name var is in the module main file
    admin_state_up = var.admin_state_up
    enable_dhcp = var.enable_dhcp
}
# calls the outputs defined in the internal-network module
output "internal_network_id" {
    value = "${module.cluster-network.internal_network_id}"
}

# runs the floating-ip module - uses public network name defined above
module "floating-ip-address" {
    source = "./floating-ip"
    public_network_name = var.public_network_name
}

# calls the outputs defined in the floating-ip module
output "floating_ip_ohpc" {
    value = module.floating-ip-address.ohpc_address
}

#output "floating_ip_ood" {
#    value = module.floating-ip-address.ood_address
#}

# runs the key-pair module - imports local public key into openstack and give it the name defined above in the variables
module "import-keypair" {
    source = "./key-pair"
    keypair_name = var.keypair_name
    ssh_public_key = var.ssh_public_key
}

# calls the outputs defined in the key-pair module
output "keypair_name" {
    value = module.import-keypair.keypair_name
}

data "openstack_networking_network_v2" "external_net" {name = var.external_network}
data "openstack_blockstorage_volume_v3" "disk" {name = var.data_volume}
data "openstack_blockstorage_volume_v3" "disk2" {name = var.xdmod_queue_data_volume}

# runs the ohpc-instance module - creates ohpc instance using variables defined above
# calls functions from dmz-network, import-keypair, and floating-ip-address modules to get values created there for use
module "create-ohpc-instance" {
    source = "./ohpc-instance"
    ohpc_instance_name = var.ohpc_instance_name
    image_ohpc = var.image_ohpc
    flavor = var.flavor
    key_pair = module.import-keypair.keypair_name
    external_network = data.openstack_networking_network_v2.external_net.id
    internal_network = "${module.cluster-network.internal_network_id}"
    internal_ip = var.ohpc_private_ip
    floating_ip_ohpc = module.floating-ip-address.ohpc_address
    ohpc_user =  var.ohpc_user
    ssh_private_key = var.ssh_private_key
    vol_ids = [
      data.openstack_blockstorage_volume_v3.disk.id,
      data.openstack_blockstorage_volume_v3.disk2.id,
    ]
}

# runs the ood-instance module - creates ood instance using variables defined above
# calls functions from cluster-network, import-keypair, and floating-ip-address modules to get values created there for use
#module "create-ood-instance" {
#    internal_subnet_id = module.cluster-network.internal_subnet_id
#    source = "./ood-instance"
#    ood_instance_name = var.ood_instance_name
#    image_ood = var.image_ood
#    flavor = var.flavor
#    key_pair = module.import-keypair.keypair_name
#    internal_network = var.internal_network
#    external_network = var.external_network
#    floating_ip_ood = module.floating-ip-address.ood_address
#    host_prefix = var.host_prefix
#    ood_user = var.ood_user
#    ssh_private_key = var.ssh_private_key
#}

# runs the nodes module - creates nodes using variables defined above
# calls functions from cluster-network and import-keypair modules to get values created there for use
module "nodes" {
    internal_subnet_id = "${module.cluster-network.internal_subnet_id}"
    source = "./nodes"
    image_compute = var.image_compute
    flavor = var.flavor
    key_pair = module.import-keypair.keypair_name
    compute_node_count = var.compute_node_count
    internal_network = "${module.cluster-network.internal_network_id}"
}

# calls the outputs defined in the ohpc-instance module
output "ohpc-ssh_host" {
    value = module.create-ohpc-instance.ssh_host
}

output "xdmod_instance_id" {
    value = module.create-ohpc-instance.xdmod_instance_id
}

# calls the outputs defined in the ood-instance module
#output "ood-ssh_host" {
#    value = module.create-ood-instance.ssh_host
#}

# compute node and ood post provision
# use single null_resource for serial provisioner runs to avoid race conditions
# that lead to inconsistent deploy successes.
resource "null_resource" "ops" {
  triggers = {
    ohpc_instance = module.create-ohpc-instance.xdmod_instance_id
    compute_instances = join(",", module.nodes.id)
  }

  connection {
    host        = module.create-ohpc-instance.ssh_host
    user        = var.ohpc_user
    private_key = var.ssh_private_key
  }

  provisioner "remote-exec" {
    inline = [
      "ls -al /CRI_XCBC",
    ]
  }
  # moves CRI_XCBC file into directory made above

provisioner "remote-exec" {
    inline = [
      "sudo sed -i -E 's/xdmod..nip.io/xdmod.rc.uab.edu/g' /etc/httpd/conf.d/xdmod.conf",
      "sudo sed -i -E 's/xdmod..nip.io/xdmod.rc.uab.edu/g' /etc/xdmod/simplesamlphp/config/config.php",
      "sudo systemctl restart httpd",        
   ]
  }

provisioner "remote-exec" {
    inline = [
      "sudo mkdir -p /data/xdmod/queues",
      "sudo mount ${module.create-ohpc-instance.device[0]}1 /var/lib/mysql",
      "sudo mount ${module.create-ohpc-instance.device[1]}1 /data/xdmod/queues",
      "sudo df -h",
      "sudo systemctl restart mariadb",
    ]
  }

  # compute node registration on ohpc
  provisioner "remote-exec" {
    inline = [
     for node, net in module.nodes.network:
    "ansible-playbook -c local -i /CRI_XCBC/hosts -l `hostname -s` -e  \"{'cod_deploy':'false', 'compute_nodes':[{'name':'${node}', 'ip':'${net[0].fixed_ip_v4}', 'mac':'${net[0].mac}', 'vnfs':'', 'sockets':'1', 'corespersocket':'1'}]}\" /CRI_XCBC/site-ops.yaml -b -v"
    ]
  }

  # ood node
#  provisioner "remote-exec" {
#    inline = [
#      for net in module.create-ood-instance.network:
#    "ansible-playbook -c local -i /CRI_XCBC/hosts -l `hostname -s` -e \"{'compute_nodes':[{'name':'${var.ood_instance_name}', 'ip':'${net.fixed_ip_v4}', 'mac':'${net.mac}', 'vnfs':'', 'sockets':'1', 'corespersocket':'1'}]}\" /CRI_XCBC/site-ops.yaml -b -v"]
#  }

}