.gitlab-ci.yml

+image: gitlab.rc.uab.edu:4567/rc/packer-openstack-hpc-image:latest
+
+variables:
+  TARGET_ENV: "dev"
+  ANSIBLE_REMOTE_TMP: "/tmp"
+  OS_REGION_NAME: "bhm1"
+  OS_INTERFACE: "public"
+  OS_IDENTITY_API_VERSION: "3"
+  OS_AUTH_TYPE: "v3applicationcredential"
+  OS_AUTH_URL: "https://keystone.cloud.rc.uab.edu:5000/v3"
+  TF_ROOT: ${CI_PROJECT_DIR}/
+  TF_VAR_external_network: "xdmod-packer-dmznet"
+  GIT_SUBMODULE_STRATEGY: "recursive"
+  GIT_SUBMODULE_UPDATE_FLAGS: "--force"
+
+cache:
+  paths:
+    - .terraform
+    - .terraform.lock.hcl
+    - terraform.tfstate
+
+stages:          # List of stages for jobs, and their order of execution
+  - validate
+  - build
+  - deploy
+  - cleanup
+
+prep-deploy:
+  stage: .pre
+  tags: [build]
+  environment:
+    name: $TARGET_ENV
+  script:
+    - export BUILD_DATE=$(TZ=America/Chicago date +%Y-%m-%dT%H%M%S)
+    - |
+      if [ "$TARGET_ENV" = "dev" ]; then
+           TF_VAR_data_volume="xdmod-dev-volume-$BUILD_DATE"
+           openstack volume create --snapshot xdmod-dev-volume-snap --size 20 $TF_VAR_data_volume
+      elif [ "$TARGET_ENV" = "staging" ]; then
+           TF_VAR_data_volume="xdmod-staging-volume-$BUILD_DATE"
+           openstack volume create --snapshot xdmod-staging-volume-snap --size 150 $TF_VAR_data_volume
+      elif [ "$TARGET_ENV" = "production" ]; then
+           echo $TARGET_ENV
+      else
+           echo "Unknown TARGET_ENV: $TARGET_ENV"
+           exit 1
+      fi
+      echo "TF_VAR_data_volume=${TF_VAR_data_volume}" | tee -a $CI_PROJECT_DIR/vars.env
+      echo "TF_VAR_flavor=${TF_VAR_flavor}" | tee -a $CI_PROJECT_DIR/vars.env
+      echo "BUILD_DATE=${BUILD_DATE}" | tee -a $CI_PROJECT_DIR/vars.env
+  artifacts:
+    reports:
+      dotenv: vars.env
+
+tf-validate:
+  stage: validate
+  tags: [build]
+  environment:
+    name: $TARGET_ENV
+  script:
+    - cd CRI_XCBC && git checkout dev && cd ..
+    - terraform --version
+    - terraform init
+    - terraform validate
+    - pwd
+    - terraform plan
+    - 'sed -i -E "s/(cod_deploy: ).*/\1false/" CRI_XCBC/group_vars/all'
+
+tf-apply:
+  cache:
+    paths:
+      - .terraform/
+  stage: build
+  tags: [build]
+  environment:
+    name: $TARGET_ENV
+  script:
+    - echo "TF_VAR_data_volume=${TF_VAR_data_volume}"
+    - echo "TF_VAR_flavor=${TF_VAR_flavor}"
+    - export TF_VAR_internal_network="xdmod-packer-clusternet-$BUILD_DATE"
+    - |
+      if [ -z $TF_VAR_image_ohpc ];
+      then
+      XDMOD_IMAGES=($(openstack image list --sort-column Name --sort-descending -f value -c Name -c ID | grep -P " xdmod-${TARGET_ENV}-\d{14}$" | awk '{print $2}'))
+      export TF_VAR_image_ohpc="${XDMOD_IMAGES[0]}"
+      echo $TF_VAR_image_ohpc
+      else
+      echo "image_ohpc defined as ${TF_VAR_image_ohpc}"
+      fi
+    - export TF_VAR_keypair_name="os-gen-keypair-$BUILD_DATE"
+    - |
+      terraform apply -auto-approve
+      if [ $? -ne 0 ]; then
+        echo "Terraform apply failed. Running terraform destroy..."
+        terraform destroy -auto-approve
+        TF_SUCCESS=false
+      else
+        TF_SUCCESS=true
+      fi
+      echo "Terraform success: $TF_SUCCESS"
+    - export instance_floating_ip=$(terraform output -raw floating_ip_ohpc)
+    - export instance_id=$(terraform output -raw xdmod_instance_id)
+    - echo "instance_floating_ip=${instance_floating_ip}" | tee -a $CI_PROJECT_DIR/vars.env
+    - echo "instance_id=${instance_id}" | tee -a $CI_PROJECT_DIR/vars.env
+  artifacts:
+    reports:
+      dotenv: vars.env
+
+staging-post-deploy:
+  environment:
+    name: staging
+  stage: build
+  tags: [build]
+  script:
+    - export old_instance_id=$(openstack floating ip show $staging_floating_ip -c port_details -f value | awk -F', ' '{print $2}' | awk -F"'" '{print $2}')
+    - echo "staging_floating_ip=${staging_floating_ip}"
+    - echo "old_instance_id=${old_instance_id}"
+    - |
+      if [[ -n "$old_instance_id" ]]; then
+        openstack server remove floating ip $old_instance_id $staging_floating_ip
+        openstack server delete $old_instance_id
+      fi
+    - openstack server remove floating ip $instance_id $instance_floating_ip
+    - openstack server add floating ip $instance_id $staging_floating_ip
+    - openstack floating ip delete $instance_floating_ip
+  needs:
+    - job: tf-apply
+      artifacts: true
+  rules:
+    - if: '$TARGET_ENV == "staging"'
+
+production-deploy:
+  environment:
+    name: production
+  stage: deploy
+  tags: [build]
+  script:
+    - export prod_instance_id=$(openstack floating ip show $prod_floating_ip -c port_details -f value | awk -F', ' '{print $2}' | awk -F"'" '{print $2}')
+    - export prod_volume=$(openstack server show $prod_instance_id -c volumes_attached -f value | awk -F"'" '{print $4}')
+    - export prod_queue_volume=$(openstack server show $prod_instance_id -c volumes_attached -f value | awk -F"'" '{print $4}' | awk 'NR==2')
+
+    - |
+      eval $(ssh-agent -s)
+      chmod 400 "${SSH_PRIV_KEY}"
+      ssh-add "${SSH_PRIV_KEY}"
+      mkdir ~/.ssh
+      chmod 700 ~/.ssh
+    - ssh -o StrictHostKeyChecking=no $SSH_USERNAME@$prod_floating_ip sudo systemctl stop mariadb
+    - ssh -o StrictHostKeyChecking=no $SSH_USERNAME@$prod_floating_ip sudo umount /var/lib/mysql && df -h
+     - ssh -o StrictHostKeyChecking=no $SSH_USERNAME@$prod_floating_ip sudo umount /data/xdmod/queues && df -h
+    - openstack server remove volume $prod_instance_id $prod_volume
+    - openstack server remove volume $prod_instance_id $prod_queue_volume
+    - openstack server stop $prod_instance_id
+
+    # Exporting Staging Volume IDs
+    - export staging_volume=$(openstack server show $instance_id -c volumes_attached -f value | awk -F"'" '{print $4}' | awk 'NR==1')
+    - export staging_queue_volume=$(openstack server show $instance_id -c volumes_attached -f value | awk -F"'" '{print $4}' | awk 'NR==2')
+
+    # Logging Information
+    - echo "Detaching staging_volume_id=${staging_volume} and staging_queue_volume_id=${staging_queue_volume} from staging_instance_id=${instance_id}"
+
+    # Stopping MariaDB Service on Staging Instance
+    - ssh -o StrictHostKeyChecking=no $SSH_USERNAME@$staging_floating_ip sudo systemctl stop mariadb
+
+    # Unmounting MySQL Data Directories
+    - ssh -o StrictHostKeyChecking=no $SSH_USERNAME@$staging_floating_ip sudo umount /var/lib/mysql && df -h
+    - ssh -o StrictHostKeyChecking=no $SSH_USERNAME@$staging_floating_ip sudo umount /data/xdmod/queues && df -h
+
+    # Removing Volumes from Server
+    - openstack server remove volume $instance_id $staging_volume
+    - openstack server remove volume $instance_id $staging_queue_volume
+
+    - echo "Attach prod volume $xdmod-prod-vol-$BUILD_DATE to staging instance $instance_id"
+    - echo "Attach prod queue volume xdmod-prod-queue-vol-$BUILD_DATE to staging instance $instance_id"
+    - openstack volume snapshot create --volume $prod_volume xdmod-prod-data-vol-snapshot-$BUILD_DATE
+    - openstack volume snapshot create --volume $prod_queue_volume xdmod-prod-queue-vol-snapshot-$BUILD_DATE
+    - openstack volume create --snapshot xdmod-prod-data-vol-snapshot-$BUILD_DATE --size 150 xdmod-prod-vol-$BUILD_DATE
+    - openstack volume create --snapshot xdmod-prod-queue-vol-snapshot-$BUILD_DATE --size 1 xdmod-prod-queue-vol-$BUILD_DATE
+    - openstack server add volume $instance_id xdmod-prod-vol-$BUILD_DATE --device /dev/vdb
+    - openstack server add volume $instance_id xdmod-prod-queue-vol-$BUILD_DATE --device /dev/vdc
+
+    - echo "swap the floating IP between prod and staging instances"
+    - openstack server remove floating ip $prod_instance_id $prod_floating_ip
+    - openstack server remove floating ip $instance_id $staging_floating_ip
+    - openstack server add floating ip $instance_id $prod_floating_ip
+
+    - ssh -o StrictHostKeyChecking=no $SSH_USERNAME@$prod_floating_ip sudo mount /dev/vdb1 /var/lib/mysql
+    - ssh -o StrictHostKeyChecking=no $SSH_USERNAME@$prod_floating_ip sudo systemctl start mariadb
+
+  when: manual
+  rules:
+    - if: '$TARGET_ENV != "dev"'
+
+clean-up:
+  stage: cleanup
+  tags: [build]
+  environment:
+    name: $TARGET_ENV
+  script:
+    - terraform destroy -auto-approve
+  when: manual

.gitmodules

@@ -2,3 +2,7 @@
 	path = CRI_XCBC
 	url = https://github.com/jprorama/CRI_XCBC.git
 	branch = feat-openstack
+	
+[submodule "CRI_Cluster_Monitor"]
+	path = CRI_Cluster_Monitor
+	url = https://github.com/eesaanatluri/CRI_Cluster_Monitor.git

CRI_Cluster_Monitor @ 14479ffd

+Subproject commit 14479ffde246b1526b7059cc7638b9b183de9f0a

CRI_XCBC @ 0d1a0653

-Subproject commit 1d7a787a562130d8b96d1b85d087c4a3ecc34f41
+Subproject commit 0d1a06538ef0e6bea7940a426bc5bd2fe628591c

external-network/main.tf

+terraform {
+required_version = ">= 0.14.0"
+  required_providers {
+    openstack = {
+      source  = "terraform-provider-openstack/openstack"
+      version = "~> 1.42.0"
+    }
+  }
+}
+
 # is created as a datasource this module and called in root module
-variable "public_network_name" {type = "string"}
+variable "public_network_name" {type = string}
 
 variable "name" {default = "dmz"}
 variable "admin_state_up" {}
 variable "enable_dhcp" {}
 
-
-
 data "openstack_networking_network_v2" "public_network" {name = var.public_network_name}
 
 # creates dmznet
@@ -20,7 +28,7 @@ resource "openstack_networking_subnet_v2" "external_subnet" {
   network_id      = openstack_networking_network_v2.external_network.id
   cidr            = "192.168.100.0/24"
   ip_version      = 4
-  dns_nameservers = ["8.8.8.8"]
+  dns_nameservers = ["172.20.0.137", "172.20.0.3", "8.8.8.8"]
   enable_dhcp     = var.enable_dhcp
 }
 
@@ -36,8 +44,9 @@ resource "openstack_networking_router_interface_v2" "router" {
   subnet_id = openstack_networking_subnet_v2.external_subnet.id
 }
 
-output "external_network_id" {
-    value = data.openstack_networking_network_v2.public_network.id
+output "id" {
+    value = openstack_networking_network_v2.external_network.id
+    depends_on = [openstack_networking_subnet_v2.external_subnet]
 }
 
 output "external_subnet_id" {
@@ -46,4 +55,4 @@ output "external_subnet_id" {
 
 output "router_id" {
     value = openstack_networking_router_v2.router.id
-}
\ No newline at end of file
+}

floating-ip/floating-main.tf

 variable "public_network_name" {}
 
+terraform {
+required_version = ">= 0.14.0"
+  required_providers {
+    openstack = {
+      source  = "terraform-provider-openstack/openstack"
+      version = "~> 1.42.0"
+    }
+  }
+}
+
+provider "openstack" {
+  use_octavia         = true
+  endpoint_overrides = {
+      "network"  = "https://neutron-api.cloud.rc.uab.edu:9696/v2.0/"
+    }
+}
+
 
 # defines where floating ip will come from using variable public_network_name defined in root module
 resource "openstack_networking_floatingip_v2" "ohpc_ip" {

internal-network/main.tf

-variable "name" {default = "cluster"}
+variable "internal_net" {default = ""}
 variable "admin_state_up" { }
 variable "enable_dhcp" {}
 
+terraform {
+required_version = ">= 0.14.0"
+  required_providers {
+    openstack = {
+      source  = "terraform-provider-openstack/openstack"
+      version = "~> 1.42.0"
+    }
+  }
+}
+
+provider "openstack" {
+  use_octavia         = true
+  endpoint_overrides = {
+      "network"  = "https://neutron-api.cloud.rc.uab.edu:9696/v2.0/"
+    }
+}
+
 # creates clusternet
 resource "openstack_networking_network_v2" "internal_network" {
-  name           = "${var.name}net"
+  name           = var.internal_net
   admin_state_up = var.admin_state_up
 }
 
 # creates clustersubnet
 # cidr is the subnet range (that subnet range and dns nameservers from the network create file in feat-openstack)
 resource "openstack_networking_subnet_v2" "internal_subnet" {
-  name        = "${var.name}subnet"
+  name        = "${var.internal_net}_subnet"
   network_id  = openstack_networking_network_v2.internal_network.id
   cidr        = "10.1.1.0/24"
   ip_version  = 4
@@ -19,9 +36,10 @@ resource "openstack_networking_subnet_v2" "internal_subnet" {
 }
 
 output "internal_network_id" {
-  value = openstack_networking_network_v2.internal_network.id
+    value = openstack_networking_network_v2.internal_network.id
+    depends_on = [openstack_networking_subnet_v2.internal_subnet]
 }
 
 output "internal_subnet_id" {
     value = openstack_networking_subnet_v2.internal_subnet.id
-}
\ No newline at end of file
+}

key-pair/main.tf

 variable "keypair_name" {}
 variable "ssh_public_key" {}
 
+terraform {
+required_version = ">= 0.14.0"
+  required_providers {
+    openstack = {
+      source  = "terraform-provider-openstack/openstack"
+      version = "~> 1.42.0"
+    }
+  }
+}
+
+provider "openstack" {
+  use_octavia         = true
+  endpoint_overrides = {
+      "network"  = "https://neutron-api.cloud.rc.uab.edu:9696/v2.0/"
+    }
+}
+
 resource "openstack_compute_keypair_v2" "keypair" {
   name       = var.keypair_name
-  public_key = file(var.ssh_public_key)
+  public_key = var.ssh_public_key
 }
 
 output "keypair_name" {
@@ -11,5 +28,5 @@ output "keypair_name" {
 }
 
 output "public_key" {
-    value = file(var.ssh_public_key)
+    value = var.ssh_public_key
 }

main.tf

-# runs the external-network module
-module "dmz-network" {
-    source = "./external-network"
-    # Default name var is in the module main file
-    admin_state_up = var.admin_state_up
-    enable_dhcp = var.enable_dhcp
-    public_network_name = var.public_network_name
-}
-# calls the outputs defined in the external-network module
-output "external_network_id" {
-    value = "${module.dmz-network.external_network_id}"
+terraform {
+required_version = ">= 0.14.0"
+  required_providers {
+    openstack = {
+      source  = "terraform-provider-openstack/openstack"
+      version = "~> 1.42.0"
+    }
+  }
 }
 
-output "router_id" {
-    value = "${module.dmz-network.router_id}"
+provider "openstack" {
+  endpoint_overrides = {
+      "network"  = "https://neutron-api.cloud.rc.uab.edu:9696/v2.0/"
+    }
 }
 
 # runs the internal-network module
 module "cluster-network" {
+    internal_net = var.internal_network
     source = "./internal-network"
     # Default name var is in the module main file
     admin_state_up = var.admin_state_up
@@ -35,12 +35,12 @@ module "floating-ip-address" {
 
 # calls the outputs defined in the floating-ip module
 output "floating_ip_ohpc" {
-    value = "${module.floating-ip-address.ohpc_address}"
+    value = module.floating-ip-address.ohpc_address
 }
 
-output "floating_ip_ood" {
-    value = "${module.floating-ip-address.ood_address}"
-}
+#output "floating_ip_ood" {
+#    value = module.floating-ip-address.ood_address
+#}
 
 # runs the key-pair module - imports local public key into openstack and give it the name defined above in the variables
 module "import-keypair" {
@@ -51,42 +51,49 @@ module "import-keypair" {
 
 # calls the outputs defined in the key-pair module
 output "keypair_name" {
-    value = "${module.import-keypair.keypair_name}"
+    value = module.import-keypair.keypair_name
 }
 
+data "openstack_networking_network_v2" "external_net" {name = var.external_network}
+data "openstack_blockstorage_volume_v3" "disk" {name = var.data_volume}
+data "openstack_blockstorage_volume_v3" "disk2" {name = var.xdmod_queue_data_volume}
+
 # runs the ohpc-instance module - creates ohpc instance using variables defined above
 # calls functions from dmz-network, import-keypair, and floating-ip-address modules to get values created there for use
 module "create-ohpc-instance" {
-    external_subnet_id = "${module.dmz-network.external_subnet_id}"
     source = "./ohpc-instance"
     ohpc_instance_name = var.ohpc_instance_name
     image_ohpc = var.image_ohpc
     flavor = var.flavor
-    key_pair = "${module.import-keypair.keypair_name}"
-    external_network = var.external_network
-    internal_network = var.internal_network
-    floating_ip_ohpc = "${module.floating-ip-address.ohpc_address}"
-    host_prefix = var.host_prefix
+    key_pair = module.import-keypair.keypair_name
+    external_network = data.openstack_networking_network_v2.external_net.id
+    internal_network = "${module.cluster-network.internal_network_id}"
+    internal_ip = var.ohpc_private_ip
+    floating_ip_ohpc = module.floating-ip-address.ohpc_address
     ohpc_user =  var.ohpc_user
     ssh_private_key = var.ssh_private_key
+    vol_ids = [
+      data.openstack_blockstorage_volume_v3.disk.id,
+      data.openstack_blockstorage_volume_v3.disk2.id,
+    ]
 }
 
 # runs the ood-instance module - creates ood instance using variables defined above
 # calls functions from cluster-network, import-keypair, and floating-ip-address modules to get values created there for use
-module "create-ood-instance" {
-    internal_subnet_id = "${module.cluster-network.internal_subnet_id}"
-    source = "./ood-instance"
-    ood_instance_name = var.ood_instance_name
-    image_ood = var.image_ood
-    flavor = var.flavor
-    key_pair = "${module.import-keypair.keypair_name}"
-    internal_network = var.internal_network
-    external_network = var.external_network
-    floating_ip_ood = "${module.floating-ip-address.ood_address}"
-    host_prefix = var.host_prefix
-    ood_user = var.ood_user
-    ssh_private_key = var.ssh_private_key
-}
+#module "create-ood-instance" {
+#    internal_subnet_id = module.cluster-network.internal_subnet_id
+#    source = "./ood-instance"
+#    ood_instance_name = var.ood_instance_name
+#    image_ood = var.image_ood
+#    flavor = var.flavor
+#    key_pair = module.import-keypair.keypair_name
+#    internal_network = var.internal_network
+#    external_network = var.external_network
+#    floating_ip_ood = module.floating-ip-address.ood_address
+#    host_prefix = var.host_prefix
+#    ood_user = var.ood_user
+#    ssh_private_key = var.ssh_private_key
+#}
 
 # runs the nodes module - creates nodes using variables defined above
 # calls functions from cluster-network and import-keypair modules to get values created there for use
@@ -95,17 +102,77 @@ module "nodes" {
     source = "./nodes"
     image_compute = var.image_compute
     flavor = var.flavor
-    key_pair = "${module.import-keypair.keypair_name}"
+    key_pair = module.import-keypair.keypair_name
     compute_node_count = var.compute_node_count
-    internal_network = var.internal_network
+    internal_network = "${module.cluster-network.internal_network_id}"
 }
 
 # calls the outputs defined in the ohpc-instance module
 output "ohpc-ssh_host" {
-    value = "${module.create-ohpc-instance.ssh_host}"
+    value = module.create-ohpc-instance.ssh_host
+}
+
+output "xdmod_instance_id" {
+    value = module.create-ohpc-instance.xdmod_instance_id
 }
 
 # calls the outputs defined in the ood-instance module
-output "ood-ssh_host" {
-    value = "${module.create-ood-instance.ssh_host}"
-}
\ No newline at end of file
+#output "ood-ssh_host" {
+#    value = module.create-ood-instance.ssh_host
+#}
+
+# compute node and ood post provision
+# use single null_resource for serial provisioner runs to avoid race conditions
+# that lead to inconsistent deploy successes.
+resource "null_resource" "ops" {
+  triggers = {
+    ohpc_instance = module.create-ohpc-instance.xdmod_instance_id
+    compute_instances = join(",", module.nodes.id)
+  }
+
+  connection {
+    host        = module.create-ohpc-instance.ssh_host
+    user        = var.ohpc_user
+    private_key = var.ssh_private_key
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "ls -al /CRI_XCBC",
+    ]
+  }
+  # moves CRI_XCBC file into directory made above
+
+provisioner "remote-exec" {
+    inline = [
+      "sudo sed -i -E 's/xdmod..nip.io/xdmod.rc.uab.edu/g' /etc/httpd/conf.d/xdmod.conf",
+      "sudo sed -i -E 's/xdmod..nip.io/xdmod.rc.uab.edu/g' /etc/xdmod/simplesamlphp/config/config.php",
+      "sudo systemctl restart httpd",        
+   ]
+  }
+
+provisioner "remote-exec" {
+    inline = [
+      "sudo mount ${module.create-ohpc-instance.device}1 /var/lib/mysql",
+      "sudo mount ${module.create-ohpc-instance.device}2 /data/xdmod/queues",
+      "sudo df -h",
+      "sudo systemctl restart mariadb",
+    ]
+  }
+
+  # compute node registration on ohpc
+  provisioner "remote-exec" {
+    inline = [
+     for node, net in module.nodes.network:
+    "ansible-playbook -c local -i /CRI_XCBC/hosts -l `hostname -s` -e  \"{'cod_deploy':'false', 'compute_nodes':[{'name':'${node}', 'ip':'${net[0].fixed_ip_v4}', 'mac':'${net[0].mac}', 'vnfs':'', 'sockets':'1', 'corespersocket':'1'}]}\" /CRI_XCBC/site-ops.yaml -b -v"
+    ]
+  }
+
+  # ood node
+#  provisioner "remote-exec" {
+#    inline = [
+#      for net in module.create-ood-instance.network:
+#    "ansible-playbook -c local -i /CRI_XCBC/hosts -l `hostname -s` -e \"{'compute_nodes':[{'name':'${var.ood_instance_name}', 'ip':'${net.fixed_ip_v4}', 'mac':'${net.mac}', 'vnfs':'', 'sockets':'1', 'corespersocket':'1'}]}\" /CRI_XCBC/site-ops.yaml -b -v"]
+#  }
+
+}

nodes/main.tf

 # is created in internal-network module and called in root module
-variable "internal_subnet_id" {type = "string"}
+variable "internal_subnet_id" {type = string}
 
 variable "image_compute" {}
 variable "flavor" {}
 
 # is created in key-pair module and called in root module
-variable "key_pair" {type = "string"}
+variable "key_pair" {type = string}
 
 variable "compute_node_count" { }
 variable "internal_network" {}
 
+terraform {
+required_version = ">= 0.14.0"
+  required_providers {
+    openstack = {
+      source  = "terraform-provider-openstack/openstack"
+      version = "~> 1.42.0"
+    }
+  }
+}
+
+provider "openstack" {
+  use_octavia         = true
+  endpoint_overrides = {
+      "network"  = "https://neutron-api.cloud.rc.uab.edu:9696/v2.0/"
+    }
+}
 
+data "openstack_images_image_v2" "compute" {
+  name = var.image_compute
+  most_recent = true
+}
 
 # creates compute node
-resource "openstack_compute_instance_v2" "c0" {
+resource "openstack_compute_instance_v2" "compute" {
   depends_on      = [var.internal_subnet_id]
   name            = "c${count.index}"
-  image_name      = var.image_compute
+  image_id        = data.openstack_images_image_v2.compute.id
   flavor_name     = var.flavor
   key_pair        = var.key_pair
   security_groups = ["default"]
   count           = var.compute_node_count
+  user_data =  <<-EOF
+    #cloud-config
+    runcmd:
+     - [ bash, -xc, 'ethernet=$(cat /sys/class/net/eth0/address); nodename=$(hostname -s); sed -e "s/MY_HWADDR/$ethernet/" -e "s/MY_NODENAME/$nodename/" -i /warewulf/config;' ]
+     - [ bash, -xc, "echo $(date) ': hello world!'; until WWGETFILES_INTERVAL=0 bash -x /warewulf/bin/wwgetfiles; do echo waiting ; rm -f /tmp/.wwgetfile.lock ; sleep 10; done;" ]
+     - [ bash, -xc, "systemctl restart munge" ]
+     - [ bash, -xc, "systemctl restart slurmd" ]
+    EOF
 
 # defines the networks of the instance
   network {
-    name = var.internal_network
+    uuid = var.internal_network
   }
-}
\ No newline at end of file
+}
+
+# output
+output "id" {
+  value = openstack_compute_instance_v2.compute.*.id
+}
+
+output "image_id" {
+  value = data.openstack_images_image_v2.compute.id
+}
+
+output "network" {
+  value = zipmap(openstack_compute_instance_v2.compute.*.name, openstack_compute_instance_v2.compute.*.network)
+}

ohpc-instance/main.tf

 # is created in external-network module and called in root module
-variable "external_subnet_id" {type = "string"}
+terraform {
+required_version = ">= 0.14.0"
+  required_providers {
+    openstack = {
+      source  = "terraform-provider-openstack/openstack"
+      version = "~> 1.42.0"
+    }
+  }
+}
+
 
 variable "ohpc_instance_name" {}
 variable "image_ohpc" {}
 variable "flavor" {}
 
 # is created in key-pair module and called in root module
-variable "key_pair" {type = "string"}
+variable "key_pair" {type = string}
 
 variable "internal_network" {}
+variable "internal_ip" {}
 variable "external_network" {}
 
 # is created in floating-ip module and called in root module
-variable "floating_ip_ohpc" {type = "string"}
+variable "floating_ip_ohpc" {type = string}
 
-variable "host_prefix" {}
 variable "ohpc_user" {}
 variable "ssh_private_key" {}
-
+variable "vol_ids" {
+  type    = list(string)
+  default = []
+}
 
 # creates details for the OHPC instance
 resource "openstack_compute_instance_v2" "ohpc" {
-  depends_on      = [var.external_subnet_id]
   name            = var.ohpc_instance_name
   image_name      = var.image_ohpc
   flavor_name     = var.flavor
   key_pair        = var.key_pair
   security_groups = ["default"]
+  user_data = <<-EOF
+     #cloud-config
+     write_files:
+     - content: |
+         10.1.1.10 ohpc ohpc.novalocal
+       owner: centos:centos
+       path: /etc/hosts
+       permissions: 0644
+     EOF
 
 # defines the networks of the instance
   network {
-    name = var.external_network
+    uuid = var.external_network
   }
   network {
-    name = var.internal_network
+    uuid = var.internal_network
+    fixed_ip_v4 = var.internal_ip
   }
 }
 
@@ -41,15 +62,23 @@ resource "openstack_compute_instance_v2" "ohpc" {
 resource "openstack_compute_floatingip_associate_v2" "ohpc" {
   floating_ip = var.floating_ip_ohpc
   instance_id = openstack_compute_instance_v2.ohpc.id
+}
 
-# defines ssh connection
-  connection {
-    host = format(var.host_prefix,element(split(".", var.floating_ip_ohpc),3,),)
-    user        = var.ohpc_user
-    private_key = file(var.ssh_private_key)
-  }
+# Attach a volume
+resource "openstack_compute_volume_attach_v2" "volume_attach" {
+  count       = length(var.vol_ids)
+  instance_id = openstack_compute_instance_v2.ohpc.id
+  volume_id   = var.vol_ids[count.index]
+}
+
+output "xdmod_instance_id" {
+    value = openstack_compute_instance_v2.ohpc.id
 }
 
 output "ssh_host" {
-    value = format(var.host_prefix,element(split(".", var.floating_ip_ohpc),3,),)
-}
\ No newline at end of file
+    value = var.floating_ip_ohpc
+}
+
+output "device" {
+    value = openstack_compute_volume_attach_v2.volume_attach[count.index].device
+}

ood-instance/main.tf

 # is created in internal-network module and called in root module
-variable "internal_subnet_id" {type = "string"}
+variable "internal_subnet_id" {type = string}
 
 variable "ood_instance_name" {}
 variable "image_ood" {}
 variable "flavor" {}
 
 # is created in key-pair module and called in root module
-variable "key_pair" {type = "string"}
+variable "key_pair" {type = string}
 
 variable "internal_network" {}
+variable "internal_ip" {}
 variable "external_network" {}
 
 # is created in floating-ip module and called in root module
-variable "floating_ip_ood" {type = "string"}
+variable "floating_ip_ood" {type = string}
 
 variable "host_prefix" {}
 variable "ood_user" {}
 variable "ssh_private_key" {}
 
+terraform {
+required_version = ">= 0.14.0"
+  required_providers {
+    openstack = {
+      source  = "terraform-provider-openstack/openstack"
+      version = "~> 1.42.0"
+    }
+  }
+}
+
+provider "openstack" {
+  use_octavia         = true
+  endpoint_overrides = {
+      "network"  = "https://neutron-api.cloud.rc.uab.edu:9696/v2.0/"
+    }
+}
 
 # creates details for the OOD instance
 resource "openstack_compute_instance_v2" "ood" {
@@ -27,6 +44,15 @@ resource "openstack_compute_instance_v2" "ood" {
   flavor_name     = var.flavor
   key_pair        = var.key_pair
   security_groups = ["default"]
+  user_data =  <<-EOF
+    #cloud-config
+    runcmd:
+     - [ bash, -xc, 'ethernet=$(cat /sys/class/net/eth1/address); nodename=$(hostname -s); sed -e "s/MY_HWADDR/$ethernet/" -e "s/MY_NODENAME/$nodename/" -i /warewulf/config;' ]
+     - [ bash, -xc, "until WWGETFILES_INTERVAL=0 bash -x /warewulf/bin/wwgetfiles; do echo waiting ; rm -f /tmp/.wwgetfile.lock ; sleep 10; done;" ]
+     - [ bash, -xc, "systemctl restart munge" ]
+     - [ bash, -xc, "cd /CRI_XCBC; ansible-playbook -c local -i /CRI_XCBC/hosts -l `hostname -s` /CRI_XCBC/ood-ops.yaml -b -v" ]
+    EOF
+
 
 # defines the networks of the instance
   network {
@@ -34,6 +60,7 @@ resource "openstack_compute_instance_v2" "ood" {
   }
   network {
     name = var.internal_network
+    fixed_ip_v4 = var.internal_ip
   }
 }
 
@@ -41,17 +68,15 @@ resource "openstack_compute_instance_v2" "ood" {
 resource "openstack_compute_floatingip_associate_v2" "ood" {
   floating_ip = var.floating_ip_ood
   instance_id = openstack_compute_instance_v2.ood.id
-
-# defines ssh connection
-  connection {
-    host = format(var.host-prefix,element(split(".", var.floating_ip_ood),3,),)
-    user        = var.ood_user
-    private_key = file(var.ssh_private_key)
-  }
 }
 
 
-
+output "id" {
+    value = openstack_compute_instance_v2.ood.id
+}
 output "ssh_host"{
     value = format(var.host_prefix,element(split(".", var.floating_ip_ood),3,),)
-}
\ No newline at end of file
+}
+output "network" {
+    value = openstack_compute_instance_v2.ood.network
+}

vars.tf

@@ -6,7 +6,7 @@ variable "admin_state_up" {default = true}
 variable "enable_dhcp" {default = true}
 
 # variable for floating-ip - also used in exrernal network creation
-variable "public_network_name" {default = "bright-external-flat-externalnet"}
+variable "public_network_name" {default = "uab-campus"}
  
 # variables for keypair module
 variable "keypair_name" {default = "os-gen-keypair"}
@@ -14,18 +14,20 @@ variable "ssh_public_key" {default = "~/.ssh/id_rsa.pub"}
  
 # variables for instance modules
 variable "ohpc_instance_name" {default = "ohpc"}
+variable "ohpc_private_ip" {default = "10.1.1.10"}
 variable "ood_instance_name" { default = "ood"}
-variable "image_ohpc" {default = "CentOS-7-x86_64-GenericCloud-1905"}
-variable "image_ood" {default = "CentOS-7-x86_64-GenericCloud-1905"}
-variable "flavor" {default = "m1.medium"}
+variable "image_ohpc" {default = "xdmod-20230614063844"}
+variable "image_ood" {default = ""}
+variable "ood_private_ip" {default = "10.1.1.11"}
+variable "flavor" {default = "m1.xlarge"}
 variable "internal_network" {default = "clusternet"}
 variable "external_network" {default = "dmznet"}
-variable "host_prefix" {default = "164.111.161.%s"}
 variable "ohpc_user" {default = "centos"}
 variable "ood_user" {default = "centos"}
 variable "ssh_private_key" {default = "~/.ssh/id_rsa"}
-
+variable "data_volume" {default = ""}
+variable "xdmod_queue_data_volume" {default = "xdmod_queue_data_volume"}
 
 # variables for node creation module
-variable "image_compute" {default = "CentOS-7-x86_64-GenericCloud-1905"}
-variable "compute_node_count" {default = 2}
+variable "image_compute" {default = "compute-v1"}
+variable "compute_node_count" {default = 1}