.gitlab-ci.yml

+image: gitlab.rc.uab.edu:4567/rc/packer-openstack-hpc-image:latest
+
+variables:
+  ANSIBLE_REMOTE_TMP: "/tmp"
+  OS_REGION_NAME: "bhm1"
+  OS_INTERFACE: "public"
+  OS_IDENTITY_API_VERSION: "3"
+  OS_AUTH_TYPE: "v3applicationcredential"
+  OS_AUTH_URL: "https://keystone.cloud.rc.uab.edu:5000/v3"
+  TF_ROOT: ${CI_PROJECT_DIR}/
+  TF_VAR_internal_network: "tf-clusternet"
+  TF_VAR_external_network: "tf-dmznet"
+  GIT_SUBMODULE_STRATEGY: "recursive"
+  GIT_SUBMODULE_UPDATE_FLAGS: "--force"
+
+cache:
+  paths:
+    - .terraform
+    - .terraform.lock.hcl
+    - terraform.tfstate
+
+stages:          # List of stages for jobs, and their order of execution
+  - validate
+  - build
+  - deploy
+  - cleanup
+
+tf-validate:
+  stage: validate
+  tags: [build]
+  script:
+    - cd CRI_XCBC && git checkout dev && cd ..
+    - terraform --version
+    - terraform init
+    - terraform validate
+    - pwd
+    - terraform plan
+    - 'sed -i -E "s/(cod_deploy: ).*/\1false/" CRI_XCBC/group_vars/all'
+
+tf-apply:
+  stage: build
+  tags: [build]
+  script:
+    - export BUILD_DATE=$(TZ=America/Chicago date +%Y%m%d)
+    - export TF_VAR_internal_network="tf-clusternet-$BUILD_DATE"
+    - export TF_VAR_external_network="tf-dmznet-$BUILD_DATE"
+    - XDMOD_IMAGES=($(openstack image list --sort-column Name --sort-descending -f value -c Name -c ID | grep -P ' xdmod-\d{14}$' | awk '{print $2}'))
+    - export TF_VAR_image_ohpc="${XDMOD_IMAGES[1]}"
+    - echo $TF_VAR_image_ohpc
+    - cd CRI_XCBC && git checkout dev && git pull && git status && cd ..
+    - 'sed -i -E "s/(cod_deploy: ).*/\1false/" CRI_XCBC/group_vars/all'
+    - terraform apply -auto-approve || terraform destroy -auto-approve && false
+
+clean-up:
+  stage: cleanup
+  tags: [build]
+  script:
+    - terraform destroy -auto-approve
+  when: manual

.gitmodules

@@ -2,3 +2,7 @@
 	path = CRI_XCBC
 	url = https://github.com/jprorama/CRI_XCBC.git
 	branch = feat-openstack
+	
+[submodule "CRI_Cluster_Monitor"]
+	path = CRI_Cluster_Monitor
+	url = https://github.com/eesaanatluri/CRI_Cluster_Monitor.git

CRI_Cluster_Monitor @ 14479ffd

+Subproject commit 14479ffde246b1526b7059cc7638b9b183de9f0a

CRI_XCBC @ 0d1a0653

-Subproject commit 1d7a787a562130d8b96d1b85d087c4a3ecc34f41
+Subproject commit 0d1a06538ef0e6bea7940a426bc5bd2fe628591c

external-network/main.tf

+terraform {
+required_version = ">= 0.14.0"
+  required_providers {
+    openstack = {
+      source  = "terraform-provider-openstack/openstack"
+      version = "~> 1.42.0"
+    }
+  }
+}
+
 # is created as a datasource this module and called in root module
-variable "public_network_name" {type = "string"}
+variable "public_network_name" {type = string}
 
 variable "name" {default = "dmz"}
 variable "admin_state_up" {}
 variable "enable_dhcp" {}
 
-
-
 data "openstack_networking_network_v2" "public_network" {name = var.public_network_name}
 
 # creates dmznet
@@ -20,7 +28,7 @@ resource "openstack_networking_subnet_v2" "external_subnet" {
   network_id      = openstack_networking_network_v2.external_network.id
   cidr            = "192.168.100.0/24"
   ip_version      = 4
-  dns_nameservers = ["8.8.8.8"]
+  dns_nameservers = ["172.20.0.137", "172.20.0.3", "8.8.8.8"]
   enable_dhcp     = var.enable_dhcp
 }
 
@@ -36,8 +44,9 @@ resource "openstack_networking_router_interface_v2" "router" {
   subnet_id = openstack_networking_subnet_v2.external_subnet.id
 }
 
-output "external_network_id" {
-    value = data.openstack_networking_network_v2.public_network.id
+output "id" {
+    value = openstack_networking_network_v2.external_network.id
+    depends_on = [openstack_networking_subnet_v2.external_subnet]
 }
 
 output "external_subnet_id" {
@@ -46,4 +55,4 @@ output "external_subnet_id" {
 
 output "router_id" {
     value = openstack_networking_router_v2.router.id
-}
\ No newline at end of file
+}

floating-ip/floating-main.tf

 variable "public_network_name" {}
 
+terraform {
+required_version = ">= 0.14.0"
+  required_providers {
+    openstack = {
+      source  = "terraform-provider-openstack/openstack"
+      version = "~> 1.42.0"
+    }
+  }
+}
+
+provider "openstack" {
+  use_octavia         = true
+  endpoint_overrides = {
+      "network"  = "https://neutron-api.cloud.rc.uab.edu:9696/v2.0/"
+    }
+}
+
 
 # defines where floating ip will come from using variable public_network_name defined in root module
 resource "openstack_networking_floatingip_v2" "ohpc_ip" {

internal-network/main.tf

@@ -2,6 +2,23 @@ variable "name" {default = "cluster"}
 variable "admin_state_up" { }
 variable "enable_dhcp" {}
 
+terraform {
+required_version = ">= 0.14.0"
+  required_providers {
+    openstack = {
+      source  = "terraform-provider-openstack/openstack"
+      version = "~> 1.42.0"
+    }
+  }
+}
+
+provider "openstack" {
+  use_octavia         = true
+  endpoint_overrides = {
+      "network"  = "https://neutron-api.cloud.rc.uab.edu:9696/v2.0/"
+    }
+}
+
 # creates clusternet
 resource "openstack_networking_network_v2" "internal_network" {
   name           = "${var.name}net"
@@ -18,10 +35,11 @@ resource "openstack_networking_subnet_v2" "internal_subnet" {
   enable_dhcp = var.enable_dhcp
 }
 
-output "internal_network_id" {
-  value = openstack_networking_network_v2.internal_network.id
+output "id" {
+    value = openstack_networking_network_v2.internal_network.id
+    depends_on = [openstack_networking_subnet_v2.internal_subnet]
 }
 
 output "internal_subnet_id" {
     value = openstack_networking_subnet_v2.internal_subnet.id
-}
\ No newline at end of file
+}

key-pair/main.tf

 variable "keypair_name" {}
 variable "ssh_public_key" {}
 
+terraform {
+required_version = ">= 0.14.0"
+  required_providers {
+    openstack = {
+      source  = "terraform-provider-openstack/openstack"
+      version = "~> 1.42.0"
+    }
+  }
+}
+
+provider "openstack" {
+  use_octavia         = true
+  endpoint_overrides = {
+      "network"  = "https://neutron-api.cloud.rc.uab.edu:9696/v2.0/"
+    }
+}
+
 resource "openstack_compute_keypair_v2" "keypair" {
   name       = var.keypair_name
-  public_key = file(var.ssh_public_key)
+  public_key = var.ssh_public_key
 }
 
 output "keypair_name" {
@@ -11,5 +28,5 @@ output "keypair_name" {
 }
 
 output "public_key" {
-    value = file(var.ssh_public_key)
+    value = var.ssh_public_key
 }

main.tf

+terraform {
+required_version = ">= 0.14.0"
+  required_providers {
+    openstack = {
+      source  = "terraform-provider-openstack/openstack"
+      version = "~> 1.42.0"
+    }
+  }
+}
+
+provider "openstack" {
+  endpoint_overrides = {
+      "network"  = "https://neutron-api.cloud.rc.uab.edu:9696/v2.0/"
+    }
+}
+
 # runs the external-network module
 module "dmz-network" {
     source = "./external-network"
     # Default name var is in the module main file
+    name = var.external_network
     admin_state_up = var.admin_state_up
     enable_dhcp = var.enable_dhcp
     public_network_name = var.public_network_name
 }
 # calls the outputs defined in the external-network module
 output "external_network_id" {
-    value = "${module.dmz-network.external_network_id}"
+    value = module.dmz-network.id
 }
 
 output "router_id" {
-    value = "${module.dmz-network.router_id}"
+    value = module.dmz-network.router_id
 }
 
 # runs the internal-network module
 module "cluster-network" {
     source = "./internal-network"
     # Default name var is in the module main file
+    name = var.internal_network
     admin_state_up = var.admin_state_up
     enable_dhcp = var.enable_dhcp
 }
 # calls the outputs defined in the internal-network module
 output "internal_network_id" {
-    value = "${module.cluster-network.internal_network_id}"
+    value = module.cluster-network.id
 }
 
 # runs the floating-ip module - uses public network name defined above
@@ -35,12 +53,12 @@ module "floating-ip-address" {
 
 # calls the outputs defined in the floating-ip module
 output "floating_ip_ohpc" {
-    value = "${module.floating-ip-address.ohpc_address}"
+    value = module.floating-ip-address.ohpc_address
 }
 
-output "floating_ip_ood" {
-    value = "${module.floating-ip-address.ood_address}"
-}
+#output "floating_ip_ood" {
+#    value = module.floating-ip-address.ood_address
+#}
 
 # runs the key-pair module - imports local public key into openstack and give it the name defined above in the variables
 module "import-keypair" {
@@ -51,61 +69,99 @@ module "import-keypair" {
 
 # calls the outputs defined in the key-pair module
 output "keypair_name" {
-    value = "${module.import-keypair.keypair_name}"
+    value = module.import-keypair.keypair_name
 }
 
 # runs the ohpc-instance module - creates ohpc instance using variables defined above
 # calls functions from dmz-network, import-keypair, and floating-ip-address modules to get values created there for use
 module "create-ohpc-instance" {
-    external_subnet_id = "${module.dmz-network.external_subnet_id}"
+    external_subnet_id = module.dmz-network.external_subnet_id
     source = "./ohpc-instance"
     ohpc_instance_name = var.ohpc_instance_name
     image_ohpc = var.image_ohpc
     flavor = var.flavor
-    key_pair = "${module.import-keypair.keypair_name}"
-    external_network = var.external_network
-    internal_network = var.internal_network
-    floating_ip_ohpc = "${module.floating-ip-address.ohpc_address}"
-    host_prefix = var.host_prefix
+    key_pair = module.import-keypair.keypair_name
+    external_network = module.dmz-network.id
+    internal_network = module.cluster-network.id
+    internal_ip = var.ohpc_private_ip
+    floating_ip_ohpc = module.floating-ip-address.ohpc_address
     ohpc_user =  var.ohpc_user
     ssh_private_key = var.ssh_private_key
 }
 
 # runs the ood-instance module - creates ood instance using variables defined above
 # calls functions from cluster-network, import-keypair, and floating-ip-address modules to get values created there for use
-module "create-ood-instance" {
-    internal_subnet_id = "${module.cluster-network.internal_subnet_id}"
-    source = "./ood-instance"
-    ood_instance_name = var.ood_instance_name
-    image_ood = var.image_ood
-    flavor = var.flavor
-    key_pair = "${module.import-keypair.keypair_name}"
-    internal_network = var.internal_network
-    external_network = var.external_network
-    floating_ip_ood = "${module.floating-ip-address.ood_address}"
-    host_prefix = var.host_prefix
-    ood_user = var.ood_user
-    ssh_private_key = var.ssh_private_key
-}
+#module "create-ood-instance" {
+#    internal_subnet_id = module.cluster-network.internal_subnet_id
+#    source = "./ood-instance"
+#    ood_instance_name = var.ood_instance_name
+#    image_ood = var.image_ood
+#    flavor = var.flavor
+#    key_pair = module.import-keypair.keypair_name
+#    internal_network = var.internal_network
+#    external_network = var.external_network
+#    floating_ip_ood = module.floating-ip-address.ood_address
+#    host_prefix = var.host_prefix
+#    ood_user = var.ood_user
+#    ssh_private_key = var.ssh_private_key
+#}
 
 # runs the nodes module - creates nodes using variables defined above
 # calls functions from cluster-network and import-keypair modules to get values created there for use
 module "nodes" {
-    internal_subnet_id = "${module.cluster-network.internal_subnet_id}"
+    internal_subnet_id = module.cluster-network.internal_subnet_id
     source = "./nodes"
     image_compute = var.image_compute
     flavor = var.flavor
-    key_pair = "${module.import-keypair.keypair_name}"
+    key_pair = module.import-keypair.keypair_name
     compute_node_count = var.compute_node_count
-    internal_network = var.internal_network
+    internal_network = module.cluster-network.id
 }
 
 # calls the outputs defined in the ohpc-instance module
 output "ohpc-ssh_host" {
-    value = "${module.create-ohpc-instance.ssh_host}"
+    value = module.create-ohpc-instance.ssh_host
 }
 
 # calls the outputs defined in the ood-instance module
-output "ood-ssh_host" {
-    value = "${module.create-ood-instance.ssh_host}"
-}
\ No newline at end of file
+#output "ood-ssh_host" {
+#    value = module.create-ood-instance.ssh_host
+#}
+
+# compute node and ood post provision
+# use single null_resource for serial provisioner runs to avoid race conditions
+# that lead to inconsistent deploy successes.
+resource "null_resource" "ops" {
+  triggers = {
+    ohpc_instance = module.create-ohpc-instance.id
+    compute_instances = join(",", module.nodes.id)
+  }
+
+  connection {
+    host        = module.create-ohpc-instance.ssh_host
+    user        = var.ohpc_user
+    private_key = var.ssh_private_key
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "ls -al /CRI_XCBC",
+    ]
+  }
+  # moves CRI_XCBC file into directory made above
+
+  # compute node registration on ohpc
+  provisioner "remote-exec" {
+    inline = [
+      "sed -i -E 's/(ServerName ).*/\1xdmod.${module.create-ohpc-instance.ssh_host}.nip.io/' /etc/httpd/conf.d/xdmod.conf",
+      for node, net in module.nodes.network:
+    "ansible-playbook -c local -i /CRI_XCBC/hosts -l `hostname -s` -e  \"{'cod_deploy':'false', 'compute_nodes':[{'name':'${node}', 'ip':'${net[0].fixed_ip_v4}', 'mac':'${net[0].mac}', 'vnfs':'', 'sockets':'1', 'corespersocket':'1'}]}\" /CRI_XCBC/site-ops.yaml -b -v"]
+  }
+
+  # ood node
+#  provisioner "remote-exec" {
+#    inline = [
+#      for net in module.create-ood-instance.network:
+#    "ansible-playbook -c local -i /CRI_XCBC/hosts -l `hostname -s` -e \"{'compute_nodes':[{'name':'${var.ood_instance_name}', 'ip':'${net.fixed_ip_v4}', 'mac':'${net.mac}', 'vnfs':'', 'sockets':'1', 'corespersocket':'1'}]}\" /CRI_XCBC/site-ops.yaml -b -v"]
+#  }
+}

nodes/main.tf

 # is created in internal-network module and called in root module
-variable "internal_subnet_id" {type = "string"}
+variable "internal_subnet_id" {type = string}
 
 variable "image_compute" {}
 variable "flavor" {}
 
 # is created in key-pair module and called in root module
-variable "key_pair" {type = "string"}
+variable "key_pair" {type = string}
 
 variable "compute_node_count" { }
 variable "internal_network" {}
 
+terraform {
+required_version = ">= 0.14.0"
+  required_providers {
+    openstack = {
+      source  = "terraform-provider-openstack/openstack"
+      version = "~> 1.42.0"
+    }
+  }
+}
+
+provider "openstack" {
+  use_octavia         = true
+  endpoint_overrides = {
+      "network"  = "https://neutron-api.cloud.rc.uab.edu:9696/v2.0/"
+    }
+}
 
+data "openstack_images_image_v2" "compute" {
+  name = var.image_compute
+  most_recent = true
+}
 
 # creates compute node
-resource "openstack_compute_instance_v2" "c0" {
+resource "openstack_compute_instance_v2" "compute" {
   depends_on      = [var.internal_subnet_id]
   name            = "c${count.index}"
-  image_name      = var.image_compute
+  image_id        = data.openstack_images_image_v2.compute.id
   flavor_name     = var.flavor
   key_pair        = var.key_pair
   security_groups = ["default"]
   count           = var.compute_node_count
+  user_data =  <<-EOF
+    #cloud-config
+    runcmd:
+     - [ bash, -xc, 'ethernet=$(cat /sys/class/net/eth0/address); nodename=$(hostname -s); sed -e "s/MY_HWADDR/$ethernet/" -e "s/MY_NODENAME/$nodename/" -i /warewulf/config;' ]
+     - [ bash, -xc, "echo $(date) ': hello world!'; until WWGETFILES_INTERVAL=0 bash -x /warewulf/bin/wwgetfiles; do echo waiting ; rm -f /tmp/.wwgetfile.lock ; sleep 10; done;" ]
+     - [ bash, -xc, "systemctl restart munge" ]
+     - [ bash, -xc, "systemctl restart slurmd" ]
+    EOF
 
 # defines the networks of the instance
   network {
-    name = var.internal_network
+    uuid = var.internal_network
   }
-}
\ No newline at end of file
+}
+
+# output
+output "id" {
+  value = openstack_compute_instance_v2.compute.*.id
+}
+
+output "image_id" {
+  value = data.openstack_images_image_v2.compute.id
+}
+
+output "network" {
+  value = zipmap(openstack_compute_instance_v2.compute.*.name, openstack_compute_instance_v2.compute.*.network)
+}

ohpc-instance/main.tf

 # is created in external-network module and called in root module
-variable "external_subnet_id" {type = "string"}
+terraform {
+required_version = ">= 0.14.0"
+  required_providers {
+    openstack = {
+      source  = "terraform-provider-openstack/openstack"
+      version = "~> 1.42.0"
+    }
+  }
+}
+
+variable "external_subnet_id" {type = string}
 
 variable "ohpc_instance_name" {}
 variable "image_ohpc" {}
 variable "flavor" {}
 
 # is created in key-pair module and called in root module
-variable "key_pair" {type = "string"}
+variable "key_pair" {type = string}
 
 variable "internal_network" {}
+variable "internal_ip" {}
 variable "external_network" {}
 
 # is created in floating-ip module and called in root module
-variable "floating_ip_ohpc" {type = "string"}
+variable "floating_ip_ohpc" {type = string}
 
-variable "host_prefix" {}
 variable "ohpc_user" {}
 variable "ssh_private_key" {}
 
@@ -27,13 +37,23 @@ resource "openstack_compute_instance_v2" "ohpc" {
   flavor_name     = var.flavor
   key_pair        = var.key_pair
   security_groups = ["default"]
+  user_data = <<-EOF
+     #cloud-config
+     write_files:
+     - content: |
+         10.1.1.10 ohpc ohpc.novalocal
+       owner: centos:centos
+       path: /etc/hosts
+       permissions: 0644
+     EOF
 
 # defines the networks of the instance
   network {
-    name = var.external_network
+    uuid = var.external_network
   }
   network {
-    name = var.internal_network
+    uuid = var.internal_network
+    fixed_ip_v4 = var.internal_ip
   }
 }
 
@@ -50,6 +70,10 @@ resource "openstack_compute_floatingip_associate_v2" "ohpc" {
   }
 }
 
+output "id" {
+    value = openstack_compute_instance_v2.ohpc.id
+}
+
 output "ssh_host" {
-    value = format(var.host_prefix,element(split(".", var.floating_ip_ohpc),3,),)
-}
\ No newline at end of file
+    value = var.floating_ip_ohpc
+}

ood-instance/main.tf

 # is created in internal-network module and called in root module
-variable "internal_subnet_id" {type = "string"}
+variable "internal_subnet_id" {type = string}
 
 variable "ood_instance_name" {}
 variable "image_ood" {}
 variable "flavor" {}
 
 # is created in key-pair module and called in root module
-variable "key_pair" {type = "string"}
+variable "key_pair" {type = string}
 
 variable "internal_network" {}
+variable "internal_ip" {}
 variable "external_network" {}
 
 # is created in floating-ip module and called in root module
-variable "floating_ip_ood" {type = "string"}
+variable "floating_ip_ood" {type = string}
 
 variable "host_prefix" {}
 variable "ood_user" {}
 variable "ssh_private_key" {}
 
+terraform {
+required_version = ">= 0.14.0"
+  required_providers {
+    openstack = {
+      source  = "terraform-provider-openstack/openstack"
+      version = "~> 1.42.0"
+    }
+  }
+}
+
+provider "openstack" {
+  use_octavia         = true
+  endpoint_overrides = {
+      "network"  = "https://neutron-api.cloud.rc.uab.edu:9696/v2.0/"
+    }
+}
 
 # creates details for the OOD instance
 resource "openstack_compute_instance_v2" "ood" {
@@ -27,6 +44,15 @@ resource "openstack_compute_instance_v2" "ood" {
   flavor_name     = var.flavor
   key_pair        = var.key_pair
   security_groups = ["default"]
+  user_data =  <<-EOF
+    #cloud-config
+    runcmd:
+     - [ bash, -xc, 'ethernet=$(cat /sys/class/net/eth1/address); nodename=$(hostname -s); sed -e "s/MY_HWADDR/$ethernet/" -e "s/MY_NODENAME/$nodename/" -i /warewulf/config;' ]
+     - [ bash, -xc, "until WWGETFILES_INTERVAL=0 bash -x /warewulf/bin/wwgetfiles; do echo waiting ; rm -f /tmp/.wwgetfile.lock ; sleep 10; done;" ]
+     - [ bash, -xc, "systemctl restart munge" ]
+     - [ bash, -xc, "cd /CRI_XCBC; ansible-playbook -c local -i /CRI_XCBC/hosts -l `hostname -s` /CRI_XCBC/ood-ops.yaml -b -v" ]
+    EOF
+
 
 # defines the networks of the instance
   network {
@@ -34,6 +60,7 @@ resource "openstack_compute_instance_v2" "ood" {
   }
   network {
     name = var.internal_network
+    fixed_ip_v4 = var.internal_ip
   }
 }
 
@@ -51,7 +78,12 @@ resource "openstack_compute_floatingip_associate_v2" "ood" {
 }
 
 
-
+output "id" {
+    value = openstack_compute_instance_v2.ood.id
+}
 output "ssh_host"{
     value = format(var.host_prefix,element(split(".", var.floating_ip_ood),3,),)
-}
\ No newline at end of file
+}
+output "network" {
+    value = openstack_compute_instance_v2.ood.network
+}

vars.tf

@@ -6,7 +6,7 @@ variable "admin_state_up" {default = true}
 variable "enable_dhcp" {default = true}
 
 # variable for floating-ip - also used in exrernal network creation
-variable "public_network_name" {default = "bright-external-flat-externalnet"}
+variable "public_network_name" {default = "uab-campus"}
  
 # variables for keypair module
 variable "keypair_name" {default = "os-gen-keypair"}
@@ -14,18 +14,19 @@ variable "ssh_public_key" {default = "~/.ssh/id_rsa.pub"}
  
 # variables for instance modules
 variable "ohpc_instance_name" {default = "ohpc"}
+variable "ohpc_private_ip" {default = "10.1.1.10"}
 variable "ood_instance_name" { default = "ood"}
-variable "image_ohpc" {default = "CentOS-7-x86_64-GenericCloud-1905"}
-variable "image_ood" {default = "CentOS-7-x86_64-GenericCloud-1905"}
-variable "flavor" {default = "m1.medium"}
+variable "image_ohpc" {default = "xdmod-20230614063844"}
+variable "image_ood" {default = ""}
+variable "ood_private_ip" {default = "10.1.1.11"}
+variable "flavor" {default = "m1.xlarge"}
 variable "internal_network" {default = "clusternet"}
 variable "external_network" {default = "dmznet"}
-variable "host_prefix" {default = "164.111.161.%s"}
 variable "ohpc_user" {default = "centos"}
 variable "ood_user" {default = "centos"}
 variable "ssh_private_key" {default = "~/.ssh/id_rsa"}
 
 
 # variables for node creation module
-variable "image_compute" {default = "CentOS-7-x86_64-GenericCloud-1905"}
-variable "compute_node_count" {default = 2}
+variable "image_compute" {default = "compute-v1"}
+variable "compute_node_count" {default = 1}