.gitlab-ci.yml

@@ -36,29 +36,50 @@ prep-deploy:
       if [ "$TARGET_ENV" = "dev" ]; then
            TF_VAR_data_volume="xdmod-dev-volume-$BUILD_DATE"
            openstack volume create --snapshot xdmod-dev-volume-snap --size 20 $TF_VAR_data_volume
+           TF_VAR_xdmod_queue_data_volume="xdmod-dev-queue-volume-$BUILD_DATE"
+           openstack volume create --snapshot xdmod-queue-volume-snap --size 1 $TF_VAR_xdmod_queue_data_volume
       elif [ "$TARGET_ENV" = "staging" ]; then
            TF_VAR_data_volume="xdmod-staging-volume-$BUILD_DATE"
            openstack volume create --snapshot xdmod-staging-vol-snapshot --size 150 $TF_VAR_data_volume
       elif [ "$TARGET_ENV" = "production" ]; then
            export prod_instance_id=$(openstack floating ip show $prod_floating_ip -c port_details -f value | awk -F', ' '{print $2}' | awk -F"'" '{print $2}')
-           export prod_volume=$(openstack server show $prod_instance_id -c volumes_attached -f value | awk -F"'" '{print $4}')
+           export volume_ids=$(openstack server show $prod_instance_id -c volumes_attached | grep -o "id='[a-zA-Z0-9\-]*'" | awk -F "'" '{print $2}')
+           for volume_id in $volume_ids; do
+             volume_name=$(openstack volume show $volume_id -c name -f value)
+             if [[ $volume_name == xdmod-dev-volume* ]]; then
+               export prod_volume=$volume_id
+             elif [[ $volume_name == xdmod-dev-queue-volume* ]]; then
+               export prod_queue_volume=$volume_id
+             fi
+           done
+
+
            echo "Detaching prod_volume=${prod_volume} from prod_instance_id=${prod_instance_id} and creating a snapshot"
+           echo "Detaching prod_queue_volume=${prod_queue_volume} from prod_instance_id=${prod_instance_id} and creating a snapshot"
            openstack server stop $prod_instance_id && openstack server show $prod_instance_id 
            openstack server remove volume $prod_instance_id $prod_volume
+           openstack server remove volume $prod_instance_id $prod_queue_volume          
            openstack volume snapshot create --volume $prod_volume xdmod-prod-vol-snapshot-$BUILD_DATE
-
+           openstack volume snapshot create --volume $prod_queue_volume xdmod-prod-queue-vol-snapshot-$BUILD_DATE
+           
            echo "Reattach the prod_volume=${prod_volume} to prod_instance_id=${prod_instance_id} after snapshot is created"
            openstack server add volume $prod_instance_id $prod_volume --device /dev/vdb
+           openstack server add volume $prod_instance_id $prod_queue_volume --device /dev/vdc
            openstack server start $prod_instance_id
 
            echo "Make a copy of prod volume $prod_volume to be used in tf-apply"
            TF_VAR_data_volume="xdmod-prod-vol-$BUILD_DATE"
            openstack volume create --snapshot xdmod-prod-vol-snapshot-$BUILD_DATE --size 150 $TF_VAR_data_volume
+
+           echo "Make a copy of prod queue volume $prod_volume to be used in tf-apply"
+           TF_VAR_xdmod_queue_data_volume="xdmod-prod-queue-vol-$BUILD_DATE"
+           openstack volume create --snapshot xdmod-prod-queue-vol-snapshot-$BUILD_DATE --size 1 $TF_VAR_xdmod_queue_data_volume
       else
            echo "Unknown TARGET_ENV: $TARGET_ENV"
            exit 1
       fi
       echo "TF_VAR_data_volume=${TF_VAR_data_volume}" | tee -a $CI_PROJECT_DIR/vars.env
+      echo "TF_VAR_xdmod_queue_data_volume=${TF_VAR_xdmod_queue_data_volume}" | tee -a $CI_PROJECT_DIR/vars.env
       echo "TF_VAR_flavor=${TF_VAR_flavor}" | tee -a $CI_PROJECT_DIR/vars.env
       echo "BUILD_DATE=${BUILD_DATE}" | tee -a $CI_PROJECT_DIR/vars.env
       echo "prod_instance_id=${prod_instance_id}" | tee -a $CI_PROJECT_DIR/vars.env
@@ -142,7 +163,21 @@ production-deploy:
   stage: deploy
   tags: [build]
   script:
-    - echo "swap the floating IP between prod and newly deployed instances"
+    - export prod_instance_id=$(openstack floating ip show $prod_floating_ip -c port_details -f value | awk -F', ' '{print $2}' | awk -F"'" '{print $2}')
+    - export prod_volume=$(openstack server show $prod_instance_id -c volumes_attached -f value | awk -F"'" '{print $4}')
+    - export prod_queue_volume=$(openstack server show $prod_instance_id -c volumes_attached -f value | awk -F"'" '{print $4}' | awk 'NR==2')
+    - echo "prod_volume $prod_volume"
+    - echo "prod_instance_id $prod_instance_id"
+    - echo "prod_queue_volume $prod_queue_volume"
+
+    - |
+      eval $(ssh-agent -s)
+      chmod 400 "${SSH_PRIV_KEY}"
+      ssh-add "${SSH_PRIV_KEY}"
+      mkdir ~/.ssh
+      chmod 700 ~/.ssh
+
+    - echo "swap the floating IP between prod and staging instances"
     - openstack server remove floating ip $prod_instance_id $prod_floating_ip
     - openstack server remove floating ip $instance_id $instance_floating_ip
     - openstack server add floating ip $instance_id $prod_floating_ip

main.tf

@@ -14,6 +14,7 @@ provider "openstack" {
     }
 }
 
+
 # runs the internal-network module
 module "cluster-network" {
     internal_net = var.internal_network
@@ -56,6 +57,7 @@ output "keypair_name" {
 
 data "openstack_networking_network_v2" "external_net" {name = var.external_network}
 data "openstack_blockstorage_volume_v3" "disk" {name = var.data_volume}
+data "openstack_blockstorage_volume_v3" "disk2" {name = var.xdmod_queue_data_volume}
 
 # runs the ohpc-instance module - creates ohpc instance using variables defined above
 # calls functions from dmz-network, import-keypair, and floating-ip-address modules to get values created there for use
@@ -71,7 +73,10 @@ module "create-ohpc-instance" {
     floating_ip_ohpc = module.floating-ip-address.ohpc_address
     ohpc_user =  var.ohpc_user
     ssh_private_key = var.ssh_private_key
-    vol_id = data.openstack_blockstorage_volume_v3.disk.id
+    vol_ids = [
+      data.openstack_blockstorage_volume_v3.disk.id,
+      data.openstack_blockstorage_volume_v3.disk2.id,
+    ]
 }
 
 # runs the ood-instance module - creates ood instance using variables defined above
@@ -132,34 +137,18 @@ resource "null_resource" "ops" {
     private_key = var.ssh_private_key
   }
 
+  # clone CRI_XCBC on node 
   provisioner "remote-exec" {
     inline = [
-      "ls -al /CRI_XCBC",
-    ]
-  }
-  # moves CRI_XCBC file into directory made above
-
-provisioner "remote-exec" {
-    inline = [
-      "sudo sed -i -E 's/xdmod..nip.io/xdmod.rc.uab.edu/g' /etc/httpd/conf.d/xdmod.conf",
-      "sudo sed -i -E 's/xdmod..nip.io/xdmod.rc.uab.edu/g' /etc/xdmod/simplesamlphp/config/config.php",
-      "sudo systemctl restart httpd",        
-   ]
-  }
-
-provisioner "remote-exec" {
-    inline = [
-      "sudo mount ${module.create-ohpc-instance.device}1 /var/lib/mysql",
-      "sudo df -h",
-      "sudo systemctl restart mariadb",
+      "git clone https://github.com/jprorama/CRI_XCBC.git /CRI_XCBC",
     ]
   }
 
   # compute node registration on ohpc
   provisioner "remote-exec" {
     inline = [
-     for node, net in module.nodes.network:
-    "ansible-playbook -c local -i /CRI_XCBC/hosts -l `hostname -s` -e  \"{'cod_deploy':'false', 'compute_nodes':[{'name':'${node}', 'ip':'${net[0].fixed_ip_v4}', 'mac':'${net[0].mac}', 'vnfs':'', 'sockets':'1', 'corespersocket':'1'}]}\" /CRI_XCBC/site-ops.yaml -b -v"
+        for node, net in module.nodes.network:
+      "ansible-playbook -c local -i /CRI_XCBC/hosts -l `hostname -s` -e  \"{'cod_deploy':'false', 'compute_nodes':[{'name':'${node}', 'ip':'${net[0].fixed_ip_v4}', 'mac':'${net[0].mac}', 'vnfs':'', 'sockets':'1', 'corespersocket':'1'}]}\" /CRI_XCBC/site-ops.yaml -b -v"
     ]
   }
 

ohpc-instance/main.tf

@@ -9,7 +9,6 @@ required_version = ">= 0.14.0"
   }
 }
 
-
 variable "ohpc_instance_name" {}
 variable "image_ohpc" {}
 variable "flavor" {}
@@ -26,7 +25,10 @@ variable "floating_ip_ohpc" {type = string}
 
 variable "ohpc_user" {}
 variable "ssh_private_key" {}
-variable "vol_id" {}
+variable "vol_ids" {
+  type    = list(string)
+  default = []
+}
 
 # creates details for the OHPC instance
 resource "openstack_compute_instance_v2" "ohpc" {
@@ -37,12 +39,24 @@ resource "openstack_compute_instance_v2" "ohpc" {
   security_groups = ["default"]
   user_data = <<-EOF
      #cloud-config
-     write_files:
-     - content: |
-         10.1.1.10 ohpc ohpc.novalocal
-       owner: centos:centos
-       path: /etc/hosts
-       permissions: 0644
+    write_files:
+    - content: |
+        10.1.1.10 ohpc ohpc.novalocal
+        owner: centos:centos
+        path: /etc/hosts
+        permissions: '0644'
+
+      runcmd:
+      - |
+          #!/bin/bash
+          sudo mkdir -p /data/xdmod/queues
+          sudo mount /dev/vdb1 /var/lib/mysql
+          sudo mount /dev/vdc1 /data/xdmod/queues
+          sudo df -h
+          sudo systemctl restart mariadb
+          sudo sed -i -E 's/xdmod..nip.io/xdmod.rc.uab.edu/g' /etc/httpd/conf.d/xdmod.conf
+          sudo sed -i -E 's/xdmod..nip.io/xdmod.rc.uab.edu/g' /etc/xdmod/simplesamlphp/config/config.php
+          sudo systemctl restart httpd
      EOF
 
 # defines the networks of the instance
@@ -61,14 +75,20 @@ resource "openstack_compute_floatingip_associate_v2" "ohpc" {
   instance_id = openstack_compute_instance_v2.ohpc.id
 }
 
-# Attach a volume
-resource "openstack_compute_volume_attach_v2" "volume_attach" {
+# Attach specific volumes
+resource "openstack_compute_volume_attach_v2" "attach_prod_volume" {
   instance_id = openstack_compute_instance_v2.ohpc.id
-  volume_id   = var.vol_id
+  volume_id   = data_volume
 }
 
+resource "openstack_compute_volume_attach_v2" "attach_prod_queue_volume" {
+  instance_id = openstack_compute_instance_v2.ohpc.id
+  volume_id   = prod_volume_id
+}
+
+
 output "xdmod_instance_id" {
-    value = openstack_compute_instance_v2.ohpc.id
+  value = openstack_compute_instance_v2.ohpc.id
 }
 
 output "ssh_host" {
@@ -76,5 +96,10 @@ output "ssh_host" {
 }
 
 output "device" {
-    value = openstack_compute_volume_attach_v2.volume_attach.device
+  value = [
+    openstack_compute_volume_attach_v2.attach_prod_volume.device,
+    openstack_compute_volume_attach_v2.attach_prod_queue_volume.device
+  ]
 }
+
+

vars.tf

@@ -26,6 +26,7 @@ variable "ohpc_user" {default = "centos"}
 variable "ood_user" {default = "centos"}
 variable "ssh_private_key" {default = "~/.ssh/id_rsa"}
 variable "data_volume" {default = ""}
+variable "xdmod_queue_data_volume" {default = ""}
 
 # variables for node creation module
 variable "image_compute" {default = "compute-v1"}