.gitlab-ci.yml

 image: gitlab.rc.uab.edu:4567/rc/packer-openstack-hpc-image:latest
 
 variables:
+  TARGET_ENV: "dev"
   ANSIBLE_REMOTE_TMP: "/tmp"
   OS_REGION_NAME: "bhm1"
   OS_INTERFACE: "public"
@@ -8,9 +9,7 @@ variables:
   OS_AUTH_TYPE: "v3applicationcredential"
   OS_AUTH_URL: "https://keystone.cloud.rc.uab.edu:5000/v3"
   TF_ROOT: ${CI_PROJECT_DIR}/
-  TF_VAR_internal_network: "xdmod-packer-clusternet"
   TF_VAR_external_network: "xdmod-packer-dmznet"
-  TF_VAR_flavor: "m1.medium"
   GIT_SUBMODULE_STRATEGY: "recursive"
   GIT_SUBMODULE_UPDATE_FLAGS: "--force"
 
@@ -26,9 +25,53 @@ stages:          # List of stages for jobs, and their order of execution
   - deploy
   - cleanup
 
+prep-deploy:
+  stage: .pre
+  tags: [build]
+  environment:
+    name: $TARGET_ENV
+  script:
+    - export BUILD_DATE=$(TZ=America/Chicago date +%Y-%m-%dT%H%M%S)
+    - |
+      if [ "$TARGET_ENV" = "dev" ]; then
+           TF_VAR_data_volume="xdmod-dev-volume-$BUILD_DATE"
+           openstack volume create --snapshot xdmod-dev-volume-snap --size 20 $TF_VAR_data_volume
+      elif [ "$TARGET_ENV" = "staging" ]; then
+           TF_VAR_data_volume="xdmod-staging-volume-$BUILD_DATE"
+           openstack volume create --snapshot xdmod-staging-vol-snapshot --size 150 $TF_VAR_data_volume
+      elif [ "$TARGET_ENV" = "production" ]; then
+           export prod_instance_id=$(openstack floating ip show $prod_floating_ip -c port_details -f value | awk -F', ' '{print $2}' | awk -F"'" '{print $2}')
+           export prod_volume=$(openstack server show $prod_instance_id -c volumes_attached -f value | awk -F"'" '{print $4}')
+           echo "Detaching prod_volume=${prod_volume} from prod_instance_id=${prod_instance_id} and creating a snapshot"
+           openstack server stop $prod_instance_id && openstack server show $prod_instance_id 
+           openstack server remove volume $prod_instance_id $prod_volume
+           openstack volume snapshot create --volume $prod_volume xdmod-prod-vol-snapshot-$BUILD_DATE
+
+           echo "Reattach the prod_volume=${prod_volume} to prod_instance_id=${prod_instance_id} after snapshot is created"
+           openstack server add volume $prod_instance_id $prod_volume --device /dev/vdb
+           openstack server start $prod_instance_id
+
+           echo "Make a copy of prod volume $prod_volume to be used in tf-apply"
+           TF_VAR_data_volume="xdmod-prod-vol-$BUILD_DATE"
+           openstack volume create --snapshot xdmod-prod-vol-snapshot-$BUILD_DATE --size 150 $TF_VAR_data_volume
+      else
+           echo "Unknown TARGET_ENV: $TARGET_ENV"
+           exit 1
+      fi
+      echo "TF_VAR_data_volume=${TF_VAR_data_volume}" | tee -a $CI_PROJECT_DIR/vars.env
+      echo "TF_VAR_flavor=${TF_VAR_flavor}" | tee -a $CI_PROJECT_DIR/vars.env
+      echo "BUILD_DATE=${BUILD_DATE}" | tee -a $CI_PROJECT_DIR/vars.env
+      echo "prod_instance_id=${prod_instance_id}" | tee -a $CI_PROJECT_DIR/vars.env
+
+  artifacts:
+    reports:
+      dotenv: vars.env
+
 tf-validate:
   stage: validate
   tags: [build]
+  environment:
+    name: $TARGET_ENV
   script:
     - cd CRI_XCBC && git checkout dev && cd ..
     - terraform --version
@@ -44,23 +87,76 @@ tf-apply:
       - .terraform/
   stage: build
   tags: [build]
+  environment:
+    name: $TARGET_ENV
   script:
-    - export BUILD_DATE=$(TZ=America/Chicago date +%Y%m%d%H%M%S)
-    - | 
+    - echo "TF_VAR_data_volume=${TF_VAR_data_volume}"
+    - echo "TF_VAR_flavor=${TF_VAR_flavor}"
+    - export TF_VAR_internal_network="xdmod-packer-clusternet-$BUILD_DATE"
+    - |
       if [ -z $TF_VAR_image_ohpc ];
       then
-      XDMOD_IMAGES=($(openstack image list --sort-column Name --sort-descending -f value -c Name -c ID | grep -P ' xdmod-\d{14}$' | awk '{print $2}'))
+      XDMOD_IMAGES=($(openstack image list --sort-column Name --sort-descending -f value -c Name -c ID | grep -P " xdmod-$TARGET_ENV-\d{4}-\d{2}-\d{2}T\d{6}$" | awk '{print $2}'))
       export TF_VAR_image_ohpc="${XDMOD_IMAGES[0]}"
       echo $TF_VAR_image_ohpc
       else
       echo "image_ohpc defined as ${TF_VAR_image_ohpc}"
-      fi   
+      fi
     - export TF_VAR_keypair_name="os-gen-keypair-$BUILD_DATE"
-    - terraform apply -auto-approve || terraform destroy -auto-approve 
+    - terraform apply -auto-approve
+    - export instance_floating_ip=$(terraform output -raw floating_ip_ohpc)
+    - export instance_id=$(terraform output -raw xdmod_instance_id)
+    - echo "instance_floating_ip=${instance_floating_ip}" | tee -a $CI_PROJECT_DIR/vars.env
+    - echo "instance_id=${instance_id}" | tee -a $CI_PROJECT_DIR/vars.env
+    - echo "prod_instance_id=${prod_instance_id}" | tee -a $CI_PROJECT_DIR/vars.env
+  artifacts:
+    reports:
+      dotenv: vars.env
+
+staging-post-deploy:
+  environment:
+    name: staging
+  stage: build
+  tags: [build]
+  script:
+    - export old_instance_id=$(openstack floating ip show $staging_floating_ip -c port_details -f value | awk -F', ' '{print $2}' | awk -F"'" '{print $2}')
+    - echo "staging_floating_ip=${staging_floating_ip}"
+    - echo "old_instance_id=${old_instance_id}"
+    - |
+      if [[ -n "$old_instance_id" ]]; then
+        openstack server remove floating ip $old_instance_id $staging_floating_ip
+        openstack server delete $old_instance_id
+      fi
+    - openstack server remove floating ip $instance_id $instance_floating_ip
+    - openstack server add floating ip $instance_id $staging_floating_ip
+    - openstack floating ip delete $instance_floating_ip
+  needs:
+    - job: tf-apply
+      artifacts: true
+  rules:
+    - if: '$TARGET_ENV == "staging"'
+
+production-deploy:
+  environment:
+    name: production
+  stage: deploy
+  tags: [build]
+  script:
+    - echo "swap the floating IP between prod and newly deployed instances"
+    - openstack server remove floating ip $prod_instance_id $prod_floating_ip
+    - openstack server remove floating ip $instance_id $instance_floating_ip
+    - openstack server add floating ip $instance_id $prod_floating_ip
+  needs:
+    - job: tf-apply
+      artifacts: true
+  rules:
+    - if: '$TARGET_ENV == "production"'
 
 clean-up:
   stage: cleanup
   tags: [build]
+  environment:
+    name: $TARGET_ENV
   script:
     - terraform destroy -auto-approve
   when: manual

internal-network/main.tf

-variable "name" {default = "cluster"}
+variable "internal_net" {default = ""}
 variable "admin_state_up" { }
 variable "enable_dhcp" {}
 
@@ -21,21 +21,21 @@ provider "openstack" {
 
 # creates clusternet
 resource "openstack_networking_network_v2" "internal_network" {
-  name           = "${var.name}net"
+  name           = var.internal_net
   admin_state_up = var.admin_state_up
 }
 
 # creates clustersubnet
 # cidr is the subnet range (that subnet range and dns nameservers from the network create file in feat-openstack)
 resource "openstack_networking_subnet_v2" "internal_subnet" {
-  name        = "${var.name}subnet"
+  name        = "${var.internal_net}_subnet"
   network_id  = openstack_networking_network_v2.internal_network.id
   cidr        = "10.1.1.0/24"
   ip_version  = 4
   enable_dhcp = var.enable_dhcp
 }
 
-output "id" {
+output "internal_network_id" {
     value = openstack_networking_network_v2.internal_network.id
     depends_on = [openstack_networking_subnet_v2.internal_subnet]
 }

main.tf

@@ -14,6 +14,19 @@ provider "openstack" {
     }
 }
 
+# runs the internal-network module
+module "cluster-network" {
+    internal_net = var.internal_network
+    source = "./internal-network"
+    # Default name var is in the module main file
+    admin_state_up = var.admin_state_up
+    enable_dhcp = var.enable_dhcp
+}
+# calls the outputs defined in the internal-network module
+output "internal_network_id" {
+    value = "${module.cluster-network.internal_network_id}"
+}
+
 # runs the floating-ip module - uses public network name defined above
 module "floating-ip-address" {
     source = "./floating-ip"
@@ -42,7 +55,7 @@ output "keypair_name" {
 }
 
 data "openstack_networking_network_v2" "external_net" {name = var.external_network}
-data "openstack_networking_network_v2" "internal_net" {name = var.internal_network}
+data "openstack_blockstorage_volume_v3" "disk" {name = var.data_volume}
 
 # runs the ohpc-instance module - creates ohpc instance using variables defined above
 # calls functions from dmz-network, import-keypair, and floating-ip-address modules to get values created there for use
@@ -53,11 +66,12 @@ module "create-ohpc-instance" {
     flavor = var.flavor
     key_pair = module.import-keypair.keypair_name
     external_network = data.openstack_networking_network_v2.external_net.id
-    internal_network = data.openstack_networking_network_v2.internal_net.id
+    internal_network = "${module.cluster-network.internal_network_id}"
     internal_ip = var.ohpc_private_ip
     floating_ip_ohpc = module.floating-ip-address.ohpc_address
     ohpc_user =  var.ohpc_user
     ssh_private_key = var.ssh_private_key
+    vol_id = data.openstack_blockstorage_volume_v3.disk.id
 }
 
 # runs the ood-instance module - creates ood instance using variables defined above
@@ -80,12 +94,13 @@ module "create-ohpc-instance" {
 # runs the nodes module - creates nodes using variables defined above
 # calls functions from cluster-network and import-keypair modules to get values created there for use
 module "nodes" {
+    internal_subnet_id = "${module.cluster-network.internal_subnet_id}"
     source = "./nodes"
     image_compute = var.image_compute
     flavor = var.flavor
     key_pair = module.import-keypair.keypair_name
     compute_node_count = var.compute_node_count
-    internal_network = data.openstack_networking_network_v2.internal_net.id
+    internal_network = "${module.cluster-network.internal_network_id}"
 }
 
 # calls the outputs defined in the ohpc-instance module
@@ -93,6 +108,10 @@ output "ohpc-ssh_host" {
     value = module.create-ohpc-instance.ssh_host
 }
 
+output "xdmod_instance_id" {
+    value = module.create-ohpc-instance.xdmod_instance_id
+}
+
 # calls the outputs defined in the ood-instance module
 #output "ood-ssh_host" {
 #    value = module.create-ood-instance.ssh_host
@@ -103,7 +122,7 @@ output "ohpc-ssh_host" {
 # that lead to inconsistent deploy successes.
 resource "null_resource" "ops" {
   triggers = {
-    ohpc_instance = module.create-ohpc-instance.id
+    ohpc_instance = module.create-ohpc-instance.xdmod_instance_id
     compute_instances = join(",", module.nodes.id)
   }
 
@@ -128,6 +147,14 @@ provisioner "remote-exec" {
    ]
   }
 
+provisioner "remote-exec" {
+    inline = [
+      "sudo mount ${module.create-ohpc-instance.device}1 /var/lib/mysql",
+      "sudo df -h",
+      "sudo systemctl restart mariadb",
+    ]
+  }
+
   # compute node registration on ohpc
   provisioner "remote-exec" {
     inline = [
@@ -142,4 +169,5 @@ provisioner "remote-exec" {
 #      for net in module.create-ood-instance.network:
 #    "ansible-playbook -c local -i /CRI_XCBC/hosts -l `hostname -s` -e \"{'compute_nodes':[{'name':'${var.ood_instance_name}', 'ip':'${net.fixed_ip_v4}', 'mac':'${net.mac}', 'vnfs':'', 'sockets':'1', 'corespersocket':'1'}]}\" /CRI_XCBC/site-ops.yaml -b -v"]
 #  }
+
 }

nodes/main.tf

+# is created in internal-network module and called in root module
+variable "internal_subnet_id" {type = string}
+
 variable "image_compute" {}
 variable "flavor" {}
 
@@ -31,6 +34,7 @@ data "openstack_images_image_v2" "compute" {
 
 # creates compute node
 resource "openstack_compute_instance_v2" "compute" {
+  depends_on      = [var.internal_subnet_id]
   name            = "c${count.index}"
   image_id        = data.openstack_images_image_v2.compute.id
   flavor_name     = var.flavor

ohpc-instance/main.tf

@@ -26,7 +26,7 @@ variable "floating_ip_ohpc" {type = string}
 
 variable "ohpc_user" {}
 variable "ssh_private_key" {}
-
+variable "vol_id" {}
 
 # creates details for the OHPC instance
 resource "openstack_compute_instance_v2" "ohpc" {
@@ -61,10 +61,20 @@ resource "openstack_compute_floatingip_associate_v2" "ohpc" {
   instance_id = openstack_compute_instance_v2.ohpc.id
 }
 
-output "id" {
+# Attach a volume
+resource "openstack_compute_volume_attach_v2" "volume_attach" {
+  instance_id = openstack_compute_instance_v2.ohpc.id
+  volume_id   = var.vol_id
+}
+
+output "xdmod_instance_id" {
     value = openstack_compute_instance_v2.ohpc.id
 }
 
 output "ssh_host" {
     value = var.floating_ip_ohpc
 }
+
+output "device" {
+    value = openstack_compute_volume_attach_v2.volume_attach.device
+}

vars.tf

@@ -25,7 +25,7 @@ variable "external_network" {default = "dmznet"}
 variable "ohpc_user" {default = "centos"}
 variable "ood_user" {default = "centos"}
 variable "ssh_private_key" {default = "~/.ssh/id_rsa"}
-
+variable "data_volume" {default = ""}
 
 # variables for node creation module
 variable "image_compute" {default = "compute-v1"}