feat: Implement production deployment and instance swap

.gitlab-ci.yml

@@ -38,9 +38,22 @@ prep-deploy:
            openstack volume create --snapshot xdmod-dev-volume-snap --size 20 $TF_VAR_data_volume
       elif [ "$TARGET_ENV" = "staging" ]; then
            TF_VAR_data_volume="xdmod-staging-volume-$BUILD_DATE"
-           openstack volume create --snapshot xdmod-staging-volume-snap --size 150 $TF_VAR_data_volume
+           openstack volume create --snapshot xdmod-staging-vol-snapshot --size 150 $TF_VAR_data_volume
       elif [ "$TARGET_ENV" = "production" ]; then
-           echo $TARGET_ENV
+           export prod_instance_id=$(openstack floating ip show $prod_floating_ip -c port_details -f value | awk -F', ' '{print $2}' | awk -F"'" '{print $2}')
+           export prod_volume=$(openstack server show $prod_instance_id -c volumes_attached -f value | awk -F"'" '{print $4}')
+           echo "Detaching prod_volume=${prod_volume} from prod_instance_id=${prod_instance_id} and creating a snapshot"
+           openstack server stop $prod_instance_id && openstack server show $prod_instance_id 
+           openstack server remove volume $prod_instance_id $prod_volume
+           openstack volume snapshot create --volume $prod_volume xdmod-prod-vol-snapshot-$BUILD_DATE
+
+           echo "Reattach the prod_volume=${prod_volume} to prod_instance_id=${prod_instance_id} after snapshot is created"
+           openstack server add volume $prod_instance_id $prod_volume --device /dev/vdb
+           openstack server start $prod_instance_id
+
+           echo "Make a copy of prod volume $prod_volume to be used in tf-apply"
+           TF_VAR_data_volume="xdmod-prod-vol-$BUILD_DATE"
+           openstack volume create --snapshot xdmod-prod-vol-snapshot-$BUILD_DATE --size 150 $TF_VAR_data_volume
       else
            echo "Unknown TARGET_ENV: $TARGET_ENV"
            exit 1
@@ -48,6 +61,8 @@ prep-deploy:
       echo "TF_VAR_data_volume=${TF_VAR_data_volume}" | tee -a $CI_PROJECT_DIR/vars.env
       echo "TF_VAR_flavor=${TF_VAR_flavor}" | tee -a $CI_PROJECT_DIR/vars.env
       echo "BUILD_DATE=${BUILD_DATE}" | tee -a $CI_PROJECT_DIR/vars.env
+      echo "prod_instance_id=${prod_instance_id}" | tee -a $CI_PROJECT_DIR/vars.env
+
   artifacts:
     reports:
       dotenv: vars.env
@@ -81,27 +96,19 @@ tf-apply:
     - |
       if [ -z $TF_VAR_image_ohpc ];
       then
-      XDMOD_IMAGES=($(openstack image list --sort-column Name --sort-descending -f value -c Name -c ID | grep -P " xdmod-${TARGET_ENV}-\d{14}$" | awk '{print $2}'))
+      XDMOD_IMAGES=($(openstack image list --sort-column Name --sort-descending -f value -c Name -c ID | grep -P " xdmod-$TARGET_ENV-\d{4}-\d{2}-\d{2}T\d{6}$" | awk '{print $2}'))
       export TF_VAR_image_ohpc="${XDMOD_IMAGES[0]}"
       echo $TF_VAR_image_ohpc
       else
       echo "image_ohpc defined as ${TF_VAR_image_ohpc}"
       fi
     - export TF_VAR_keypair_name="os-gen-keypair-$BUILD_DATE"
-    - |
-      terraform apply -auto-approve
-      if [ $? -ne 0 ]; then
-        echo "Terraform apply failed. Running terraform destroy..."
-        terraform destroy -auto-approve
-        TF_SUCCESS=false
-      else
-        TF_SUCCESS=true
-      fi
-      echo "Terraform success: $TF_SUCCESS"
+    - terraform apply -auto-approve
     - export instance_floating_ip=$(terraform output -raw floating_ip_ohpc)
     - export instance_id=$(terraform output -raw xdmod_instance_id)
     - echo "instance_floating_ip=${instance_floating_ip}" | tee -a $CI_PROJECT_DIR/vars.env
     - echo "instance_id=${instance_id}" | tee -a $CI_PROJECT_DIR/vars.env
+    - echo "prod_instance_id=${prod_instance_id}" | tee -a $CI_PROJECT_DIR/vars.env
   artifacts:
     reports:
       dotenv: vars.env
@@ -181,15 +188,13 @@ production-deploy:
 
     - echo "swap the floating IP between prod and staging instances"
     - openstack server remove floating ip $prod_instance_id $prod_floating_ip
-    - openstack server remove floating ip $instance_id $staging_floating_ip
+    - openstack server remove floating ip $instance_id $instance_floating_ip
     - openstack server add floating ip $instance_id $prod_floating_ip
-
-    - ssh -o StrictHostKeyChecking=no $SSH_USERNAME@$prod_floating_ip sudo mount /dev/vdb1 /var/lib/mysql
-    - ssh -o StrictHostKeyChecking=no $SSH_USERNAME@$prod_floating_ip sudo systemctl start mariadb
-
-  when: manual
+  needs:
+    - job: tf-apply
+      artifacts: true
   rules:
-    - if: '$TARGET_ENV != "dev"'
+    - if: '$TARGET_ENV == "production"'
 
 clean-up:
   stage: cleanup