Merge branch 'feat-prod-pipeline-update' into 'master'

Update prod pipeline

See merge request rc/terraform-openstack!35

.gitlab-ci.yml

@@ -40,7 +40,27 @@ prep-deploy:
            TF_VAR_data_volume="xdmod-staging-volume-$BUILD_DATE"
            openstack volume create --snapshot xdmod-staging-vol-snapshot --size 150 $TF_VAR_data_volume
       elif [ "$TARGET_ENV" = "production" ]; then
-           echo $TARGET_ENV
+           export prod_instance_id=$(openstack floating ip show $prod_floating_ip -c port_details -f value | awk -F', ' '{print $2}' | awk -F"'" '{print $2}')
+           export prod_volume=$(openstack server show $prod_instance_id -c volumes_attached -f value | awk -F"'" '{print $4}')
+           echo "Detaching prod_volume=${prod_volume} from prod_instance_id=${prod_instance_id} and creating a snapshot"
+           eval $(ssh-agent -s)
+           chmod 400 "${SSH_PRIV_KEY}"
+           ssh-add "${SSH_PRIV_KEY}"
+           mkdir ~/.ssh
+           chmod 700 ~/.ssh
+           ssh -o StrictHostKeyChecking=no $SSH_USERNAME@$prod_floating_ip sudo systemctl stop mariadb
+           ssh -o StrictHostKeyChecking=no $SSH_USERNAME@$prod_floating_ip sudo umount /var/lib/mysql && df -h
+           openstack server remove volume $prod_instance_id $prod_volume
+           openstack volume snapshot create --volume $prod_volume xdmod-prod-vol-snapshot-$BUILD_DATE
+
+           echo "Reattach the prod_volume=${prod_volume} to prod_instance_id=${prod_instance_id} after snapshot is created"
+           openstack server add volume $prod_instance_id $prod_volume --device /dev/vdb
+           ssh -o StrictHostKeyChecking=no $SSH_USERNAME@$prod_floating_ip sudo mount /dev/vdb1 /var/lib/mysql && df -h
+           ssh -o StrictHostKeyChecking=no $SSH_USERNAME@$prod_floating_ip sudo systemctl start mariadb
+
+           echo "Make a copy of prod volume $prod_volume to be used in tf-apply"
+           TF_VAR_data_volume="xdmod-prod-vol-$BUILD_DATE"
+           openstack volume create --snapshot xdmod-prod-vol-snapshot-$BUILD_DATE --size 150 $TF_VAR_data_volume
       else
            echo "Unknown TARGET_ENV: $TARGET_ENV"
            exit 1
@@ -48,6 +68,8 @@ prep-deploy:
       echo "TF_VAR_data_volume=${TF_VAR_data_volume}" | tee -a $CI_PROJECT_DIR/vars.env
       echo "TF_VAR_flavor=${TF_VAR_flavor}" | tee -a $CI_PROJECT_DIR/vars.env
       echo "BUILD_DATE=${BUILD_DATE}" | tee -a $CI_PROJECT_DIR/vars.env
+      echo "prod_instance_id=${prod_instance_id}" | tee -a $CI_PROJECT_DIR/vars.env
+
   artifacts:
     reports:
       dotenv: vars.env
@@ -93,6 +115,7 @@ tf-apply:
     - export instance_id=$(terraform output -raw xdmod_instance_id)
     - echo "instance_floating_ip=${instance_floating_ip}" | tee -a $CI_PROJECT_DIR/vars.env
     - echo "instance_id=${instance_id}" | tee -a $CI_PROJECT_DIR/vars.env
+    - echo "prod_instance_id=${prod_instance_id}" | tee -a $CI_PROJECT_DIR/vars.env
   artifacts:
     reports:
       dotenv: vars.env
@@ -126,42 +149,15 @@ production-deploy:
   stage: deploy
   tags: [build]
   script:
-    - export prod_instance_id=$(openstack floating ip show $prod_floating_ip -c port_details -f value | awk -F', ' '{print $2}' | awk -F"'" '{print $2}')
-    - export prod_volume=$(openstack server show $prod_instance_id -c volumes_attached -f value | awk -F"'" '{print $4}')
-    - echo "Detaching prod_volume_id=${prod_volume} from prod_instance_id=${prod_instance_id}"
-    - |
-      eval $(ssh-agent -s)
-      chmod 400 "${SSH_PRIV_KEY}"
-      ssh-add "${SSH_PRIV_KEY}"
-      mkdir ~/.ssh
-      chmod 700 ~/.ssh
-    - ssh -o StrictHostKeyChecking=no $SSH_USERNAME@$prod_floating_ip sudo systemctl stop mariadb
-    - ssh -o StrictHostKeyChecking=no $SSH_USERNAME@$prod_floating_ip sudo umount /var/lib/mysql && df -h
-    - openstack server remove volume $prod_instance_id $prod_volume
-    - openstack server stop $prod_instance_id
-
-    - export staging_volume=$(openstack server show $instance_id -c volumes_attached -f value | awk -F"'" '{print $4}')
-    - echo "Detaching staging_volume_id=${staging_volume} from staging_instance_id=${instance_id}"
-    - ssh -o StrictHostKeyChecking=no $SSH_USERNAME@$staging_floating_ip sudo systemctl stop mariadb
-    - ssh -o StrictHostKeyChecking=no $SSH_USERNAME@$staging_floating_ip sudo umount /var/lib/mysql && df -h
-    - openstack server remove volume $instance_id $staging_volume
-
-    - echo "Attach prod volume $xdmod-prod-vol-$BUILD_DATE to staging instance $instance_id"
-    - openstack volume snapshot create --volume $prod_volume xdmod-prod-data-vol-snapshot-$BUILD_DATE
-    - openstack volume create --snapshot xdmod-prod-data-vol-snapshot-$BUILD_DATE --size 150 xdmod-prod-vol-$BUILD_DATE
-    - openstack server add volume $instance_id xdmod-prod-vol-$BUILD_DATE --device /dev/vdb
-
-    - echo "swap the floating IP between prod and staging instances"
+    - echo "swap the floating IP between prod and newly deployed instances"
     - openstack server remove floating ip $prod_instance_id $prod_floating_ip
-    - openstack server remove floating ip $instance_id $staging_floating_ip
+    - openstack server remove floating ip $instance_id $instance_floating_ip
     - openstack server add floating ip $instance_id $prod_floating_ip
-
-    - ssh -o StrictHostKeyChecking=no $SSH_USERNAME@$prod_floating_ip sudo mount /dev/vdb1 /var/lib/mysql
-    - ssh -o StrictHostKeyChecking=no $SSH_USERNAME@$prod_floating_ip sudo systemctl start mariadb
-
-  when: manual
+  needs:
+    - job: tf-apply
+      artifacts: true
   rules:
-    - if: '$TARGET_ENV != "dev"'
+    - if: '$TARGET_ENV == "production"'
 
 clean-up:
   stage: cleanup