Integration Testing: Sequential Merge and Validation of Four Feature Branches (PR #43,#44, #45,#46)

The purpose of this branch is to facilitate the testing and integration of changes from four separate feature branches:

1. Remove remote-exec provisioners in Terraform script (Merge Request #46): This commit focuses on removing remote-exec provisioners from the Terraform script. By eliminating these provisioners, we aim to enhance the reliability and security of the infrastructure setup.

2. Include mount commands in user_data_script for Terraform instance initialization (Merge Request #45): This merge request introduces mount commands in the user_data_script of the Terraform script. The inclusion of these commands ensures proper initialization of instances by facilitating the mounting of specified devices to relevant directories during the initial boot process. This change enhances the robustness and efficiency of instance initialization procedures.

3. WIP: Feat vol download from s3 (Merge Request #44): This work-in-progress merge request aims to implement functionality for downloading volumes from Amazon S3 (Simple Storage Service) within the Terraform setup. Once completed, this feature will enable the seamless integration of volume download capabilities, enhancing data management and accessibility within the infrastructure.

4. feat: Add new data volume for XDMod queue data (Merge Request #43): This merge request focuses on the addition of a new data volume specifically designed for XDMod queue data. By incorporating this new volume into the infrastructure configuration, we aim to optimize data storage and management processes, thereby improving the overall efficiency and functionality of the system.

.gitlab-ci.yml

@@ -35,25 +35,38 @@ prep-deploy:
     - |
       if [ "$TARGET_ENV" = "dev" ]; then
            TF_VAR_data_volume="xdmod-dev-volume-$BUILD_DATE"
+           TF_VAR_xdmod_queue_data_volume= "xdmod-queue-volume-$BUILD_DATE"
            openstack volume create --snapshot xdmod-dev-volume-snap --size 20 $TF_VAR_data_volume
+           openstack volume create --snapshot xdmod-queue-volume-snap --size 20 $TF_VAR_xdmod_queue_data_volume
       elif [ "$TARGET_ENV" = "staging" ]; then
            TF_VAR_data_volume="xdmod-staging-volume-$BUILD_DATE"
            openstack volume create --snapshot xdmod-staging-vol-snapshot --size 150 $TF_VAR_data_volume
       elif [ "$TARGET_ENV" = "production" ]; then
            export prod_instance_id=$(openstack floating ip show $prod_floating_ip -c port_details -f value | awk -F', ' '{print $2}' | awk -F"'" '{print $2}')
-           export prod_volume=$(openstack server show $prod_instance_id -c volumes_attached -f value | awk -F"'" '{print $4}')
+           export volume_ids=$(openstack server show $prod_instance_id -c volumes_attached | grep -o "id='[a-zA-Z0-9\-]*'" | awk -F "'" '{print $2}')
+           export prod_volume=$(echo "$volume_ids" | awk 'NR==1')
+           export prod_queue_volume=$(echo "$volume_ids" | awk 'NR==2')
+
            echo "Detaching prod_volume=${prod_volume} from prod_instance_id=${prod_instance_id} and creating a snapshot"
+           echo "Detaching prod_queue_volume=${prod_queue_volume} from prod_instance_id=${prod_instance_id} and creating a snapshot"
            openstack server stop $prod_instance_id && openstack server show $prod_instance_id 
            openstack server remove volume $prod_instance_id $prod_volume
+           openstack server remove volume $prod_instance_id $prod_queue_volume          
            openstack volume snapshot create --volume $prod_volume xdmod-prod-vol-snapshot-$BUILD_DATE
-
+           openstack volume snapshot create --volume $prod_queue_volume xdmod-prod_queue_volume-snapshot-$BUILD_DATE
+           
            echo "Reattach the prod_volume=${prod_volume} to prod_instance_id=${prod_instance_id} after snapshot is created"
            openstack server add volume $prod_instance_id $prod_volume --device /dev/vdb
+           openstack server add volume $prod_instance_id $prod_queue_volume --device /dev/vdc
            openstack server start $prod_instance_id
 
            echo "Make a copy of prod volume $prod_volume to be used in tf-apply"
-           TF_VAR_data_volume="xdmod-prod-vol-$BUILD_DATE"
+           TF_VAR_data_volume="xdmod-dev-vol-$BUILD_DATE"
            openstack volume create --snapshot xdmod-prod-vol-snapshot-$BUILD_DATE --size 150 $TF_VAR_data_volume
+
+           echo "Make a copy of prod volume $prod_volume to be used in tf-apply"
+           TF_VAR_xdmod_queue_data_volume="xdmod-prod-vol-$BUILD_DATE"
+           openstack volume create --snapshot xdmod-prod_queue_volume-snapshot-$BUILD_DATE --size 1 $TF_VAR_xdmod_queue_data_volume
       else
            echo "Unknown TARGET_ENV: $TARGET_ENV"
            exit 1
@@ -155,39 +168,6 @@ production-deploy:
       ssh-add "${SSH_PRIV_KEY}"
       mkdir ~/.ssh
       chmod 700 ~/.ssh
-    - ssh -o StrictHostKeyChecking=no $SSH_USERNAME@$prod_floating_ip sudo systemctl stop mariadb
-    - ssh -o StrictHostKeyChecking=no $SSH_USERNAME@$prod_floating_ip sudo umount /var/lib/mysql && df -h
-    - ssh -o StrictHostKeyChecking=no $SSH_USERNAME@$prod_floating_ip sudo umount /data/xdmod/queues && df -h
-    - openstack server remove volume $prod_instance_id $prod_volume
-    - openstack server remove volume $prod_instance_id $prod_queue_volume
-    - openstack server stop $prod_instance_id
-
-    # Exporting Staging Volume IDs
-    - export staging_volume=$(openstack server show $instance_id -c volumes_attached -f value | awk -F"'" '{print $4}' | awk 'NR==1')
-    - export staging_queue_volume=$(openstack server show $instance_id -c volumes_attached -f value | awk -F"'" '{print $4}' | awk 'NR==2')
-
-    # Logging Information
-    - echo "Detaching staging_volume_id=${staging_volume} and staging_queue_volume_id=${staging_queue_volume} from staging_instance_id=${instance_id}"
-
-    # Stopping MariaDB Service on Staging Instance
-    - ssh -o StrictHostKeyChecking=no $SSH_USERNAME@$staging_floating_ip sudo systemctl stop mariadb
-
-    # Unmounting MySQL Data Directories
-    - ssh -o StrictHostKeyChecking=no $SSH_USERNAME@$staging_floating_ip sudo umount /var/lib/mysql && df -h
-    - ssh -o StrictHostKeyChecking=no $SSH_USERNAME@$staging_floating_ip sudo umount /data/xdmod/queues && df -h
-
-    # Removing Volumes from Server
-    - openstack server remove volume $instance_id $staging_volume
-    - openstack server remove volume $instance_id $staging_queue_volume
-
-    - echo "Attach prod volume $xdmod-prod-vol-$BUILD_DATE to staging instance $instance_id"
-    - echo "Attach prod queue volume xdmod-prod-queue-vol-$BUILD_DATE to staging instance $instance_id"
-    - openstack volume snapshot create --volume $prod_volume xdmod-prod-data-vol-snapshot-$BUILD_DATE
-    - openstack volume snapshot create --volume $prod_queue_volume xdmod-prod-queue-vol-snapshot-$BUILD_DATE
-    - openstack volume create --snapshot xdmod-prod-data-vol-snapshot-$BUILD_DATE --size 150 xdmod-prod-vol-$BUILD_DATE
-    - openstack volume create --snapshot xdmod-prod-queue-vol-snapshot-$BUILD_DATE --size 1 xdmod-prod-queue-vol-$BUILD_DATE
-    - openstack server add volume $instance_id xdmod-prod-vol-$BUILD_DATE --device /dev/vdb
-    - openstack server add volume $instance_id xdmod-prod-queue-vol-$BUILD_DATE --device /dev/vdc
 
     - echo "swap the floating IP between prod and staging instances"
     - openstack server remove floating ip $prod_instance_id $prod_floating_ip