added manual trigger for each stage of pipeline

.gitlab-ci.yml

-image: $CI_REGISTRY_IMAGE:latest
+default:
+  image: $CI_REGISTRY_IMAGE:latest
 
 variables:
   CAMPUS_IP: 138.26.48.47
@@ -41,6 +42,7 @@ workflow:
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
     - if: $CI_PIPELINE_SOURCE == 'schedule'
+    - if: $CI_PIPELINE_SOURCE == 'manual'
 
 .get_build_date: &get_build_date
   - export BUILD_DATE=$(TZ=America/Chicago date +%Y-%m-%dT%H%M%S)
@@ -102,7 +104,6 @@ build_docker_image:
       changes:
         - Dockerfile
       allow_failure: true
-    - when: manual  # Add manual trigger
 
 build_base_image:
   stage: build
@@ -127,42 +128,75 @@ build_base_image:
       fi
     - packer init openstack
     - packer validate openstack
-    - packer build -force -timestamp-ui openstack
-  rules:
-    - when: manual  # Add manual trigger
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-        - packer/**
+    - packer build -machine-readable openstack | tee base_build.log
+    - export BUILT_BASE_IMAGE_ID=$(grep 'Image:' base_build.log | awk '{print $4}')
+    - echo BUILT_BASE_IMAGE_ID=${BUILT_BASE_IMAGE_ID} | tee -a $CI_PROJECT_DIR/image.env
+    - openstack image unset --property signature_verified $BUILT_BASE_IMAGE_ID
+  artifacts:
+    reports:
+      dotenv: image.env
+    expire_in: 30 days
 
 build_compute_image:
   stage: build
+  needs: [build_base_image]
   tags:
     - build
   script:
     - *update_ansible_repo
     - *get_ansible_files
-    - export PKR_VAR_build_instance_name="compute-${CRI_XCBC_HEAD}"
-    - export PKR_VAR_image_name="compute-${CRI_XCBC_HEAD}-${BUILD_DATE}"
-    - packer init openstack
-    - packer validate openstack
-    - packer build -force -timestamp-ui openstack
-  rules:
-    - when: manual  # Add manual trigger
+    - export PKR_VAR_source_image=${BUILT_BASE_IMAGE_ID}
+    - export REPO_HEAD=$(git rev-parse --short HEAD)
+    - export PKR_VAR_flavor="${COMPUTE_BUILD_FLAVOR:-$PKR_VAR_flavor}"
+    - export PKR_VAR_build_instance_name="compute-${REPO_HEAD}"
+    - export PKR_VAR_image_date_suffix=false
+    - |
+      if [ $CI_PIPELINE_SOURCE == 'merge_request_event' ]; then
+        export PKR_VAR_image_name="compute-PR-${CI_MERGE_REQUEST_IID}"
+      elif [ $CI_PIPELINE_SOURCE == 'schedule' ]; then
+        export PKR_VAR_image_name="compute-${BUILD_DATE}"
+      fi
+    - packer init openstack-compute
+    - packer validate openstack-compute
+    - packer build -machine-readable openstack-compute | tee compute_build.log
 
 build_gpu_image:
   stage: build
+  needs: [build_base_image]
   tags:
     - build
   script:
     - *update_ansible_repo
     - *get_ansible_files
-    - export PKR_VAR_build_instance_name="gpu-${CRI_XCBC_HEAD}"
-    - export PKR_VAR_image_name="gpu-${CRI_XCBC_HEAD}-${BUILD_DATE}"
-    - packer init openstack
-    - packer validate openstack
-    - packer build -force -timestamp-ui openstack
+    - FAILED=false
+    - export GPU_PLACEHOLDER_NAME="gpu1-placeholder"
+    - export GPU_PLACEHOLDER_FLAVOR="gpu1.medium"
+    - export GPU_PLACEHOLDER_IMAGE="CentOS-7-x86_64-GenericCloud-2009"
+    - export PKR_VAR_source_image=${BUILT_BASE_IMAGE_ID}
+    - export REPO_HEAD=$(git rev-parse --short HEAD)
+    - export PKR_VAR_flavor="${GPU_BUILD_FLAVOR:-gpu1.medium}"
+    - export PKR_VAR_build_instance_name="gpu-${REPO_HEAD}"
+    - export PKR_VAR_image_date_suffix=false
+    - |
+      if [ $CI_PIPELINE_SOURCE == 'merge_request_event' ]; then
+        export PKR_VAR_image_name="gpu-PR-${CI_MERGE_REQUEST_IID}"
+      elif [ $CI_PIPELINE_SOURCE == 'schedule' ]; then
+        export PKR_VAR_image_name="gpu-${BUILD_DATE}"
+      fi
+    - packer init openstack-gpu
+    - packer validate openstack-gpu
+    - openstack server delete --wait $GPU_PLACEHOLDER_NAME
+    - packer build -machine-readable openstack-gpu | tee gpu_build.log || FAILED=true
+    - openstack server create --image $GPU_PLACEHOLDER_IMAGE --network cicd-net --flavor $GPU_PLACEHOLDER_FLAVOR $GPU_PLACEHOLDER_NAME
+    - |
+      if [ "$FAILED" = true ]; then
+        exit 1
+      fi
   rules:
-    - when: manual  # Add manual trigger
+    - if: $SKIP_GPU_BUILD == "true"
+      when: never
+    - if: $CI_PIPELINE_SOURCE == "manual"
+      when: manual
 
 build_ood_image:
   stage: build
@@ -171,35 +205,338 @@ build_ood_image:
   script:
     - *update_ansible_repo
     - *get_ansible_files
-    - export PKR_VAR_build_instance_name="ood-${CRI_XCBC_HEAD}"
-    - export PKR_VAR_image_name="ood-${CRI_XCBC_HEAD}-${BUILD_DATE}"
-    - packer init openstack
-    - packer validate openstack
-    - packer build -force -timestamp-ui openstack
-  rules:
-    - when: manual  # Add manual trigger
+    - >
+      curl --header "PRIVATE-TOKEN: ${ANSIBLE_VAR_TOKEN}"
+      "${CI_API_V4_URL}/projects/2836/repository/files/knightly/raw?ref=main"
+      -o CRI_XCBC/group_vars/knightly
+    - 'sed -i -E "s/(lts_access_key: ).*/\1\"${AWS_ACCESS_KEY_ID}\"/" CRI_XCBC/group_vars/knightly'
+    - 'sed -i -E "s/(lts_secret_key: ).*/\1\"${AWS_SECRET_ACCESS_KEY}\"/" CRI_XCBC/group_vars/knightly'
+    - 'sed -i -E "s/(user_register_app_key: ).*/\1\"${SELF_REG_APP_KEY}\"/" CRI_XCBC/group_vars/knightly'
+    - 'sed -i -E "s/(celery_user_password: ).*/\1\"${CELERY_PASSWD}\"/" CRI_XCBC/group_vars/knightly'
+    - 'sed -i -E "s|(ssh_pub_key: ).*|\1\"{{ lookup(''file'', ''${SSH_PUB_KEY}'') }}\"|" CRI_XCBC/group_vars/knightly'
+    - export PKR_VAR_flavor="${OOD_BUILD_FLAVOR:-$PKR_VAR_flavor}"
+    - packer init openstack-ood
+    - packer validate openstack-ood
+    - |
+      if [ $CI_PIPELINE_SOURCE == 'merge_request_event' ]; then
+        export PKR_VAR_image_name="ood-PR-${CI_MERGE_REQUEST_IID}"
+        echo INSTANCE_FLAVOR="${PKR_VAR_flavor}" | tee -a $CI_PROJECT_DIR/image.env
+        echo OOD_INSTANCE_NAME="ood-PR-${CI_MERGE_REQUEST_IID}" | tee -a $CI_PROJECT_DIR/image.env
+        export FLOATING_IP=$(openstack floating ip create uab-campus -f value -c floating_ip_address)
+        echo FLOATING_IP=$FLOATING_IP | tee -a $CI_PROJECT_DIR/image.env
+        sed -i -E "s/(ood_servername: ).*/\1\"$CI_COMMIT_REF_SLUG.$FLOATING_IP.nip.io\"/" CRI_XCBC/group_vars/knightly
+      elif [ $CI_PIPELINE_SOURCE == 'schedule' ]; then
+        export PKR_VAR_image_name="ood-${BUILD_DATE}"
+        echo INSTANCE_FLAVOR="${OOD_INSTANCE_FLAVOR:-cpu16-64g}" | tee -a $CI_PROJECT_DIR/image.env
+        echo OOD_INSTANCE_NAME="ood-knightly" | tee -a $CI_PROJECT_DIR/image.env
+        echo FLOATING_IP=$TEST_IP | tee -a $CI_PROJECT_DIR/image.env
+      fi
+    - >
+      PKR_VAR_build_instance_name="ood-${CRI_XCBC_HEAD}"
+      PKR_VAR_image_date_suffix=false
+      packer build -machine-readable openstack-ood | tee ood_build.log
+    - export BUILT_OOD_IMAGE_ID=$(grep 'Image:' ood_build.log | awk '{print $4}')
+    - echo BUILT_OOD_IMAGE_ID=${BUILT_OOD_IMAGE_ID} | tee -a $CI_PROJECT_DIR/image.env
+    - openstack image set --property CRI_XCBC_prod=${CRI_XCBC_prod} --property CRI_XCBC_dev=${CRI_XCBC_dev} --property PACKER_IMAGE_HEAD=${PACKER_IMAGE_HEAD} ${BUILT_OOD_IMAGE_ID}
+  artifacts:
+    reports:
+      dotenv: image.env
 
 test_ood_image:
   stage: test
+  needs: [build_ood_image]
+  environment:
+    name: knightly
+  tags:
+    - build
+  script:
+    - openstack image set --accept $BUILT_OOD_IMAGE_ID
+    - FAILED=false
+    - |
+      eval $(ssh-agent -s)
+      chmod 400 "$SSH_PRIV_KEY"
+      ssh-add "$SSH_PRIV_KEY"
+      mkdir ~/.ssh
+      chmod 700 ~/.ssh
+    - OLD_INSTANCE_IP=$(openstack floating ip list --floating-ip-address $CHEAHA_IP -c "Fixed IP Address" -f value)
+    - echo $OLD_INSTANCE_IP
+    - |
+      if [ ! -z $OLD_INSTANCE_IP ]; then
+        export OLD_INSTANCE_ID=$(openstack server list --name $OOD_INSTANCE_NAME --ip $OLD_INSTANCE_IP -c ID -f value)
+      fi
+    - echo OLD_INSTANCE_ID=$OLD_INSTANCE_ID | tee -a instance.env
+    - |
+      cat > user_data.txt << OEOF
+      #!/bin/bash
+      echo "Starting user_data: \$(date)"
+      cat > /etc/resolv.conf << EOF
+      search openstack.internal cm.cluster rc.uab.edu ib.cluster drac.cluster eth.cluster ib-hdr.cluster
+      nameserver 172.20.0.25
+      EOF
+      echo "$DEV_KEY" >> /root/.ssh/authorized_keys
+      mkdir -p /run/shibboleth
+      chown shibd:shibd /run/shibboleth
+      echo "Installing s3cmd: \$(date)"
+      pip3 install s3cmd
+      echo "Downloading hostkey via s3cmd: \$(date)"
+      s3cmd get --force -r --access_key=$AWS_ACCESS_KEY_ID --secret_key=$AWS_SECRET_ACCESS_KEY --host=$AWS_HOST --host-bucket=$AWS_HOST s3://knightly-key/ /etc/ssh/
+      echo "Download completed: \$(date)"
+      OEOF
+    - >
+      export NEW_INSTANCE_ID=$(openstack server create
+      -c id -f value --image $BUILT_OOD_IMAGE_ID
+      --network $OOD_INSTANCE_NETWORK
+      --security-group ood-https-ports
+      --security-group node-exporter
+      --security-group allow-ssh
+      --user-data user_data.txt
+      --flavor $INSTANCE_FLAVOR
+      --wait
+      $OOD_INSTANCE_NAME)
+    - echo NEW_INSTANCE_ID=$NEW_INSTANCE_ID | tee -a instance.env
+    - openstack server add floating ip $NEW_INSTANCE_ID $FLOATING_IP
+    - >
+      curl --retry 10 --retry-delay 20 --retry-connrefused https://knightly.rc.uab.edu/Shibboleth.sso/Metadata --resolve knightly.rc.uab.edu:443:$FLOATING_IP -kf
+      || FAILED=true
+    - |
+      cp "$SSH_KNOWN_HOSTS" ~/.ssh/known_hosts
+      chmod 644 ~/.ssh/known_hosts
+      until ssh acctsvc@$FLOATING_IP hostname; do sleep 5; done
+      ssh acctsvc@$FLOATING_IP '[ $(mount | grep "etc/auto" | wc -l) -eq 6 ]' || FAILED=true
+    - |
+      if [ "$FAILED" = true ]; then
+        if [ "${DELETE_WHEN_FAILED-true}" = true ]; then
+          openstack server delete $NEW_INSTANCE_ID
+          echo "DELETE_BUILT_IMAGE=true" | tee -a instance.env
+        fi
+        false
+      fi
+    - openstack server remove floating ip $NEW_INSTANCE_ID $FLOATING_IP
+  artifacts:
+    reports:
+      dotenv: instance.env
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+      when: always
+    - if: $CI_PIPELINE_SOURCE == "manual"
+      when: manual
+
+test_ood_image_mr:
+  stage: test
+  needs: [build_ood_image]
+  tags:
+    - build
+  script:
+    - export OOD_INSTANCE_NETWORK="cicd-net"
+    - FAILED=false
+    - |
+      eval $(ssh-agent -s)
+      chmod 400 "$SSH_PRIV_KEY"
+      ssh-add "$SSH_PRIV_KEY"
+      mkdir ~/.ssh
+      chmod 700 ~/.ssh
+    - |
+      cat > user_data.txt << OEOF
+      #!/bin/bash
+      cat > /etc/resolv.conf << EOF
+      search openstack.internal cm.cluster rc.uab.edu ib.cluster drac.cluster eth.cluster ib-hdr.cluster
+      nameserver 172.20.0.25
+      EOF
+      echo "$DEV_KEY" >> /root/.ssh/authorized_keys
+      mkdir -p /run/shibboleth
+      chown shibd:shibd /run/shibboleth
+      OEOF
+    - >
+      export NEW_INSTANCE_ID=$(openstack server create
+      -c id -f value --image $BUILT_OOD_IMAGE_ID
+      --network $OOD_INSTANCE_NETWORK
+      --security-group ood-https-ports
+      --security-group allow-ssh
+      --user-data user_data.txt
+      --flavor $INSTANCE_FLAVOR
+      --wait
+      $OOD_INSTANCE_NAME)
+    - echo NEW_INSTANCE_ID=$NEW_INSTANCE_ID | tee -a instance.env
+    - openstack server add floating ip $NEW_INSTANCE_ID $FLOATING_IP
+    - >
+      curl --retry 10 --retry-delay 20 --retry-connrefused https://knightly.rc.uab.edu/Shibboleth.sso/Metadata --resolve knightly.rc.uab.edu:443:$FLOATING_IP -kf
+      || FAILED=true
+    - ssh -o StrictHostKeyChecking=no acctsvc@$FLOATING_IP '[ $(mount | grep "etc/auto" | wc -l) -eq 6 ]' || FAILED=true
+    - |
+      if [ "$FAILED" = true ]; then
+        if [ "${DELETE_WHEN_FAILED-true}" = true ]; then
+          openstack server delete $NEW_INSTANCE_ID
+          openstack image delete $BUILT_OOD_IMAGE_ID
+        fi
+        false
+      fi
+  artifacts:
+    reports:
+      dotenv: instance.env
+  rules:
+    - if: $CI_MERGE_REQUEST_ID
+    - if: $CI_PIPELINE_SOURCE == "manual"
+      when: manual
+
+deploy_review:
+  stage: deploy
+  script:
+    - echo "Deploy Review App"
+  environment:
+    name: review/$CI_COMMIT_REF_SLUG
+    url: https://$CI_COMMIT_REF_SLUG.$FLOATING_IP.nip.io
+    on_stop: stop_review
+    auto_stop_in: 2 days
+  tags:
+    - build
+  rules:
+    - if: $CI_MERGE_REQUEST_ID
+    - if: $CI_PIPELINE_SOURCE == "manual"
+      when: manual
+
+stop_review:
+  stage: deploy
+  script:
+    - openstack server delete $NEW_INSTANCE_ID
+    - openstack image delete $BUILT_OOD_IMAGE_ID
+    - openstack floating ip delete $FLOATING_IP
+  environment:
+    name: review/$CI_COMMIT_REF_SLUG
+    action: stop
+  tags:
+    - build
+  rules:
+    - if: $CI_MERGE_REQUEST_ID
+      when: manual
+    - if: $CI_PIPELINE_SOURCE == "manual"
+      when: manual
+
+deploy_knightly:
+  stage: deploy
+  environment:
+    name: knightly
+  tags:
+    - build
+  script:
+    - |
+      if [ ! -z $OLD_INSTANCE_ID ]; then
+        openstack server remove floating ip $OLD_INSTANCE_ID $CAMPUS_IP
+        openstack server remove floating ip $OLD_INSTANCE_ID $CHEAHA_IP
+      fi
+    - |
+      if [ ! -z $NEW_INSTANCE_ID ]; then
+        openstack server add floating ip $NEW_INSTANCE_ID $CAMPUS_IP
+        openstack server add floating ip $NEW_INSTANCE_ID $CHEAHA_IP
+      fi
+  only:
+    - schedules
+
+deploy_cheaha:
+  stage: deploy
+  environment:
+    name: cheaha
+  tags:
+    - build
+  script:
+    - echo "Job placeholder to deploy to Cheaha"
+  when: manual
+  only:
+    - main
+
+cleanup_knightly:
+  stage: cleanup
+  environment:
+    name: knightly
   tags:
-    - test
+    - build
   script:
-    - export INSTANCE_NAME="ood-${CRI_XCBC_HEAD}-${BUILD_DATE}"
-    - openstack server create --image ${BUILT_OOD_IMAGE_ID} --flavor $OOD_FLAVOR --key-name "root" --network $OOD_INSTANCE_NETWORK $INSTANCE_NAME
-    - openstack floating ip create $OOD_FLOATING_IP_NETWORK
-    - export OOD_FLOATING_IP=$(openstack floating ip list --status DOWN -c "Floating IP Address" -f value | head -n 1)
-    - openstack server add floating ip $INSTANCE_NAME $OOD_FLOATING_IP
-    - ssh -o StrictHostKeyChecking=no centos@$OOD_FLOATING_IP 'hostname'
-    - ssh -o StrictHostKeyChecking=no centos@$OOD_FLOATING_IP '/opt/ood_scripts/test_ood.sh'
+    - >
+      SERVER_TO_BE_DELETE=($(openstack server list --name $OOD_INSTANCE_NAME --sort-column Image --sort-descending -f value -c ID
+      | awk -v NSTK=$NUM_SERVER_TO_KEEP -v OID=$OLD_INSTANCE_ID '$0 != OID {count++}
+      $0 != OID && count>NSTK {print}'))
+    - |
+      for svr in ${SERVER_TO_BE_DELETE[@]}; do
+        echo "Deleting server $svr"
+        openstack server delete ${svr}
+      done
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+      when: always
+    - if: $CI_PIPELINE_SOURCE == "manual"
+      when: manual
+
+cleanup_integration:
+  stage: cleanup
+  tags:
+    - build
+  script:
+    - OS_PROJECT_ID=$(openstack application credential show $OS_APPLICATION_CREDENTIAL_ID -f value -c project_id)
+    - openstack image list --sort-column Name --sort-descending -f value -c Name -c ID --property owner=$OS_PROJECT_ID > images.txt
+    - |
+      if [ "${DELETE_BUILT_IMAGE-false}" = true ]; then
+        openstack image delete $BUILT_OOD_IMAGE_ID
+      fi
+    - >
+      OOD_IMAGE_TO_BE_DELETE=($(cat images.txt
+      | awk -v NITK=$NUM_IMAGE_TO_KEEP -v REGEX=ood-$TIMESTAMP_REGEX
+      '{if ($0 ~ REGEX) result[count++] = $1}
+      END {for(i=NITK;i<count;i++) print result[i]}'))
+    - >
+      BASE_IMAGE_TO_BE_DELETE=($(cat images.txt
+      | awk -v NITK=$NUM_IMAGE_TO_KEEP -v REGEX=base-$TIMESTAMP_REGEX
+      '{if ($0 ~ REGEX) result[count++] = $1}
+      END {for(i=NITK;i<count;i++) print result[i]}'))
+    - >
+      COMPUTE_IMAGE_TO_BE_DELETE=($(cat images.txt
+      | awk -v NITK=$NUM_IMAGE_TO_KEEP -v REGEX=compute-$TIMESTAMP_REGEX
+      '{if ($0 ~ REGEX) result[count++] = $1}
+      END {for(i=NITK;i<count;i++) print result[i]}'))
+    - >
+      GPU_IMAGE_TO_BE_DELETE=($(cat images.txt
+      | awk -v NITK=$NUM_IMAGE_TO_KEEP -v REGEX=gpu-$TIMESTAMP_REGEX
+      '{if ($0 ~ REGEX) result[count++] = $1}
+      END {for(i=NITK;i<count;i++) print result[i]}'))
+    - |
+      for img in ${OOD_IMAGE_TO_BE_DELETE[@]}; do
+        echo "Deleting image $img"
+        openstack image delete ${img}
+      done
+    - |
+      for img in ${BASE_IMAGE_TO_BE_DELETE[@]}; do
+        echo "Deleting image $img"
+        openstack image delete ${img}
+      done
+    - |
+      for img in ${COMPUTE_IMAGE_TO_BE_DELETE[@]}; do
+        echo "Deleting image $img"
+        openstack image delete ${img}
+      done
+    - |
+      for img in ${GPU_IMAGE_TO_BE_DELETE[@]}; do
+        echo "Deleting image $img"
+        openstack image delete ${img}
+      done
   rules:
-    - when: manual  # Add manual trigger
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+      when: always
+    - if: $CI_PIPELINE_SOURCE == "manual"
+      when: manual
 
-cleanup:
+cleanup_mr:
   stage: cleanup
   tags:
-    - cleanup
+    - build
   script:
-    - openstack server delete $INSTANCE_NAME
-    - openstack floating ip delete $OOD_FLOATING_IP
+    - OS_PROJECT_ID=$(openstack application credential show $OS_APPLICATION_CREDENTIAL_ID -f value -c project_id)
+    - >
+      IMAGE_TO_BE_DELETE=($(openstack image list --sort-column Name --sort-descending -f value -c Name -c ID --property owner=$OS_PROJECT_ID
+      | awk -v REGEX="(ood|base|compute|gpu)-PR-$CI_MERGE_REQUEST_IID" '{if ($0 ~ REGEX) print $1}'))
+    - |
+      for img in ${IMAGE_TO_BE_DELETE[@]}; do
+        echo "Deleting image $img"
+        openstack image delete ${img}
+      done
   rules:
-    - when: manual  # Add manual trigger
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      when: always
+