.gitlab-ci.yml

+default:
+  image: $CI_REGISTRY_IMAGE:latest
+
+variables:
+  CAMPUS_IP: 138.26.48.47
+  CHEAHA_IP: 172.20.10.9
+  TEST_IP: 138.26.49.134
+  ANSIBLE_REMOTE_TMP: "/tmp"
+  AWS_DEFAULT_REGION: "bhm"
+  AWS_HOST: "s3.lts.rc.uab.edu"
+  FF_SCRIPT_SECTIONS: "true"
+  OS_AUTH_TYPE: "v3applicationcredential"
+  OS_AUTH_URL: "https://keystone.cloud.rc.uab.edu:5000/v3"
+  OS_IDENTITY_API_VERSION: "3"
+  OS_INTERFACE: "public"
+  OS_REGION_NAME: "bhm1"
+  OOD_INSTANCE_NETWORK: "knightly-network"
+  PKR_VAR_flavor: "m1.medium-ruffner"
+  PKR_VAR_source_image: "CentOS-7-x86_64-GenericCloud-2009"
+  PKR_VAR_floating_ip_network: "uab-campus"
+  PKR_VAR_security_groups: '["allow-ssh"]'
+  PKR_VAR_skip_create_image: "false"
+  PKR_VAR_ssh_username: "centos"
+  PKR_VAR_networks: '["8cf2f12e-905d-46d9-bc70-b0897c65f75a"]'
+  PKR_VAR_image_membership: '["cf6fa1e53d4c40a49f4e0e469c440359"]'
+  GIT_AUTHOR_NAME: "Gitlab runner"
+  GIT_AUTHOR_EMAIL: "gitlab@runner"
+  NUM_SERVER_TO_KEEP: 1
+  NUM_IMAGE_TO_KEEP: 30
+  TIMESTAMP_REGEXP: '[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{6}'
+  PKR_VAR_root_ssh_key: "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAFqqWgmYpEaGtHBeTu27ntVJpYjwq/x5aBefrvfhk8Z9lE3cuZ26vJ9n/9tGE4Zn2Pew1mpZgi6PzfJ3vMt8yA= root@master"
+  DEV_KEY: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCpncAcYosVHt7HsUcE2XOYDuCi4HQnmFJv279LOcpZgXtZ6o0BM1fe5FgJS0X1ohBXQUFRuYJuJSW/GSmC1K8T+wCrKjZLJdMbqrubHV27diUZfdoVkoJy1vcAQF5nEcoTC7MpAFbBomdn2rsrpgQe8DGiURV7+soqybXV1OsIR3FFf6npnUaskHYT/oVtG9eBOnscyBxoVgbxzlmyoBLXED/sHKFw4nQSF/glYKEFiDu6TRTsBBEGvv23Qo/66QpQiFJ6TNfApNiyY9L1X+Dy8EWU6lozmNgwGDjXQ70Lr6xHnA0QGVALJlHXa6QjpgtpC5Nefsdvtf1hpfFo2VutpbSB+aq9jk3gWNN+XkhrWN5PiwP7YYJNw/WozyfL+IhwjfHZGxkuws+wGR6ZKxlX9W9Vrsq9ncYNKuhy2SdsR6s2XECQtrEQ6ZlX5jRt6Yh5M9ls5fMsWEqknDPmr1Ui6wV7NxprYngo9fLSdYO/ETIO3S6PB0aEHOZOyGitGaM06EmNpvjQn/QkkaVgt/O8wKL1o1AVzXhDMAFvtG6ejppV6kuTUHXFgSGZF6N9fnP91HuytyzC09F+NMWcmnRdrgXlHapjuuL3zzi+XLCQvk8+aYTzBKx1nU2FPMDRZ9sInGmqdTuM002E7qVbaCy4OxcWaAS/L2UVhGnHr+egYw== louistw@uab.edu"
+
+stages:
+  - pre-build
+  - build
+  - test
+  - deploy
+  - cleanup
+
+workflow:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+    - if: $CI_PIPELINE_SOURCE == 'schedule'
+
+.get_build_date: &get_build_date
+  - export BUILD_DATE=$(TZ=America/Chicago date +%Y-%m-%dT%H%M%S)
+  - echo BUILD_DATE=${BUILD_DATE}
+
+.update_ansible_repo: &update_ansible_repo
+  - *get_build_date
+  - |
+    if [ ! -d $CI_PROJECT_DIR/CRI_XCBC ]; then
+      git clone https://github.com/uabrc/CRI_XCBC.git
+      cd CRI_XCBC
+      git remote add upstream https://github.com/jprorama/CRI_XCBC.git
+      cd ..
+    fi
+  - cd CRI_XCBC
+  - git config user.name "${GIT_AUTHOR_NAME}"
+  - git config user.email "${GIT_AUTHOR_EMAIL}"
+  - git fetch origin uab-prod
+  - git fetch upstream dev
+  - git checkout uab-prod
+  - git merge origin/uab-prod
+  - git checkout -b integration
+  - git merge upstream/dev
+  - export CRI_XCBC_HEAD=$(git rev-parse --short HEAD)
+  - export CRI_XCBC_dev=$(git rev-parse --short upstream/dev)
+  - export CRI_XCBC_prod=$(git rev-parse --short origin/uab-prod)
+  - cd ..
+  - export PACKER_IMAGE_HEAD=$(git rev-parse --short HEAD)
+  - echo CRI_XCBC_HEAD=${CRI_XCBC_HEAD} | tee -a $CI_PROJECT_DIR/image.env
+  - echo CRI_XCBC_dev=${CRI_XCBC_dev} | tee -a $CI_PROJECT_DIR/image.env
+  - echo CRI_XCBC_prod=${CRI_XCBC_prod} | tee -a $CI_PROJECT_DIR/image.env
+  - echo PACKER_IMAGE_HEAD=${PACKER_IMAGE_HEAD} | tee -a $CI_PROJECT_DIR/image.env
+
+.get_ansible_files: &get_ansible_files
+  - s3cmd get --force -r --host=$AWS_HOST --host-bucket=$AWS_HOST s3://cheaha-cloud-ansible-files/ ansible/files/
+
+build_docker_image:
+  image: docker:20.10.17
+  stage: pre-build
+  services:
+    - docker:20.10.16-dind
+  tags:
+    - dind
+  before_script:
+    - *get_build_date
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+  script:
+    - docker build -t $CI_REGISTRY_IMAGE:$BUILD_DATE -t $CI_REGISTRY_IMAGE:latest .
+    - >
+      docker run --rm $CI_REGISTRY_IMAGE bash -c
+      'ansible --version &&
+      openstack --version &&
+      packer version &&
+      s3cmd --version &&
+      terraform --version'
+    - docker push --all-tags $CI_REGISTRY_IMAGE
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+        - Dockerfile
+      allow_failure: true
+
+build_base_image:
+  stage: build
+  tags:
+    - build
+  script:
+    - |
+      if [ -n "${BUILT_BASE_IMAGE_ID}" ]; then
+        exit 0
+      fi
+    - *update_ansible_repo
+    - *get_ansible_files
+    - export REPO_HEAD=$(git rev-parse --short HEAD)
+    - export PKR_VAR_flavor="${BASE_BUILD_FLAVOR:-$PKR_VAR_flavor}"
+    - export PKR_VAR_build_instance_name="base-${REPO_HEAD}"
+    - export PKR_VAR_image_date_suffix=false
+    - |
+      if [ $CI_PIPELINE_SOURCE == 'merge_request_event' ]; then
+        export PKR_VAR_image_name="base-PR-${CI_MERGE_REQUEST_IID}"
+      elif [ $CI_PIPELINE_SOURCE == 'schedule' ]; then
+        export PKR_VAR_image_name="base-${BUILD_DATE}"
+      fi
+    - packer init openstack
+    - packer validate openstack
+    - packer build -machine-readable openstack | tee base_build.log
+    - export BUILT_BASE_IMAGE_ID=$(grep 'Image:' base_build.log | awk '{print $4}')
+    - echo BUILT_BASE_IMAGE_ID=${BUILT_BASE_IMAGE_ID} | tee -a $CI_PROJECT_DIR/image.env
+    - openstack image unset --property signature_verified $BUILT_BASE_IMAGE_ID
+  artifacts:
+    reports:
+      dotenv: image.env
+    expire_in: 30 days
+
+build_compute_image:
+  stage: build
+  needs: [build_base_image]
+  tags:
+    - build
+  script:
+    - *update_ansible_repo
+    - *get_ansible_files
+    - export PKR_VAR_source_image=${BUILT_BASE_IMAGE_ID}
+    - export REPO_HEAD=$(git rev-parse --short HEAD)
+    - export PKR_VAR_flavor="${COMPUTE_BUILD_FLAVOR:-$PKR_VAR_flavor}"
+    - export PKR_VAR_build_instance_name="compute-${REPO_HEAD}"
+    - export PKR_VAR_image_date_suffix=false
+    - |
+      if [ $CI_PIPELINE_SOURCE == 'merge_request_event' ]; then
+        export PKR_VAR_image_name="compute-PR-${CI_MERGE_REQUEST_IID}"
+      elif [ $CI_PIPELINE_SOURCE == 'schedule' ]; then
+        export PKR_VAR_image_name="compute-${BUILD_DATE}"
+      fi
+    - packer init openstack-compute
+    - packer validate openstack-compute
+    - packer build -machine-readable openstack-compute | tee compute_build.log
+
+build_gpu_image:
+  stage: build
+  needs: [build_base_image]
+  tags:
+    - build
+  script:
+    - *update_ansible_repo
+    - *get_ansible_files
+    - FAILED=false
+    - export GPU_PLACEHOLDER_NAME="gpu1-placeholder"
+    - export GPU_PLACEHOLDER_FLAVOR="gpu1.medium"
+    - export GPU_PLACEHOLDER_IMAGE="CentOS-7-x86_64-GenericCloud-2009"
+    - export PKR_VAR_source_image=${BUILT_BASE_IMAGE_ID}
+    - export REPO_HEAD=$(git rev-parse --short HEAD)
+    - export PKR_VAR_flavor="${GPU_BUILD_FLAVOR:-gpu1.medium}"
+    - export PKR_VAR_build_instance_name="gpu-${REPO_HEAD}"
+    - export PKR_VAR_image_date_suffix=false
+    - |
+      if [ $CI_PIPELINE_SOURCE == 'merge_request_event' ]; then
+        export PKR_VAR_image_name="gpu-PR-${CI_MERGE_REQUEST_IID}"
+      elif [ $CI_PIPELINE_SOURCE == 'schedule' ]; then
+        export PKR_VAR_image_name="gpu-${BUILD_DATE}"
+      fi
+    - packer init openstack-gpu
+    - packer validate openstack-gpu
+    - openstack server delete --wait $GPU_PLACEHOLDER_NAME
+    - packer build -machine-readable openstack-gpu | tee gpu_build.log || FAILED=true
+    - openstack server create --image $GPU_PLACEHOLDER_IMAGE --network cicd-net --flavor $GPU_PLACEHOLDER_FLAVOR $GPU_PLACEHOLDER_NAME
+    - |
+      if [ "$FAILED" = true ]; then
+        exit 1
+      fi
+  rules:
+    - if: $SKIP_GPU_BUILD == "true"
+      when: never
+    - when: always
+
+build_ood_image:
+  stage: build
+  tags:
+    - build
+  script:
+    - *update_ansible_repo
+    - *get_ansible_files
+    - >
+      curl --header "PRIVATE-TOKEN: ${ANSIBLE_VAR_TOKEN}"
+      "${CI_API_V4_URL}/projects/2836/repository/files/knightly/raw?ref=main"
+      -o CRI_XCBC/group_vars/knightly
+    - 'sed -i -E "s/(lts_access_key: ).*/\1\"${AWS_ACCESS_KEY_ID}\"/" CRI_XCBC/group_vars/knightly'
+    - 'sed -i -E "s/(lts_secret_key: ).*/\1\"${AWS_SECRET_ACCESS_KEY}\"/" CRI_XCBC/group_vars/knightly'
+    - 'sed -i -E "s/(user_register_app_key: ).*/\1\"${SELF_REG_APP_KEY}\"/" CRI_XCBC/group_vars/knightly'
+    - 'sed -i -E "s/(celery_user_password: ).*/\1\"${CELERY_PASSWD}\"/" CRI_XCBC/group_vars/knightly'
+    - 'sed -i -E "s|(ssh_pub_key: ).*|\1\"{{ lookup(''file'', ''${SSH_PUB_KEY}'') }}\"|" CRI_XCBC/group_vars/knightly'
+    - export PKR_VAR_flavor="${OOD_BUILD_FLAVOR:-$PKR_VAR_flavor}"
+    - packer init openstack-ood
+    - packer validate openstack-ood
+    - |
+      if [ $CI_PIPELINE_SOURCE == 'merge_request_event' ]; then
+        export PKR_VAR_image_name="ood-PR-${CI_MERGE_REQUEST_IID}"
+        echo INSTANCE_FLAVOR="${PKR_VAR_flavor}" | tee -a $CI_PROJECT_DIR/image.env
+        echo OOD_INSTANCE_NAME="ood-PR-${CI_MERGE_REQUEST_IID}" | tee -a $CI_PROJECT_DIR/image.env
+        export FLOATING_IP=$(openstack floating ip create uab-campus -f value -c floating_ip_address)
+        echo FLOATING_IP=$FLOATING_IP | tee -a $CI_PROJECT_DIR/image.env
+        sed -i -E "s/(ood_servername: ).*/\1\"$CI_COMMIT_REF_SLUG.$FLOATING_IP.nip.io\"/" CRI_XCBC/group_vars/knightly
+      elif [ $CI_PIPELINE_SOURCE == 'schedule' ]; then
+        export PKR_VAR_image_name="ood-${BUILD_DATE}"
+        echo INSTANCE_FLAVOR="${OOD_INSTANCE_FLAVOR:-cpu16-64g}" | tee -a $CI_PROJECT_DIR/image.env
+        echo OOD_INSTANCE_NAME="ood-knightly" | tee -a $CI_PROJECT_DIR/image.env
+        echo FLOATING_IP=$TEST_IP | tee -a $CI_PROJECT_DIR/image.env
+      fi
+    - >
+      PKR_VAR_build_instance_name="ood-${CRI_XCBC_HEAD}"
+      PKR_VAR_image_date_suffix=false
+      packer build -machine-readable openstack-ood | tee ood_build.log
+    - export BUILT_OOD_IMAGE_ID=$(grep 'Image:' ood_build.log | awk '{print $4}')
+    - echo BUILT_OOD_IMAGE_ID=${BUILT_OOD_IMAGE_ID} | tee -a $CI_PROJECT_DIR/image.env
+    - openstack image set --property CRI_XCBC_prod=${CRI_XCBC_prod} --property CRI_XCBC_dev=${CRI_XCBC_dev} --property PACKER_IMAGE_HEAD=${PACKER_IMAGE_HEAD} ${BUILT_OOD_IMAGE_ID}
+  artifacts:
+    reports:
+      dotenv: image.env
+
+test_ood_image:
+  stage: test
+  needs: [build_ood_image]
+  environment:
+    name: knightly
+  tags:
+    - build
+  script:
+    - openstack image set --accept $BUILT_OOD_IMAGE_ID
+    - FAILED=false
+    - |
+      eval $(ssh-agent -s)
+      chmod 400 "$SSH_PRIV_KEY"
+      ssh-add "$SSH_PRIV_KEY"
+      mkdir ~/.ssh
+      chmod 700 ~/.ssh
+    - OLD_INSTANCE_IP=$(openstack floating ip list --floating-ip-address $CHEAHA_IP -c "Fixed IP Address" -f value)
+    - echo $OLD_INSTANCE_IP
+    - |
+      if [ ! -z $OLD_INSTANCE_IP ]; then
+        export OLD_INSTANCE_ID=$(openstack server list --name $OOD_INSTANCE_NAME --ip $OLD_INSTANCE_IP -c ID -f value)
+      fi
+    - echo OLD_INSTANCE_ID=$OLD_INSTANCE_ID | tee -a instance.env
+    - |
+      cat > user_data.txt << OEOF
+      #!/bin/bash
+      echo "Starting user_data: \$(date)"
+      cat > /etc/resolv.conf << EOF
+      search openstack.internal cm.cluster rc.uab.edu ib.cluster drac.cluster eth.cluster ib-hdr.cluster
+      nameserver 172.20.0.25
+      EOF
+      echo "$DEV_KEY" >> /root/.ssh/authorized_keys
+      mkdir -p /run/shibboleth
+      chown shibd:shibd /run/shibboleth
+      echo "Installing s3cmd: \$(date)"
+      pip3 install s3cmd
+      echo "Downloading hostkey via s3cmd: \$(date)"
+      s3cmd get --force -r --access_key=$AWS_ACCESS_KEY_ID --secret_key=$AWS_SECRET_ACCESS_KEY --host=$AWS_HOST --host-bucket=$AWS_HOST s3://knightly-key/ /etc/ssh/
+      echo "Download completed: \$(date)"
+      OEOF
+    - >
+      export NEW_INSTANCE_ID=$(openstack server create
+      -c id -f value --image $BUILT_OOD_IMAGE_ID
+      --network $OOD_INSTANCE_NETWORK
+      --security-group ood-https-ports
+      --security-group node-exporter
+      --security-group allow-ssh
+      --user-data user_data.txt
+      --flavor $INSTANCE_FLAVOR
+      --wait
+      $OOD_INSTANCE_NAME)
+    - echo NEW_INSTANCE_ID=$NEW_INSTANCE_ID | tee -a instance.env
+    - openstack server add floating ip $NEW_INSTANCE_ID $FLOATING_IP
+    - >
+      curl --retry 10 --retry-delay 20 --retry-connrefused https://knightly.rc.uab.edu/Shibboleth.sso/Metadata --resolve knightly.rc.uab.edu:443:$FLOATING_IP -kf
+      || FAILED=true
+    - |
+      cp "$SSH_KNOWN_HOSTS" ~/.ssh/known_hosts
+      chmod 644 ~/.ssh/known_hosts
+      until ssh acctsvc@$FLOATING_IP hostname; do sleep 5; done
+      ssh acctsvc@$FLOATING_IP '[ $(mount | grep "etc/auto" | wc -l) -eq 6 ]' || FAILED=true
+    - |
+      if [ "$FAILED" = true ]; then
+        if [ "${DELETE_WHEN_FAILED-true}" = true ]; then
+          openstack server delete $NEW_INSTANCE_ID
+          echo "DELETE_BUILT_IMAGE=true" | tee -a instance.env
+        fi
+        false
+      fi
+    - openstack server remove floating ip $NEW_INSTANCE_ID $FLOATING_IP
+  artifacts:
+    reports:
+      dotenv: instance.env
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+      when: always
+
+test_ood_image_mr:
+  stage: test
+  needs: [build_ood_image]
+  tags:
+    - build
+  script:
+    - export OOD_INSTANCE_NETWORK="cicd-net"
+    - FAILED=false
+    - |
+      eval $(ssh-agent -s)
+      chmod 400 "$SSH_PRIV_KEY"
+      ssh-add "$SSH_PRIV_KEY"
+      mkdir ~/.ssh
+      chmod 700 ~/.ssh
+    - |
+      cat > user_data.txt << OEOF
+      #!/bin/bash
+      cat > /etc/resolv.conf << EOF
+      search openstack.internal cm.cluster rc.uab.edu ib.cluster drac.cluster eth.cluster ib-hdr.cluster
+      nameserver 172.20.0.25
+      EOF
+      echo "$DEV_KEY" >> /root/.ssh/authorized_keys
+      mkdir -p /run/shibboleth
+      chown shibd:shibd /run/shibboleth
+      OEOF
+    - >
+      export NEW_INSTANCE_ID=$(openstack server create
+      -c id -f value --image $BUILT_OOD_IMAGE_ID
+      --network $OOD_INSTANCE_NETWORK
+      --security-group ood-https-ports
+      --security-group allow-ssh
+      --user-data user_data.txt
+      --flavor $INSTANCE_FLAVOR
+      --wait
+      $OOD_INSTANCE_NAME)
+    - echo NEW_INSTANCE_ID=$NEW_INSTANCE_ID | tee -a instance.env
+    - openstack server add floating ip $NEW_INSTANCE_ID $FLOATING_IP
+    - >
+      curl --retry 10 --retry-delay 20 --retry-connrefused https://knightly.rc.uab.edu/Shibboleth.sso/Metadata --resolve knightly.rc.uab.edu:443:$FLOATING_IP -kf
+      || FAILED=true
+    - ssh -o StrictHostKeyChecking=no acctsvc@$FLOATING_IP '[ $(mount | grep "etc/auto" | wc -l) -eq 6 ]' || FAILED=true
+    - |
+      if [ "$FAILED" = true ]; then
+        if [ "${DELETE_WHEN_FAILED-true}" = true ]; then
+          openstack server delete $NEW_INSTANCE_ID
+          openstack image delete $BUILT_OOD_IMAGE_ID
+        fi
+        false
+      fi
+  artifacts:
+    reports:
+      dotenv: instance.env
+  rules:
+    - if: $CI_MERGE_REQUEST_ID
+
+deploy_review:
+  stage: deploy
+  script:
+    - echo "Deploy Review App"
+  environment:
+    name: review/$CI_COMMIT_REF_SLUG
+    url: https://$CI_COMMIT_REF_SLUG.$FLOATING_IP.nip.io
+    on_stop: stop_review
+    auto_stop_in: 2 days
+  tags:
+    - build
+  rules:
+    - if: $CI_MERGE_REQUEST_ID
+
+stop_review:
+  stage: deploy
+  script:
+    - openstack server delete $NEW_INSTANCE_ID
+    - openstack image delete $BUILT_OOD_IMAGE_ID
+    - openstack floating ip delete $FLOATING_IP
+  environment:
+    name: review/$CI_COMMIT_REF_SLUG
+    action: stop
+  tags:
+    - build
+  rules:
+    - if: $CI_MERGE_REQUEST_ID
+      when: manual
+
+deploy_knightly:
+  stage: deploy
+  environment:
+    name: knightly
+  tags:
+    - build
+  script:
+    - |
+      if [ ! -z $OLD_INSTANCE_ID ]; then
+        openstack server remove floating ip $OLD_INSTANCE_ID $CAMPUS_IP
+        openstack server remove floating ip $OLD_INSTANCE_ID $CHEAHA_IP
+      fi
+    - |
+      if [ ! -z $NEW_INSTANCE_ID ]; then
+        openstack server add floating ip $NEW_INSTANCE_ID $CAMPUS_IP
+        openstack server add floating ip $NEW_INSTANCE_ID $CHEAHA_IP
+      fi
+  only:
+    - schedules
+
+deploy_cheaha:
+  stage: deploy
+  environment:
+    name: cheaha
+  tags:
+    - build
+  script:
+    - echo "Job placeholder to deploy to Cheaha"
+  when: manual
+  only:
+    - main
+
+cleanup_knightly:
+  stage: cleanup
+  environment:
+    name: knightly
+  tags:
+    - build
+  script:
+    - >
+      SERVER_TO_BE_DELETE=($(openstack server list --name $OOD_INSTANCE_NAME --sort-column Image --sort-descending -f value -c ID
+      | awk -v NSTK=$NUM_SERVER_TO_KEEP -v OID=$OLD_INSTANCE_ID '$0 != OID {count++}
+      $0 != OID && count>NSTK {print}'))
+    - |
+      for svr in ${SERVER_TO_BE_DELETE[@]}; do
+        echo "Deleting server $svr"
+        openstack server delete ${svr}
+      done
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+      when: always
+
+cleanup_integration:
+  stage: cleanup
+  tags:
+    - build
+  script:
+    - OS_PROJECT_ID=$(openstack application credential show $OS_APPLICATION_CREDENTIAL_ID -f value -c project_id)
+    - openstack image list --sort-column Name --sort-descending -f value -c Name -c ID --property owner=$OS_PROJECT_ID > images.txt
+    - |
+      if [ "${DELETE_BUILT_IMAGE-false}" = true ]; then
+        openstack image delete $BUILT_OOD_IMAGE_ID
+      fi
+    - >
+      OOD_IMAGE_TO_BE_DELETE=($(cat images.txt
+      | awk -v NITK=$NUM_IMAGE_TO_KEEP -v REGEX=ood-$TIMESTAMP_REGEX
+      '{if ($0 ~ REGEX) result[count++] = $1}
+      END {for(i=NITK;i<count;i++) print result[i]}'))
+    - >
+      BASE_IMAGE_TO_BE_DELETE=($(cat images.txt
+      | awk -v NITK=$NUM_IMAGE_TO_KEEP -v REGEX=base-$TIMESTAMP_REGEX
+      '{if ($0 ~ REGEX) result[count++] = $1}
+      END {for(i=NITK;i<count;i++) print result[i]}'))
+    - >
+      COMPUTE_IMAGE_TO_BE_DELETE=($(cat images.txt
+      | awk -v NITK=$NUM_IMAGE_TO_KEEP -v REGEX=compute-$TIMESTAMP_REGEX
+      '{if ($0 ~ REGEX) result[count++] = $1}
+      END {for(i=NITK;i<count;i++) print result[i]}'))
+    - >
+      GPU_IMAGE_TO_BE_DELETE=($(cat images.txt
+      | awk -v NITK=$NUM_IMAGE_TO_KEEP -v REGEX=gpu-$TIMESTAMP_REGEX
+      '{if ($0 ~ REGEX) result[count++] = $1}
+      END {for(i=NITK;i<count;i++) print result[i]}'))
+    - |
+      for img in ${OOD_IMAGE_TO_BE_DELETE[@]}; do
+        echo "Deleting image $img"
+        openstack image delete ${img}
+      done
+    - |
+      for img in ${BASE_IMAGE_TO_BE_DELETE[@]}; do
+        echo "Deleting image $img"
+        openstack image delete ${img}
+      done
+    - |
+      for img in ${COMPUTE_IMAGE_TO_BE_DELETE[@]}; do
+        echo "Deleting image $img"
+        openstack image delete ${img}
+      done
+    - |
+      for img in ${GPU_IMAGE_TO_BE_DELETE[@]}; do
+        echo "Deleting image $img"
+        openstack image delete ${img}
+      done
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+      when: always
+
+cleanup_mr:
+  stage: cleanup
+  tags:
+    - build
+  script:
+    - OS_PROJECT_ID=$(openstack application credential show $OS_APPLICATION_CREDENTIAL_ID -f value -c project_id)
+    - >
+      IMAGE_TO_BE_DELETE=($(openstack image list --sort-column Name --sort-descending -f value -c Name -c ID --property owner=$OS_PROJECT_ID
+      | awk -v REGEX="(ood|base|compute|gpu)-PR-$CI_MERGE_REQUEST_IID" '{if ($0 ~ REGEX) print $1}'))
+    - |
+      for img in ${IMAGE_TO_BE_DELETE[@]}; do
+        echo "Deleting image $img"
+        openstack image delete ${img}
+      done
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      when: always

Dockerfile

+FROM python:3.8-slim
+
+ENV S3CMD_VER=2.3.0
+ENV ANSIBLE_VER=4.10.0
+ENV OSC_VER=5.8.0
+ENV TF_VER=1.4.6
+ENV PACKER_VER=1.9.4
+
+RUN apt-get update && apt-get install --no-install-recommends -y \
+    git \
+    ssh \
+    curl \
+    wget \
+    unzip \
+ && rm -rf /var/lib/apt/lists/*
+RUN wget https://releases.hashicorp.com/packer/${PACKER_VER}/packer_${PACKER_VER}_linux_amd64.zip \
+    && unzip packer_${PACKER_VER}_linux_amd64.zip -d /usr/local/bin \
+    && rm packer_${PACKER_VER}_linux_amd64.zip
+RUN wget https://releases.hashicorp.com/terraform/${TF_VER}/terraform_${TF_VER}_linux_amd64.zip \
+    && unzip terraform_${TF_VER}_linux_amd64.zip -d /usr/local/bin \
+    && rm terraform_${TF_VER}_linux_amd64.zip
+RUN pip install --no-cache-dir --upgrade pip \
+ && pip install --no-cache-dir \
+    s3cmd==$S3CMD_VER \
+    ansible==$ANSIBLE_VER \
+    python-openstackclient==$OSC_VER

ansible/base.yml

+---
+- name: Install base packages
+  hosts: default
+  become: true
+  roles:
+    - { name: 'fix_centos_repo', tags: 'fix_centos_repo' }
+    - { name: 'install_packages', tags: 'install_packages' }

ansible/cheaha.yml

+---
+- name: Setup node for use as a virtual cheaha node
+  hosts: default
+  become: true
+  roles:
+    - { name: 'cheaha.node', tags: 'cheaha.node' }
+    - { name: 'nfs_mounts', tags: 'nfs_mounts' }
+    - { name: 'ldap_config', tags: 'ldap_config' }
+    - { name: 'slurm_client', tags: 'slurm_client' }

ansible/compute.yml

+---
+- name: Setup node for use as a virtual compute node
+  hosts: default
+  become: true
+  roles:
+    - { name: 'fix_centos_repo', tags: 'fix_centos_repo' }
+    - { name: 'install_packages', tags: 'install_packages' }
+    - { name: 'pam_slurm_adopt', tags: 'pam_slurm_adopt' }
+    - { name: 'install_nhc', tags: 'install_nhc'}
+
+- name: Setup node for use as a virtual cheaha node
+  ansible.builtin.import_playbook: cheaha.yml

ansible/files/TurboVNC.repo

+[TurboVNC]
+name=TurboVNC official RPMs
+baseurl=https://sourceforge.net/projects/turbovnc/files
+gpgcheck=1
+gpgkey=https://sourceforge.net/projects/turbovnc/files/VGL-GPG-KEY
+       https://sourceforge.net/projects/turbovnc/files/VGL-GPG-KEY-1024
+enabled=1

ansible/files/build-zsh.sh

+#!/bin/bash
+name=zsh
+version=5.9
+
+mkdir -p BUILD RPMS SOURCES SPECS SRPMS
+
+prereqs="git epel-release bzip2 xz gzip tar"
+rpm -q $prereqs
+
+if [ $? -ne 0 ]; then sudo yum -y install $prereqs; fi
+
+git clone https://src.fedoraproject.org/rpms/${name}.git
+
+cp ${name}/${name}.spec SPECS/
+cp ${name}/*.rhs SOURCES/
+cp ${name}/dot* SOURCES/
+
+# Perl may not be available, switching to sed
+#perl -pi -e 's/^(BuildRequires: glibc-langpack-ja)/#$1/g;' ${name}/${name}.spec
+sed -i '/BuildRequires: glibc-langpack-ja/s/^/#/' SPECS/${name}.spec
+
+pkgs="rpm-build $(grep -E '^BuildRequires|^Requires' SPECS/${name}.spec | awk '{print $2}' | tr '\n' ' ')"; echo $pkgs
+rpm -q $pkgs
+
+if [ $? -ne 0 ]; then sudo yum install -y $pkgs; fi
+
+if [ ! -f SOURCES/${name}-${version}.tar.xz ] ; then
+  curl -L -o SOURCES/${name}-${version}.tar.xz https://downloads.sourceforge.net/${name}/${name}-${version}.tar.xz
+fi
+
+if [ ! -f SOURCES/${name}-${version}.tar.xz.asc ] ; then
+  curl -L -o SOURCES/${name}-${version}.tar.xz.asc https://downloads.sourceforge.net/${name}/${name}-${version}.tar.xz.asc
+fi
+
+cd SOURCES
+sha512sum -c ../${name}/sources
+retval=$?
+cd -
+if [ $retval -ne 0 ]; then
+#  echo "SOURCES/${name}-${version}.tar.gz did not match checksum in ${name}/source"
+  exit 1
+fi
+
+rpmbuild --define "_topdir `pwd`" -bb SPECS/${name}.spec

ansible/files/ceph.repo

+[Ceph]
+priority=1
+name=Ceph $basearch
+baseurl=https://download.ceph.com/rpm-octopus/el7/$basearch
+enabled=1
+gpgcheck=1
+gpgkey=https://download.ceph.com/keys/release.asc
+
+[Ceph-noarch]
+priority=1
+name=Ceph noarch
+baseurl=https://download.ceph.com/rpm-octopus/el7/noarch
+enabled=1
+gpgcheck=1
+gpgkey=https://download.ceph.com/keys/release.asc
+
+[Ceph-source]
+priority=1
+name=Ceph SRPMS
+baseurl=https://download.ceph.com/rpm-octopus/el7/SRPMS
+enabled=1
+gpgcheck=1
+gpgkey=https://download.ceph.com/keys/release.asc
+

ansible/files/dell-system-update.repo

+[dell-system-update_independent]
+name=dell-system-update_independent
+baseurl=https://linux.dell.com/repo/hardware/dsu/os_independent/
+gpgcheck=1
+gpgkey=https://linux.dell.com/repo/pgp_pubkeys/0x756ba70b1019ced6.asc
+      https://linux.dell.com/repo/pgp_pubkeys/0x1285491434D8786F.asc
+     https://linux.dell.com/repo/pgp_pubkeys/0xca77951d23b66a9d.asc
+                    https://linux.dell.com/repo/pgp_pubkeys/0x3CA66B4946770C59.asc
+enabled=1
+exclude=dell-system-update*.i386
+
+[dell-system-update_dependent]
+name=dell-system-update_dependent
+mirrorlist=https://linux.dell.com/repo/hardware/dsu/mirrors.cgi?osname=el$releasever&basearch=$basearch&native=1
+gpgcheck=1
+gpgkey=https://linux.dell.com/repo/pgp_pubkeys/0x756ba70b1019ced6.asc
+      https://linux.dell.com/repo/pgp_pubkeys/0x1285491434D8786F.asc
+     https://linux.dell.com/repo/pgp_pubkeys/0xca77951d23b66a9d.asc
+                    https://linux.dell.com/repo/pgp_pubkeys/0x3CA66B4946770C59.asc
+enabled=1

ansible/files/elastic.repo

+[elastic-7.x]
+name=Elastic repository for 7.x packages
+baseurl=https://artifacts.elastic.co/packages/7.x/yum
+gpgcheck=1
+gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
+enabled=1
+autorefresh=1
+type=rpm-md

ansible/files/nux-dextop.repo

+[nux-dextop]
+name=Nux.Ro RPMs for general desktop use
+baseurl=http://li.nux.ro/download/nux/dextop/el7/$basearch/
+enabled=1
+gpgcheck=1
+gpgkey=http://li.nux.ro/download/nux/RPM-GPG-KEY-nux.ro
+protect=0
+
+[nux-dextop-testing]
+name=Nux.Ro RPMs for general desktop use - testing
+baseurl=http://li.nux.ro/download/nux/dextop-testing/el7/$basearch/
+enabled=0
+gpgcheck=1
+gpgkey=http://li.nux.ro/download/nux/RPM-GPG-KEY-nux.ro
+protect=0

ansible/gpu.yml

+---
+- name: Setup node for use as a cluster host with gpu drivers/pkgs
+  hosts: default
+  become: true
+  roles:
+    - { name: 'fix_centos_repo', tags: 'fix_centos_repo' }
+    - { name: 'install_packages', tags: 'install_packages' }
+    - { name: 'cuda_driver', tags: 'cuda_driver' }
+    - { name: 'pam_slurm_adopt', tags: 'pam_slurm_adopt' }
+    - { name: 'install_nhc', tags: 'install_nhc'}
+
+- name: Setup node for use as a virtual cheaha node
+  ansible.builtin.import_playbook: cheaha.yml

ansible/group_vars/all

+---
+  zsh_ver: 5.7.1
+  zsh_src_url: "https://www.zsh.org/pub/old/zsh-{{ zsh_ver }}.tar.xz"
+  yum_repo_files: []
+  pkg_list: []
+  slurm_version: 18.08.9
+  
+# NHC related
+  nhc_download_url: "https://github.com/mej/nhc/releases/download/1.4.3/lbnl-nhc-1.4.3-1.el7.noarch.rpm"
+  nhc_download_path: "/tmp"
+  nhc_git_repo: "https://gitlab.rc.uab.edu/rc/nhc.git"
+  nhc_git_repo_path: "/tmp/nhc"

ansible/group_vars/compute

+---
+  yum_repo_files:
+    - TurboVNC.repo
+    - cm.repo
+  pkg_list:
+    - "Lmod-7.8.11"
+    - "atftp-server"
+    - "cluster-tools-dell"
+    - "cluster-tools-slave"
+    - "cm-boost"
+    - "cm-config-ceph-release-luminous"
+    - "cm-config-cm"
+    - "cm-config-dhclient"
+    - "cm-config-dracut-slave"
+    - "cm-config-grub"
+    - "cm-config-ldap-client"
+    - "cm-config-limits"
+    - "cm-config-man"
+    - "cm-config-named"
+    - "cm-config-network-slave"
+    - "cm-config-nfsclient"
+    - "cm-config-rootfiles-slave"
+    - "cm-config-selinux"
+    - "cm-config-ssh-slave"
+    - "cm-config-sysctl-slave"
+    - "cm-config-syslog-slave"
+    - "cm-config-systemd"
+    - "cm-config-xntp-slave"
+    - "cm-config-yum"
+    - "cm-curl"
+    - "cm-dhcp"
+    - "cm-freeipmi"
+    - "cm-ipmitool"
+    - "cm-ipxe-slave"
+    - "cm-libpam"
+    - "cm-libprometheus"
+    - "cm-lua"
+    - "cm-mariadb-libs"
+    - "cm-openssl"
+    - "cm-python2"
+    - "cm-python36"
+    - "cm-slave"
+    - "cm-uge-client"
+    - "cmburn"
+    - "cmburn-slave"
+    - "cmdaemon"
+    - "cmdaemon-remotecm"
+    - "confuse"
+    - "gcc-recent"
+    - "gdb-recent"
+    - "lshw"
+    - "lua-bit32"
+    - "lua-filesystem"
+    - "lua-json"
+    - "lua-lpeg"
+    - "lua-posix"
+    - "lua-term"
+    - "mysql++"
+    - "net-snmp-recent"
+    - "node-installer-slave"
+    - "openvpn"
+    - "perl-Config-IniFiles"
+    - "python-dogpile-cache"
+    - "python-isodate"
+    - "python-netaddr"
+    - "python-netifaces"
+    - "python-setuptools_scm"
+    - "python-testtools"
+    - "python-websockify"
+    - "python2-cliff"
+    - "python2-debtcollector"
+    - "python2-deprecation"
+    - "python2-fixtures"
+    - "python2-funcsigs"
+    - "python2-ipaddress"
+    - "python2-pbr"
+    - "python2-positional"
+    - "python2-pysocks"
+    - "python2-pyyaml"
+    - "python2-requests-oauthlib"
+    - "python2-requestsexceptions"
+    - "python2-rfc3986"
+    - "python2-six"
+    - "python2-stevedore"
+    - "sdparm"
+    - "sshpass"
+    - "swig"
+    - "turbovnc-2.2.6*"

ansible/group_vars/gpu

+---
+  yum_repo_files:
+    - TurboVNC.repo
+    - cm.repo
+  pkg_list:
+    - "Lmod-7.8.11"
+    - "atftp-server"
+    - "cluster-tools-dell"
+    - "cluster-tools-slave"
+    - "cm-boost"
+    - "cm-config-ceph-release-luminous"
+    - "cm-config-cm"
+    - "cm-config-dhclient"
+    - "cm-config-dracut-slave"
+    - "cm-config-grub"
+    - "cm-config-ldap-client"
+    - "cm-config-limits"
+    - "cm-config-man"
+    - "cm-config-named"
+    - "cm-config-network-slave"
+    - "cm-config-nfsclient"
+    - "cm-config-rootfiles-slave"
+    - "cm-config-selinux"
+    - "cm-config-ssh-slave"
+    - "cm-config-sysctl-slave"
+    - "cm-config-syslog-slave"
+    - "cm-config-systemd"
+    - "cm-config-xntp-slave"
+    - "cm-config-yum"
+    - "cm-curl"
+    - "cm-dhcp"
+    - "cm-freeipmi"
+    - "cm-ipmitool"
+    - "cm-ipxe-slave"
+    - "cm-libpam"
+    - "cm-libprometheus"
+    - "cm-lua"
+    - "cm-mariadb-libs"
+    - "cm-openssl"
+    - "cm-python2"
+    - "cm-python36"
+    - "cm-slave"
+    - "cm-uge-client"
+    - "cmburn"
+    - "cmburn-slave"
+    - "cmdaemon"
+    - "cmdaemon-remotecm"
+    - "confuse"
+    - "gcc-recent"
+    - "gdb-recent"
+    - "lshw"
+    - "lua-bit32"
+    - "lua-filesystem"
+    - "lua-json"
+    - "lua-lpeg"
+    - "lua-posix"
+    - "lua-term"
+    - "mysql++"
+    - "net-snmp-recent"
+    - "node-installer-slave"
+    - "openvpn"
+    - "perl-Config-IniFiles"
+    - "python-dogpile-cache"
+    - "python-isodate"
+    - "python-netaddr"
+    - "python-netifaces"
+    - "python-setuptools_scm"
+    - "python-testtools"
+    - "python-websockify"
+    - "python2-cliff"
+    - "python2-debtcollector"
+    - "python2-deprecation"
+    - "python2-fixtures"
+    - "python2-funcsigs"
+    - "python2-ipaddress"
+    - "python2-pbr"
+    - "python2-positional"
+    - "python2-pysocks"
+    - "python2-pyyaml"
+    - "python2-requests-oauthlib"
+    - "python2-requestsexceptions"
+    - "python2-rfc3986"
+    - "python2-six"
+    - "python2-stevedore"
+    - "sdparm"
+    - "sshpass"
+    - "swig"
+    - "turbovnc-2.2.6*"
+    - "cuda-dcgm"
+    - "cuda-dcgm-libs"
+    - "cuda-dcgm-nvvs"
+    - "cuda-driver"

ansible/group_vars/ood

+---
+  yum_repo_files:
+    - cm.repo
+  pkg_list:
+    - autofs
+    - Lmod
+    - tmux
+    - vim

ansible/node-gpu.yml

----
-- name: Setup node for use as a cluster host with gpu packages
-  hosts: default
-  become: true
-  roles:
-    - cheaha.node

ansible/node.yml

----
-- name: Setup node for use as a cluster host
-  hosts: default
-  become: true
-  roles:
-    - cheaha.node

ansible/ood.yml

+---
+- name: Setup node for use as a virtual ood node
+  hosts: default
+  become: true
+  roles:
+    - { name: 'fix_centos_repo', tags: 'fix_centos_repo' }
+    - { name: 'install_packages', tags: 'install_packages' }
+    - { name: 'install_zsh', tags: 'install_zsh' }
+
+- name: Setup node for use as a virtual cheaha node
+  ansible.builtin.import_playbook: cheaha.yml