.gitlab-ci.yml

+default:
+  image: $CI_REGISTRY_IMAGE:latest
+
+variables:
+  CAMPUS_IP: 138.26.48.47
+  CHEAHA_IP: 172.20.10.9
+  TEST_IP: 138.26.49.134
+  ANSIBLE_REMOTE_TMP: "/tmp"
+  AWS_DEFAULT_REGION: "bhm"
+  AWS_HOST: "s3.lts.rc.uab.edu"
+  OS_AUTH_TYPE: "v3applicationcredential"
+  OS_AUTH_URL: "https://keystone.cloud.rc.uab.edu:5000/v3"
+  OS_IDENTITY_API_VERSION: "3"
+  OS_INTERFACE: "public"
+  OS_REGION_NAME: "bhm1"
+  PROXY_NETWORK: "proxy-net"
+  PKR_VAR_flavor: "m1.medium-ruffner"
+  PKR_VAR_source_image: "CentOS-7-x86_64-GenericCloud-2009"
+  PKR_VAR_floating_ip_network: "uab-campus"
+  PKR_VAR_security_groups: '["allow-ssh"]'
+  PKR_VAR_skip_create_image: "false"
+  PKR_VAR_ssh_username: "centos"
+  PKR_VAR_networks: '["8cf2f12e-905d-46d9-bc70-b0897c65f75a"]'
+  GIT_AUTHOR_NAME: "Gitlab runner"
+  GIT_AUTHOR_EMAIL: "gitlab@runner"
+  INSTANCE_FLAVOR: "m1.medium-ruffner"
+  HTTP_PROXY_INSTANCE_NAME: "http-proxy"
+  SSH_PROXY_INSTANCE_NAME: "ssh-proxy"
+
+stages:
+  - build
+  - deploy
+
+workflow:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "web"
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+
+.get_build_date: &get_build_date
+  - export BUILD_DATE=$(TZ=America/Chicago date +%Y-%m-%dT%H%M%S)
+  - echo BUILD_DATE=${BUILD_DATE}
+
+.update_ansible_repo: &update_ansible_repo
+  - *get_build_date
+  - |
+    export EXT_REPO_DIR=$(basename -s .git $EXT_PR_TARGET_REPO)
+    if [ ! -d $CI_PROJECT_DIR/$EXT_REPO_DIR ]; then
+      git clone ${EXT_PR_TARGET_REPO} ${EXT_REPO_DIR}
+      cd ${EXT_REPO_DIR}
+      git remote add upstream ${EXT_PR_SRC_REPO}
+      cd ..
+    fi
+  - cd ${EXT_REPO_DIR}
+  - git config user.name "${GIT_AUTHOR_NAME}"
+  - git config user.email "${GIT_AUTHOR_EMAIL}"
+  - git checkout ${EXT_PR_TARGET_BRANCH}
+  - git fetch origin ${EXT_PR_TARGET_BRANCH}
+  - git merge origin/${EXT_PR_TARGET_BRANCH}
+  - git checkout -b integration
+  - git fetch upstream ${EXT_PR_SRC_BRANCH}
+  - git merge upstream/${EXT_PR_SRC_BRANCH}
+  # export vars into job artifacts
+  - export EXT_REPO_HEAD=$(git rev-parse --short HEAD)
+  - export EXT_PR_SRC_BRANCH_SHA=$(git rev-parse --short upstream/${EXT_PR_SRC_BRANCH})
+  - export EXT_PR_TARGET_BRANCH_SHA=$(git rev-parse --short origin/${EXT_PR_TARGET_BRANCH})
+  - cd ..
+  - export PACKER_IMAGE_HEAD=$(git rev-parse --short HEAD)
+  - echo EXT_REPO_HEAD=${EXT_REPO_HEAD} | tee -a $CI_PROJECT_DIR/image.env
+  - echo EXT_PR_SRC_BRANCH_SHA=${EXT_PR_SRC_BRANCH_SHA} | tee -a $CI_PROJECT_DIR/image.env
+  - echo EXT_PR_TARGET_BRANCH_SHA=${EXT_PR_TARGET_BRANCH_SHA} | tee -a $CI_PROJECT_DIR/image.env
+  - echo PACKER_IMAGE_HEAD=${PACKER_IMAGE_HEAD} | tee -a $CI_PROJECT_DIR/image.env
+
+.get_ansible_files: &get_ansible_files
+  - s3cmd get --force -r --host=$AWS_HOST --host-bucket=$AWS_HOST s3://cheaha-cloud-ansible-files/ ansible/files/
+
+.build_proxy_image_template: &build_proxy_image_template
+  script:
+    - *update_ansible_repo
+    - *get_ansible_files
+    # packer vars for job env
+    - export PKR_VAR_flavor="${PROXY_BUILD_FLAVOR:-$PKR_VAR_flavor}"
+    - export PKR_VAR_build_instance_name="${BUILD_TARGET}-${EXT_REPO_HEAD}"
+    - export PKR_VAR_image_date_suffix=false
+    - |
+      if [ $CI_PIPELINE_SOURCE == 'merge_request_event' ]; then
+        export PKR_VAR_image_name="${BUILD_TARGET}-PR-${CI_MERGE_REQUEST_IID}"
+      elif [ $CI_PIPELINE_SOURCE == 'schedule' ]; then
+        export PKR_VAR_image_name="${BUILD_TARGET}-${BUILD_TAG:-${BUILD_DATE}}"
+      fi
+    # Ansible var overrides
+    - |
+      if [ -n "${PROXY_ENABLE_VAR}" ]; then
+        sed -i -E "s/(${PROXY_ENABLE_VAR}: ).*/\1true/" $EXT_REPO_DIR/group_vars/all
+      fi
+    - 'sed -i -E "s|(s3_endpoint: ).*|\1\"${S3_ENDPOINT}\"|" $EXT_REPO_DIR/group_vars/all'
+    - 'sed -i -E "s/(lts_access_key: ).*/\1\"${AWS_ACCESS_KEY_ID}\"/" $EXT_REPO_DIR/group_vars/all'
+    - 'sed -i -E "s/(lts_secret_key: ).*/\1\"${AWS_SECRET_ACCESS_KEY}\"/" $EXT_REPO_DIR/group_vars/all'
+    - 'sed -i -E "s/(s3_shibboleth_bucket_name: ).*/\1\"${S3_SHIBBOLETH_BUCKET_NAME}\"/" $EXT_REPO_DIR/group_vars/all'
+    - 'sed -i -E "s/(s3_shibboleth_object_name: ).*/\1\"${S3_SHIBBOLETH_OBJECT_NAME}\"/" $EXT_REPO_DIR/group_vars/all'
+    - 'sed -i -E "s|(ssh_pub_key: ).*|\1\"{{ lookup(''file'', ''${SSH_PUB_KEY}'') }}\"|" $EXT_REPO_DIR/group_vars/all'
+    # packer commands
+    - packer init openstack-proxy
+    - packer validate openstack-proxy
+    - packer build -machine-readable openstack-proxy | tee proxy_build.log
+    - export BUILT_PROXY_IMAGE_ID=$(grep 'Image:' proxy_build.log | awk '{print $4}')
+    - echo BUILT_PROXY_IMAGE_ID=${BUILT_PROXY_IMAGE_ID} | tee -a $CI_PROJECT_DIR/image.env
+    # set image properties with repo state
+    - openstack image set --property EXT_PR_SRC_REPO=${EXT_PR_SRC_REPO} --property EXT_PR_SRC_BRANCH_SHA=${EXT_PR_SRC_BRANCH_SHA} --property EXT_PR_TARGET_REPO=${EXT_PR_TARGET_REPO} --property EXT_PR_TARGET_BRANCH_SHA=${EXT_PR_TARGET_BRANCH_SHA} --property PACKER_IMAGE_HEAD=${PACKER_IMAGE_HEAD} ${BUILT_PROXY_IMAGE_ID}
+  artifacts:
+    reports:
+      dotenv: image.env
+
+build_http_proxy_image:
+  stage: build
+  environment:
+    name: build
+  tags:
+    - build
+  variables:
+    PROXY_ENABLE_VAR: "enable_http_proxy"
+  <<: *build_proxy_image_template
+  rules:
+    - if: $PIPELINE_TARGET == "build" && $BUILD_TARGET == "http-proxy"
+      when: always
+
+build_ssh_proxy_image:
+  stage: build
+  environment:
+    name: build
+  tags:
+    - build
+  variables:
+    PROXY_ENABLE_VAR: "enable_ssh_proxy"
+  <<: *build_proxy_image_template
+  rules:
+    - if: $PIPELINE_TARGET == "build" && $BUILD_TARGET == "ssh-proxy"
+      when: always
+
+.build_login_image_template: &build_login_image_template
+  script:
+    - *update_ansible_repo
+    - *get_ansible_files
+    # packer vars for job env
+    - export PKR_VAR_flavor="${PROXY_BUILD_FLAVOR:-$PKR_VAR_flavor}"
+    - export PKR_VAR_build_instance_name="${BUILD_TARGET}-${EXT_REPO_HEAD}"
+    - export PKR_VAR_image_date_suffix=false
+    - |
+      if [ $CI_PIPELINE_SOURCE == 'merge_request_event' ]; then
+        export PKR_VAR_image_name="${BUILD_TARGET}-PR-${CI_MERGE_REQUEST_IID}"
+      elif [ $CI_PIPELINE_SOURCE == 'schedule' ]; then
+        export PKR_VAR_image_name="${BUILD_TARGET}-${BUILD_TAG:-${BUILD_DATE}}"
+      fi
+    # packer commands
+    - packer init openstack-login
+    - packer validate openstack-login
+    - packer build -machine-readable openstack-login | tee login_build.log
+    - export BUILT_LOGIN_IMAGE_ID=$(grep 'Image:' login_build.log | awk '{print $4}')
+    - echo BUILT_LOGIN_IMAGE_ID=${BUILT_LOGIN_IMAGE_ID} | tee -a $CI_PROJECT_DIR/image.env
+    # set image properties with repo state
+    - openstack image set --property EXT_PR_SRC_REPO=${EXT_PR_SRC_REPO} --property EXT_PR_SRC_BRANCH_SHA=${EXT_PR_SRC_BRANCH_SHA} --property EXT_PR_TARGET_REPO=${EXT_PR_TARGET_REPO} --property EXT_PR_TARGET_BRANCH_SHA=${EXT_PR_TARGET_BRANCH_SHA} --property PACKER_IMAGE_HEAD=${CI_COMMIT_SHORT_SHA} ${BUILT_LOGIN_IMAGE_ID}
+  artifacts:
+    reports:
+      dotenv: image.env
+
+build_login_image:
+  stage: build
+  environment:
+    name: build
+  tags:
+    - build
+  <<: *build_login_image_template
+  rules:
+    - if: $PIPELINE_TARGET == "build" && $BUILD_TARGET == "login"
+      when: always
+
+build_ood_image:
+  stage: build
+  environment:
+    name: build
+  tags:
+    - build
+  script:
+    - *update_ansible_repo
+    - *get_ansible_files
+    # packer vars for job env
+    - export PKR_VAR_flavor="${OOD_BUILD_FLAVOR:-$PKR_VAR_flavor}"
+    - export PKR_VAR_build_instance_name="${BUILD_TARGET}-${EXT_REPO_HEAD}"
+    - export PKR_VAR_image_date_suffix=false
+    - export PKR_VAR_image_name="${BUILD_TARGET}-${BUILD_TAG:-${BUILD_DATE}}"
+    - |
+      if [ $ENV = 'knightly' ] || [ $ENV = 'prod' ]; then
+        curl --header "PRIVATE-TOKEN: ${ANSIBLE_VAR_TOKEN}" \
+        "${CI_API_V4_URL}/projects/2836/repository/files/$ENV/raw?ref=main" \
+        -o CRI_XCBC/group_vars/$ENV
+        sed -i -E "s/(lts_access_key: ).*/\1\"${AWS_ACCESS_KEY_ID}\"/" CRI_XCBC/group_vars/$ENV
+        sed -i -E "s/(lts_secret_key: ).*/\1\"${AWS_SECRET_ACCESS_KEY}\"/" CRI_XCBC/group_vars/$ENV
+        sed -i -E "s/(user_register_app_key: ).*/\1\"${SELF_REG_APP_KEY}\"/" CRI_XCBC/group_vars/$ENV
+        sed -i -E "s/(celery_user_password: ).*/\1\"${CELERY_PASSWD}\"/" CRI_XCBC/group_vars/$ENV
+        sed -i -E "s|(ssh_pub_key: ).*|\1\"{{ lookup('file', '${SSH_PUB_KEY}') }}\"|" CRI_XCBC/group_vars/$ENV
+      fi
+    # packer commands
+    - packer init openstack-ood
+    - packer validate openstack-ood
+    - packer build -machine-readable openstack-ood | tee ood_build.log
+    - export BUILT_OOD_IMAGE_ID=$(grep 'Image:' ood_build.log | awk '{print $4}')
+    - echo BUILT_OOD_IMAGE_ID=${BUILT_OOD_IMAGE_ID} | tee -a $CI_PROJECT_DIR/image.env
+    # set image properties with repo state
+    - openstack image set --property EXT_PR_SRC_REPO=${EXT_PR_SRC_REPO} --property EXT_PR_SRC_BRANCH_SHA=${EXT_PR_SRC_BRANCH_SHA} --property EXT_PR_TARGET_REPO=${EXT_PR_TARGET_REPO} --property EXT_PR_TARGET_BRANCH_SHA=${EXT_PR_TARGET_BRANCH_SHA} --property PACKER_IMAGE_HEAD=${CI_COMMIT_SHORT_SHA} ${BUILT_OOD_IMAGE_ID}
+  artifacts:
+    reports:
+      dotenv: image.env
+  rules:
+    - if: $BUILD_TARGET == "ood"
+      when: always
+
+deploy_http_proxy_node:
+  stage: deploy
+  environment:
+    name: $ENV
+  tags:
+    - build
+  script:
+    - openstack image set --accept $HTTP_PROXY_IMAGE_ID || true
+    - FAILED=false
+    - |
+      cat > user_data.txt <<EOF
+      #!/bin/bash
+      cat >> /etc/NetworkManager/conf.d/90-dns-none.conf<<EEOF
+      [main]
+      dns=none
+      EEOF
+      systemctl reload NetworkManager
+      echo "$DEV_KEY" >> /root/.ssh/authorized_keys
+      ip route replace default via ${DEFAULT_GATEWAY_IP} dev eth0
+      git clone ${CI_REPOSITORY_URL} /tmp/${CI_PROJECT_NAME}
+      cd /tmp/${CI_PROJECT_NAME}
+      git checkout ${CI_COMMIT_REF_NAME}
+      cat >> ansible/hosts<<EEOF
+      [$ENV]
+      127.0.0.1
+      EEOF
+      ansible-playbook -c local -i ansible/hosts --extra-vars="$EXTRA_VARS" ansible/cluster.yml | tee -a /tmp/ansible.log
+      rm -rf /tmp/${CI_PROJECT_NAME}
+      EOF
+    - |
+      export cmd="openstack server create"
+      cmd+=" -c id -f value --image $HTTP_PROXY_IMAGE_ID"
+      cmd+=" --flavor $INSTANCE_FLAVOR"
+      for security_group in ${SECURITY_GROUP_LIST[@]};
+      do
+        cmd+=" --security-group $security_group"
+      done
+      cmd+=" --user-data user_data.txt"
+      if [ -n "$PROXY_NETWORK" ];then cmd+=" --network $PROXY_NETWORK"; fi
+      if [ -n "$HTTP_PROXY_PORT" ];then cmd+=" --port $HTTP_PROXY_PORT"; fi
+      cmd+=" --wait $HTTP_PROXY_INSTANCE_NAME"
+    - export HTTP_PROXY_INSTANCE_ID=$(bash -c "$cmd")
+    - |
+      # Associate the floating IP(s) with the HTTP Proxy instance
+      for HTTP_PROXY_FLOATING_IP in ${HTTP_PROXY_FLOATING_IP_LIST[@]};
+      do
+        echo "Associating FLOATING_IP $HTTP_PROXY_FLOATING_IP with HTTP_PROXY_INSTANCE_ID $HTTP_PROXY_INSTANCE_ID"
+        openstack server add floating ip $HTTP_PROXY_INSTANCE_ID $HTTP_PROXY_FLOATING_IP
+      done
+  rules:
+    - if: $PIPELINE_TARGET == "deploy" && $HTTP_PROXY_IMAGE_ID
+      when: always
+
+deploy_ssh_proxy_node:
+  stage: deploy
+  environment:
+    name: $ENV
+  tags:
+    - build
+  script:
+    - openstack image set --accept $SSH_PROXY_IMAGE_ID || true
+    - FAILED=false
+    - |
+      cat > user_data.txt <<EOF
+      #!/bin/bash
+      cat >> /etc/NetworkManager/conf.d/90-dns-none.conf<<EEOF
+      [main]
+      dns=none
+      EEOF
+      systemctl reload NetworkManager
+      echo "$DEV_KEY" >> /root/.ssh/authorized_keys
+      ip route replace default via ${DEFAULT_GATEWAY_IP} dev eth0
+      git clone ${CI_REPOSITORY_URL} /tmp/${CI_PROJECT_NAME}
+      cd /tmp/${CI_PROJECT_NAME}
+      git checkout ${CI_COMMIT_REF_NAME}
+      cat >> ansible/hosts<<EEOF
+      [$ENV]
+      127.0.0.1
+      EEOF
+      ansible-playbook -c local -i ansible/hosts --extra-vars="$EXTRA_VARS" ansible/cluster.yml | tee -a /tmp/ansible.log
+      rm -rf /tmp/${CI_PROJECT_NAME}
+      EOF
+    - |
+      export cmd="openstack server create"
+      cmd+=" -c id -f value --image $SSH_PROXY_IMAGE_ID"
+      cmd+=" --flavor $INSTANCE_FLAVOR"
+      for security_group in ${SECURITY_GROUP_LIST[@]};
+      do
+        cmd+=" --security-group $security_group"
+      done
+      cmd+=" --user-data user_data.txt"
+      if [ -n "$PROXY_NETWORK" ];then cmd+=" --network $PROXY_NETWORK"; fi
+      if [ -n "$SSH_PROXY_PORT" ];then cmd+=" --port $SSH_PROXY_PORT"; fi
+      cmd+=" --wait $SSH_PROXY_INSTANCE_NAME"
+    - export SSH_PROXY_INSTANCE_ID=$(bash -c "$cmd")
+    - |
+      # Associate the floating IP(s) with the SSH Proxy instance
+      for SSH_PROXY_FLOATING_IP in ${SSH_PROXY_FLOATING_IP_LIST[@]};
+      do
+        echo "Associating FLOATING_IP $SSH_PROXY_FLOATING_IP with SSH_PROXY_INSTANCE_ID $SSH_PROXY_INSTANCE_ID"
+        openstack server add floating ip $SSH_PROXY_INSTANCE_ID $SSH_PROXY_FLOATING_IP
+      done
+  rules:
+    - if: $PIPELINE_TARGET == "deploy" && $SSH_PROXY_IMAGE_ID
+      when: always
+
+deploy_login_node:
+  stage: deploy
+  environment:
+    name: $ENV
+  tags:
+    - build
+  script:
+    - openstack image set --accept $LOGIN_IMAGE_ID || true
+    - FAILED=false
+    - |
+      cat > user_data.txt <<EOF
+      #!/bin/bash
+      cat >> /etc/NetworkManager/conf.d/90-dns-none.conf<<EEOF
+      [main]
+      dns=none
+      EEOF
+      systemctl reload NetworkManager
+      echo "$DEV_KEY" >> /root/.ssh/authorized_keys
+      ip route replace default via ${DEFAULT_GATEWAY_IP} dev eth0
+      git clone ${CI_REPOSITORY_URL} /tmp/${CI_PROJECT_NAME}
+      cd /tmp/${CI_PROJECT_NAME}
+      git checkout ${CI_COMMIT_REF_NAME}
+      cat >> ansible/hosts<<EEOF
+      [$ENV]
+      127.0.0.1
+      EEOF
+      s3cmd get --force -r --access_key=$AWS_ACCESS_KEY_ID --secret_key=$AWS_SECRET_ACCESS_KEY --host=$AWS_HOST --host-bucket=$AWS_HOST s3://cheaha-cloud-ansible-files/ /tmp/${CI_PROJECT_NAME}/ansible/files/ 
+      ansible-playbook -c local -i ansible/hosts --extra-vars="$EXTRA_VARS" ansible/cluster.yml | tee -a /tmp/ansible.log
+      rm -rf /tmp/${CI_PROJECT_NAME}
+      EOF
+    - |
+      export cmd="openstack server create"
+      cmd+=" -c id -f value --image $LOGIN_IMAGE_ID"
+      cmd+=" --flavor $INSTANCE_FLAVOR"
+      for security_group in ${SECURITY_GROUP_LIST[@]};
+      do
+        cmd+=" --security-group $security_group"
+      done
+      cmd+=" --user-data user_data.txt"
+      if [ -n "$INSTANCE_NETWORK" ];then cmd+=" --network $INSTANCE_NETWORK"; fi
+      if [ -n "$LOGIN_PORT" ];then cmd+=" --port $LOGIN_PORT"; fi
+      cmd+=" --wait $LOGIN_INSTANCE_NAME"
+    - export LOGIN_INSTANCE_ID=$(bash -c "$cmd")
+    - |
+      # Associate the floating IP(s) with the SSH Proxy instance
+      for LOGIN_FLOATING_IP in ${LOGIN_FLOATING_IP_LIST[@]};
+      do
+        echo "Associating FLOATING_IP $LOGIN_FLOATING_IP with LOGIN_INSTANCE_ID $LOGIN_INSTANCE_ID"
+        openstack server add floating ip $LOGIN_INSTANCE_ID $LOGIN_FLOATING_IP
+      done
+  rules:
+    - if: $PIPELINE_TARGET == "deploy" && $LOGIN_IMAGE_ID
+      when: always
+
+deploy_ood_node:
+  stage: deploy
+  environment:
+    name: $ENV
+  tags:
+    - build
+  before_script:
+    - |
+      for OOD_FLOATING_IP in ${OOD_FLOATING_IP_LIST[@]}; do
+        OOD_FIXED_IP=$(
+          openstack floating ip list \
+          --floating-ip-address $OOD_FLOATING_IP -c "Fixed IP Address" -f value)
+        CURRENT_INSTANCE_ID=$(
+          openstack server list \
+          --name $OOD_INSTANCE_NAME --ip $OOD_FIXED_IP -c ID -f value)
+        openstack server remove floating ip $CURRENT_INSTANCE_ID $OOD_FLOATING_IP
+      done
+    - |
+      if [ -n "$OOD_PORT" ];then
+        openstack server remove port $CURRENT_INSTANCE_ID $OOD_PORT
+      fi
+
+  script:
+    - OOD_IMAGE_ID="${BUILT_OOD_IMAGE_ID:-$OOD_IMAGE_ID}"
+    - openstack image set --accept $OOD_IMAGE_ID || true
+    - FAILED=false
+    - |
+      cat > user_data.txt <<EOF
+      #!/bin/bash
+      cat >> /etc/NetworkManager/conf.d/90-dns-none.conf<<EEOF
+      [main]
+      dns=none
+      EEOF
+      systemctl reload NetworkManager
+      echo "$DEV_KEY" >> /root/.ssh/authorized_keys
+      ip route replace default via ${DEFAULT_GATEWAY_IP} dev eth0
+      git clone ${CI_REPOSITORY_URL} /tmp/${CI_PROJECT_NAME}
+      cd /tmp/${CI_PROJECT_NAME}
+      git checkout ${CI_COMMIT_REF_NAME}
+      cat >> ansible/hosts<<EEOF
+      [$ENV]
+      127.0.0.1
+      EEOF
+      s3cmd get --force -r --access_key=$AWS_ACCESS_KEY_ID --secret_key=$AWS_SECRET_ACCESS_KEY --host=$AWS_HOST --host-bucket=$AWS_HOST s3://cheaha-cloud-ansible-files/ /tmp/${CI_PROJECT_NAME}/ansible/files/
+      ansible-playbook -c local -i ansible/hosts --extra-vars="$EXTRA_VARS" ansible/cluster.yml | tee -a /tmp/ansible.log
+      rm -rf /tmp/${CI_PROJECT_NAME}
+      EOF
+    - |
+      export cmd="openstack server create"
+      cmd+=" -c id -f value --image $OOD_IMAGE_ID"
+      cmd+=" --flavor $INSTANCE_FLAVOR"
+      for security_group in ${SECURITY_GROUP_LIST[@]};
+      do
+        cmd+=" --security-group $security_group"
+      done
+      cmd+=" --user-data user_data.txt"
+      if [ -n "$INSTANCE_NETWORK" ];then cmd+=" --network $INSTANCE_NETWORK"; fi
+      if [ -n "$OOD_PORT" ];then cmd+=" --port $OOD_PORT"; fi
+      cmd+=" --wait $OOD_INSTANCE_NAME"
+    - export OOD_INSTANCE_ID=$(bash -c "$cmd")
+    - |
+      # Associate the floating IP(s) with the SSH Proxy instance
+      for OOD_FLOATING_IP in ${OOD_FLOATING_IP_LIST[@]};
+      do
+        echo "Associating FLOATING_IP $OOD_FLOATING_IP with OOD_INSTANCE_ID $OOD_INSTANCE_ID"
+        openstack server add floating ip $OOD_INSTANCE_ID $OOD_FLOATING_IP
+      done
+  rules:
+    - if: $DEPLOY_TARGET == "ood"
+      when: always

Dockerfile

+FROM python:3.8-slim
+
+ENV S3CMD_VER=2.3.0
+ENV ANSIBLE_VER=4.10.0
+ENV OSC_VER=5.8.0
+ENV TF_VER=1.4.6
+ENV PACKER_VER=1.9.4
+
+RUN apt-get update && apt-get install --no-install-recommends -y \
+    git \
+    ssh \
+    curl \
+    wget \
+    unzip \
+ && rm -rf /var/lib/apt/lists/*
+RUN wget https://releases.hashicorp.com/packer/${PACKER_VER}/packer_${PACKER_VER}_linux_amd64.zip \
+    && unzip packer_${PACKER_VER}_linux_amd64.zip -d /usr/local/bin \
+    && rm packer_${PACKER_VER}_linux_amd64.zip
+RUN wget https://releases.hashicorp.com/terraform/${TF_VER}/terraform_${TF_VER}_linux_amd64.zip \
+    && unzip terraform_${TF_VER}_linux_amd64.zip -d /usr/local/bin \
+    && rm terraform_${TF_VER}_linux_amd64.zip
+RUN pip install --no-cache-dir --upgrade pip \
+ && pip install --no-cache-dir \
+    s3cmd==$S3CMD_VER \
+    ansible==$ANSIBLE_VER \
+    python-openstackclient==$OSC_VER

README.md

+# Create Openstack images for use with Cheaha
+
+# Prerequities
+
+1. [packer](https://www.packer.io/downloads)
+2. python 3.5+
+3. an openstack RC file
+4. Copies of certain files you can find around cheaha (place in `ansible/files`):
+  * `/etc/yum.repos.d/cm.repo` from a bright node
+  * `/etc/pki/rpm-gpg/RPM-GPG-KEY-cm` from a bright node
+  * `/etc/munge/munge.key` from a bright node
+  * `/cm/local/apps/opendlap/etc/certs/*.{key,pem}` (all certs in this directory) from a bright node
+  * `/etc/nslcd.conf` from a bright node
+
+# Quickstart
+
+Use your openstack rc file and use the following and you should be able to quickly create an instance:
+
+```bash
+python -mvenv env
+. env/bin/activate
+pip install ansible
+cp openstack.pkrvars.hcl{.example,}
+packer build -var-file=openstack.pkrvars.hcl openstack
+```
+
+# Running
+
+1. Use your openstack rc file
+2. Install ansible (preferably in a virtualenv) and ensure `ansible-playbook` is in `PATH`
+3. Copy `openstack.pkrvars.hcl.example` to `openstack.pkrvars.hcl` and edit in the documented variables
+4. Run `packer build -var-file=openstack.pkrvars.hcl openstack`
+
+# Bright considerations
+
+In order to have this node "behave" as a normal slurm node, you'll need to do the following:
+
+1. Create an entry for it in devices - this is specifically to create the dns entry (the most important thing to set up is the IP under interfaces)
+2. Add the node entry to `/etc/slurm/slurm.conf` (both as a node definition and to a partition)
+
+# Running the openstack image
+
+1. Create an instance with the image, at whatever size you need
+2. Make sure to assign it to the `openstack-cheaha-internal` network with the `cheaha-network` security group
+3. Associagte a cheaha floating ip from the `cheaha-cluster` pool

ansible/ansible.cfg

+[defaults]
+# change the default callback, you can only have one 'stdout' type  enabled at a time.
+#stdout_callback = skippy
+stdout_callback = yaml
+
+## Ansible ships with some plugins that require whitelisting,
+## this is done to avoid running all of a type by default.
+## These setting lists those that you want enabled for your system.
+## Custom plugins should not need this unless plugin author specifies it.
+
+# enable callback plugins, they can output to stdout but cannot be 'stdout' type.
+callbacks_enabled = timer, debug, profile_roles, profile_tasks, minimal
+
+# Force color
+force_color = true

ansible/base.yml

+---
+- name: Install base packages
+  hosts: default
+  become: true
+  roles:
+    - { name: 'fix_centos_repo', tags: 'fix_centos_repo' }
+    - { name: 'install_packages', tags: 'install_packages' }

ansible/cluster.yml

+---
+- name: Setup node for use as a virtual cheaha node
+  hosts: all
+  become: true
+  roles:
+    - { name: 'cheaha.node', tags: 'cheaha.node' }
+    - { name: 'nfs_mounts', tags: 'nfs_mounts', when: enable_nfs_mounts }
+    - { name: 'ldap_config', tags: 'ldap_config' }
+    - { name: 'slurm_client', tags: 'slurm_client', when: enable_slurm_client }
+    - { name: 'ssh_host_keys', tags: 'ssh_host_keys' }
+    - { name: 'ssh_proxy_config', tags: 'ssh_proxy_config', when: enable_ssh_proxy_config }
+    - { name: 'ssl_cert', tags: 'ssl_cert', when: enable_ssl_certs }
+    - { name: 'rsyslog_config', tags: 'rsyslog_config', when: enable_rsyslog_config }
+    - { name: 'rewrite_map', tags: 'rewrite_map', when: enable_rewrite_map }
+    - { name: 'fail2ban', tags: 'fail2ban', when: enable_fail2ban }
+    - { name: 'install_node_exporter', tags: 'install_node_exporter', when: enable_node_exporter }

ansible/files/TurboVNC.repo

+[TurboVNC]
+name=TurboVNC official RPMs
+baseurl=https://sourceforge.net/projects/turbovnc/files
+gpgcheck=1
+gpgkey=https://sourceforge.net/projects/turbovnc/files/VGL-GPG-KEY
+       https://sourceforge.net/projects/turbovnc/files/VGL-GPG-KEY-1024
+enabled=1

ansible/files/build-zsh.sh

+#!/bin/bash
+name=zsh
+version=5.9
+
+mkdir -p BUILD RPMS SOURCES SPECS SRPMS
+
+prereqs="git epel-release bzip2 xz gzip tar"
+rpm -q $prereqs
+
+if [ $? -ne 0 ]; then sudo yum -y install $prereqs; fi
+
+git clone https://src.fedoraproject.org/rpms/${name}.git
+
+cp ${name}/${name}.spec SPECS/
+cp ${name}/*.rhs SOURCES/
+cp ${name}/dot* SOURCES/
+
+# Perl may not be available, switching to sed
+#perl -pi -e 's/^(BuildRequires: glibc-langpack-ja)/#$1/g;' ${name}/${name}.spec
+sed -i '/BuildRequires: glibc-langpack-ja/s/^/#/' SPECS/${name}.spec
+
+pkgs="rpm-build $(grep -E '^BuildRequires|^Requires' SPECS/${name}.spec | awk '{print $2}' | tr '\n' ' ')"; echo $pkgs
+rpm -q $pkgs
+
+if [ $? -ne 0 ]; then sudo yum install -y $pkgs; fi
+
+if [ ! -f SOURCES/${name}-${version}.tar.xz ] ; then
+  curl -L -o SOURCES/${name}-${version}.tar.xz https://downloads.sourceforge.net/${name}/${name}-${version}.tar.xz
+fi
+
+if [ ! -f SOURCES/${name}-${version}.tar.xz.asc ] ; then
+  curl -L -o SOURCES/${name}-${version}.tar.xz.asc https://downloads.sourceforge.net/${name}/${name}-${version}.tar.xz.asc
+fi
+
+cd SOURCES
+sha512sum -c ../${name}/sources
+retval=$?
+cd -
+if [ $retval -ne 0 ]; then
+#  echo "SOURCES/${name}-${version}.tar.gz did not match checksum in ${name}/source"
+  exit 1
+fi
+
+rpmbuild --define "_topdir `pwd`" -bb SPECS/${name}.spec

ansible/files/ceph.repo

+[Ceph]
+priority=1
+name=Ceph $basearch
+baseurl=https://download.ceph.com/rpm-octopus/el7/$basearch
+enabled=1
+gpgcheck=1
+gpgkey=https://download.ceph.com/keys/release.asc
+
+[Ceph-noarch]
+priority=1
+name=Ceph noarch
+baseurl=https://download.ceph.com/rpm-octopus/el7/noarch
+enabled=1
+gpgcheck=1
+gpgkey=https://download.ceph.com/keys/release.asc
+
+[Ceph-source]
+priority=1
+name=Ceph SRPMS
+baseurl=https://download.ceph.com/rpm-octopus/el7/SRPMS
+enabled=1
+gpgcheck=1
+gpgkey=https://download.ceph.com/keys/release.asc
+

ansible/files/dell-system-update.repo

+[dell-system-update_independent]
+name=dell-system-update_independent
+baseurl=https://linux.dell.com/repo/hardware/dsu/os_independent/
+gpgcheck=1
+gpgkey=https://linux.dell.com/repo/pgp_pubkeys/0x756ba70b1019ced6.asc
+      https://linux.dell.com/repo/pgp_pubkeys/0x1285491434D8786F.asc
+     https://linux.dell.com/repo/pgp_pubkeys/0xca77951d23b66a9d.asc
+                    https://linux.dell.com/repo/pgp_pubkeys/0x3CA66B4946770C59.asc
+enabled=1
+exclude=dell-system-update*.i386
+
+[dell-system-update_dependent]
+name=dell-system-update_dependent
+mirrorlist=https://linux.dell.com/repo/hardware/dsu/mirrors.cgi?osname=el$releasever&basearch=$basearch&native=1
+gpgcheck=1
+gpgkey=https://linux.dell.com/repo/pgp_pubkeys/0x756ba70b1019ced6.asc
+      https://linux.dell.com/repo/pgp_pubkeys/0x1285491434D8786F.asc
+     https://linux.dell.com/repo/pgp_pubkeys/0xca77951d23b66a9d.asc
+                    https://linux.dell.com/repo/pgp_pubkeys/0x3CA66B4946770C59.asc
+enabled=1

ansible/files/elastic.repo

+[elastic-7.x]
+name=Elastic repository for 7.x packages
+baseurl=https://artifacts.elastic.co/packages/7.x/yum
+gpgcheck=1
+gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
+enabled=1
+autorefresh=1
+type=rpm-md

ansible/files/nux-dextop.repo

+[nux-dextop]
+name=Nux.Ro RPMs for general desktop use
+baseurl=http://li.nux.ro/download/nux/dextop/el7/$basearch/
+enabled=1
+gpgcheck=1
+gpgkey=http://li.nux.ro/download/nux/RPM-GPG-KEY-nux.ro
+protect=0
+
+[nux-dextop-testing]
+name=Nux.Ro RPMs for general desktop use - testing
+baseurl=http://li.nux.ro/download/nux/dextop-testing/el7/$basearch/
+enabled=0
+gpgcheck=1
+gpgkey=http://li.nux.ro/download/nux/RPM-GPG-KEY-nux.ro
+protect=0

ansible/gpu.yml

+---
+- name: Setup node for use as a cluster host with gpu drivers/pkgs
+  hosts: default
+  become: true
+  roles:
+    - { name: 'fix_centos_repo', tags: 'fix_centos_repo' }
+    - { name: 'install_packages', tags: 'install_packages' }
+    - { name: 'cuda_driver', tags: 'cuda_driver' }
+    - { name: 'pam_slurm_adopt', tags: 'pam_slurm_adopt' }
+    - { name: 'install_nhc', tags: 'install_nhc'}
+
+- name: Setup node for use as a virtual cheaha node
+  ansible.builtin.import_playbook: cheaha.yml

ansible/group_vars/all

+---
+  zsh_ver: 5.7.1
+  zsh_src_url: "https://www.zsh.org/pub/old/zsh-{{ zsh_ver }}.tar.xz"
+  yum_repo_files: []
+  pkg_list: []
+  slurm_version: 18.08.9
+  enable_slurm_client: false
+
+# NHC related
+  nhc_download_url: "https://github.com/mej/nhc/releases/download/1.4.3/lbnl-nhc-1.4.3-1.el7.noarch.rpm"
+  nhc_download_path: "/tmp"
+  nhc_git_repo: "https://gitlab.rc.uab.edu/rc/nhc.git"
+  nhc_git_repo_path: "/tmp/nhc"
+
+  root_ssh_key: ""
+
+# cheaha.node related
+  hostname_lookup_table:
+    - "10.141.255.254 master.cm.cluster master localmaster.cm.cluster localmaster ldapserver.cm.cluster ldapserver"
+  domain_search_list:
+    - openstack.internal
+    - cm.cluster
+  nameserver_list:
+    - 10.141.255.254
+
+# ldap_config related
+  ldap_cert_path: "/etc/openldap/certs"
+  ldap_uri: "ldap://ldapserver"
+
+# nfs_mounts related
+  enable_nfs_mounts: true
+  use_autofs: false
+  use_fstab: false
+  mount_points:
+    - { "src": "master:/gpfs4", "path": "/gpfs4", "opts": "ro,sync,hard", "mode": "0755" }
+    - { "src": "master:/gpfs5", "path": "/gpfs5", "opts": "ro,sync,hard", "mode": "0755" }
+  autofs_mounts:
+    - { "src": "master:/gpfs4/&", "path": "/gpfs4", "opts": "fstype=nfs,vers=3,_netdev,default", "mode": '0755', "mount_point": "/gpfs4", "map_name": "gpfs4", key: "*" }
+    - { "src": "master:/gpfs5/&", "path": "/gpfs5", "opts": "fstype=nfs,vers=3,_netdev,default", "mode": '0755', "mount_point": "/gpfs5", "map_name": "gpfs5", key: "*" }
+
+#SSH Host Keys
+  S3_ENDPOINT: ""
+  SSH_HOST_KEYS_S3_BUCKET: ""
+  SSH_HOST_KEYS_S3_OBJECT: ""
+
+# AWS credentials
+  LTS_ACCESS_KEY: ""
+  LTS_SECRET_KEY: ""
+
+# ssh proxy
+  enable_ssh_proxy_config: false
+  sshpiper_dest_dir: "/opt/sshpiper"
+
+# rsyslog
+  enable_rsyslog_config: true
+  rsyslog_target: "*.* @master:514"
+
+# ssl certs
+  enable_ssl_certs: false
+  ssl_cert_s3_bucket: ""
+  ssl_cert_key_location: "/etc/pki/tls/private"
+  ssl_cert_file_location: "/etc/pki/tls/certs"
+  ssl_cert_key: ""
+  ssl_cert_file: ""
+  ssl_cert_chain_file: ""
+  ssl_apache_config: ""
+  apache_service: "httpd"
+
+# rewrite map
+  enable_rewrite_map: false
+  target_groups:
+    - {"name": "gpfs4", "host": "login001", "default": True }
+    - {"name": "gpfs5", "host": "login002", "default": False }
+
+# account app
+  account_app_port: 8000
+
+# fail2ban
+  enable_fail2ban: false
+  maxretry: 1
+  findtime: 600
+  bantime: 1200
+  fail2ban_white_list: "127.0.0.1/8"
+
+# Node Exporter
+  enable_node_exporter: false
+  node_exporter_ver: "1.8.2"
+  node_exporter_filename: "node_exporter-{{ node_exporter_ver }}.linux-amd64"
+  node_exporter_user: node_exporter
+  node_exporter_group: node_exporter
+  node_exporter_port: 9100
+
+# CentOS Repo
+  centos_base_url: "http://vault.centos.org"

ansible/group_vars/compute

+---
+  yum_repo_files:
+    - TurboVNC.repo
+    - cm.repo
+  pkg_list:
+    - "Lmod-7.8.11"
+    - "atftp-server"
+    - "cluster-tools-dell"
+    - "cluster-tools-slave"
+    - "cm-boost"
+    - "cm-config-ceph-release-luminous"
+    - "cm-config-cm"
+    - "cm-config-dhclient"
+    - "cm-config-dracut-slave"
+    - "cm-config-grub"
+    - "cm-config-ldap-client"
+    - "cm-config-limits"
+    - "cm-config-man"
+    - "cm-config-named"
+    - "cm-config-network-slave"
+    - "cm-config-nfsclient"
+    - "cm-config-rootfiles-slave"
+    - "cm-config-selinux"
+    - "cm-config-ssh-slave"
+    - "cm-config-sysctl-slave"
+    - "cm-config-syslog-slave"
+    - "cm-config-systemd"
+    - "cm-config-xntp-slave"
+    - "cm-config-yum"
+    - "cm-curl"
+    - "cm-dhcp"
+    - "cm-freeipmi"
+    - "cm-ipmitool"
+    - "cm-ipxe-slave"
+    - "cm-libpam"
+    - "cm-libprometheus"
+    - "cm-lua"
+    - "cm-mariadb-libs"
+    - "cm-openssl"
+    - "cm-python2"
+    - "cm-python36"
+    - "cm-slave"
+    - "cm-uge-client"
+    - "cmburn"
+    - "cmburn-slave"
+    - "cmdaemon"
+    - "cmdaemon-remotecm"
+    - "confuse"
+    - "gcc-recent"
+    - "gdb-recent"
+    - "lshw"
+    - "lua-bit32"
+    - "lua-filesystem"
+    - "lua-json"
+    - "lua-lpeg"
+    - "lua-posix"
+    - "lua-term"
+    - "mysql++"
+    - "net-snmp-recent"
+    - "node-installer-slave"
+    - "openvpn"
+    - "perl-Config-IniFiles"
+    - "python-dogpile-cache"
+    - "python-isodate"
+    - "python-netaddr"
+    - "python-netifaces"
+    - "python-setuptools_scm"
+    - "python-testtools"
+    - "python-websockify"
+    - "python2-cliff"
+    - "python2-debtcollector"
+    - "python2-deprecation"
+    - "python2-fixtures"
+    - "python2-funcsigs"
+    - "python2-ipaddress"
+    - "python2-pbr"
+    - "python2-positional"
+    - "python2-pysocks"
+    - "python2-pyyaml"
+    - "python2-requests-oauthlib"
+    - "python2-requestsexceptions"
+    - "python2-rfc3986"
+    - "python2-six"
+    - "python2-stevedore"
+    - "sdparm"
+    - "sshpass"
+    - "swig"
+    - "turbovnc-2.2.6*"

ansible/group_vars/gpu

+---
+  yum_repo_files:
+    - TurboVNC.repo
+    - cm.repo
+  pkg_list:
+    - "Lmod-7.8.11"
+    - "atftp-server"
+    - "cluster-tools-dell"
+    - "cluster-tools-slave"
+    - "cm-boost"
+    - "cm-config-ceph-release-luminous"
+    - "cm-config-cm"
+    - "cm-config-dhclient"
+    - "cm-config-dracut-slave"
+    - "cm-config-grub"
+    - "cm-config-ldap-client"
+    - "cm-config-limits"
+    - "cm-config-man"
+    - "cm-config-named"
+    - "cm-config-network-slave"
+    - "cm-config-nfsclient"
+    - "cm-config-rootfiles-slave"
+    - "cm-config-selinux"
+    - "cm-config-ssh-slave"
+    - "cm-config-sysctl-slave"
+    - "cm-config-syslog-slave"
+    - "cm-config-systemd"
+    - "cm-config-xntp-slave"
+    - "cm-config-yum"
+    - "cm-curl"
+    - "cm-dhcp"
+    - "cm-freeipmi"
+    - "cm-ipmitool"
+    - "cm-ipxe-slave"
+    - "cm-libpam"
+    - "cm-libprometheus"
+    - "cm-lua"
+    - "cm-mariadb-libs"
+    - "cm-openssl"
+    - "cm-python2"
+    - "cm-python36"
+    - "cm-slave"
+    - "cm-uge-client"
+    - "cmburn"
+    - "cmburn-slave"
+    - "cmdaemon"
+    - "cmdaemon-remotecm"
+    - "confuse"
+    - "gcc-recent"
+    - "gdb-recent"
+    - "lshw"
+    - "lua-bit32"
+    - "lua-filesystem"
+    - "lua-json"
+    - "lua-lpeg"
+    - "lua-posix"
+    - "lua-term"
+    - "mysql++"
+    - "net-snmp-recent"
+    - "node-installer-slave"
+    - "openvpn"
+    - "perl-Config-IniFiles"
+    - "python-dogpile-cache"
+    - "python-isodate"
+    - "python-netaddr"
+    - "python-netifaces"
+    - "python-setuptools_scm"
+    - "python-testtools"
+    - "python-websockify"
+    - "python2-cliff"
+    - "python2-debtcollector"
+    - "python2-deprecation"
+    - "python2-fixtures"
+    - "python2-funcsigs"
+    - "python2-ipaddress"
+    - "python2-pbr"
+    - "python2-positional"
+    - "python2-pysocks"
+    - "python2-pyyaml"
+    - "python2-requests-oauthlib"
+    - "python2-requestsexceptions"
+    - "python2-rfc3986"
+    - "python2-six"
+    - "python2-stevedore"
+    - "sdparm"
+    - "sshpass"
+    - "swig"
+    - "turbovnc-2.2.6*"
+    - "cuda-dcgm"
+    - "cuda-dcgm-libs"
+    - "cuda-dcgm-nvvs"
+    - "cuda-driver"

ansible/group_vars/ood

+---
+  yum_repo_files:
+    - cm.repo
+  pkg_list:
+    - autofs
+    - Lmod
+    - tmux
+    - vim

ansible/group_vars/prod

+---
+  # cheaha.node related
+  hostname_lookup_table:
+    - "172.20.0.24 cheaha-master02.cm.cluster cheaha-master02"
+    - "172.20.0.22 cheaha-master01.cm.cluster cheaha-master01"
+    - "172.20.0.25 master.cm.cluster master localmaster.cm.cluster localmaster ldapserver.cm.cluster ldapserver"
+  domain_search_list:
+    - cm.cluster
+    - rc.uab.edu
+    - ib.cluster
+    - drac.cluster
+    - eth.cluster
+    - ib-hdr.cluster
+  nameserver_list:
+    - 172.20.0.25
+
+  bright_openldap_path: "/cm/local/apps/openldap"
+  ldap_cert_path: "{{bright_openldap_path}}/etc/certs"
+  ldap_uri: "ldaps://ldapserver"
+
+  # proxy_config
+  target_groups:
+    - {"name": "gpfs5", "host": "login002", "default": False, "authorized_keys":"/gpfs5/data/user/home/$DOWNSTREAM_USER/.ssh/authorized_keys", "private_key":"/gpfs5/data/user/home/$DOWNSTREAM_USER/.ssh/id_ecdsa"}
+    - {"name": "gpfs4", "host": "login001", "default": True, "authorized_keys":"/gpfs4/data/user/home/$DOWNSTREAM_USER/.ssh/authorized_keys", "private_key":"/gpfs4/data/user/home/$DOWNSTREAM_USER/.ssh/id_ecdsa"}

ansible/login.yml

+---
+- name: Setup node for use as a virtual compute node
+  hosts: default
+  become: true
+  roles:
+    - { name: 'fix_centos_repo', tags: 'fix_centos_repo' }
+    - { name: 'install_packages', tags: 'install_packages' }
+    - { name: 'install_nhc', tags: 'install_nhc'}
+