Merge branch 'feat-ci-proxy-builds' into 'feat-hpc-factory'

Feat ci proxy builds

See merge request rc/packer-openstack-hpc-image!103

.gitlab-ci.yml

@@ -14,15 +14,22 @@ variables:
   OS_INTERFACE: "public"
   OS_REGION_NAME: "bhm1"
   PROXY_NETWORK: "proxy-net"
+  PKR_VAR_flavor: "m1.medium-ruffner"
+  PKR_VAR_source_image: "CentOS-7-x86_64-GenericCloud-2009"
+  PKR_VAR_floating_ip_network: "uab-campus"
+  PKR_VAR_security_groups: '["allow-ssh"]'
+  PKR_VAR_skip_create_image: "false"
+  PKR_VAR_ssh_username: "centos"
+  PKR_VAR_networks: '["8cf2f12e-905d-46d9-bc70-b0897c65f75a"]'
   GIT_AUTHOR_NAME: "Gitlab runner"
   GIT_AUTHOR_EMAIL: "gitlab@runner"
-  PKR_VAR_floating_ip_network: "uab-campus"
   DEV_KEY: "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBBXBfBmQDq1HK8U0zK6gDF0jyfWChuFYEPiRp1Wgl0XFJU7JoUngqkE/GAic8kCzKW0hPMiweSjmWdKlAapv/dk= krishmoodbidri@krishs-MacBook-Pro-3.local"
   INSTANCE_FLAVOR: "m1.medium-ruffner"
   HTTP_PROXY_INSTANCE_NAME: "http-proxy"
   SSH_PROXY_INSTANCE_NAME: "ssh-proxy"
 
 stages:
+  - build
   - deploy
 
 workflow:
@@ -30,6 +37,105 @@ workflow:
     - if: $CI_PIPELINE_SOURCE == "web"
     - if: $CI_PIPELINE_SOURCE == "schedule"
 
+.get_build_date: &get_build_date
+  - export BUILD_DATE=$(TZ=America/Chicago date +%Y-%m-%dT%H%M%S)
+  - echo BUILD_DATE=${BUILD_DATE}
+
+.update_ansible_repo: &update_ansible_repo
+  - *get_build_date
+  - |
+    if [ ! -d $CI_PROJECT_DIR/CRI_XCBC ]; then
+      git clone https://github.com/uabrc/CRI_XCBC.git
+      cd CRI_XCBC
+      git remote add upstream https://github.com/jprorama/CRI_XCBC.git
+      cd ..
+    fi
+  - cd CRI_XCBC
+  - git config user.name "${GIT_AUTHOR_NAME}"
+  - git config user.email "${GIT_AUTHOR_EMAIL}"
+  - git fetch origin uab-prod
+  - git fetch upstream dev
+  - git checkout uab-prod
+  - git merge origin/uab-prod
+  - git checkout -b integration
+  - git merge upstream/dev
+  - export CRI_XCBC_HEAD=$(git rev-parse --short HEAD)
+  - export CRI_XCBC_dev=$(git rev-parse --short upstream/dev)
+  - export CRI_XCBC_prod=$(git rev-parse --short origin/uab-prod)
+  - cd ..
+  - export PACKER_IMAGE_HEAD=$(git rev-parse --short HEAD)
+  - echo CRI_XCBC_HEAD=${CRI_XCBC_HEAD} | tee -a $CI_PROJECT_DIR/image.env
+  - echo CRI_XCBC_dev=${CRI_XCBC_dev} | tee -a $CI_PROJECT_DIR/image.env
+  - echo CRI_XCBC_prod=${CRI_XCBC_prod} | tee -a $CI_PROJECT_DIR/image.env
+  - echo PACKER_IMAGE_HEAD=${PACKER_IMAGE_HEAD} | tee -a $CI_PROJECT_DIR/image.env
+
+.get_ansible_files: &get_ansible_files
+  - s3cmd get --force -r --host=$AWS_HOST --host-bucket=$AWS_HOST s3://cheaha-cloud-ansible-files/ ansible/files/
+
+.build_proxy_image_template: &build_proxy_image_template
+  script:
+    - *update_ansible_repo
+    - *get_ansible_files
+    # packer vars for job env
+    - export PKR_VAR_flavor="${PROXY_BUILD_FLAVOR:-$PKR_VAR_flavor}"
+    - export PKR_VAR_build_instance_name="${PROXY_NAME}-${CRI_XCBC_HEAD}"
+    - export PKR_VAR_image_date_suffix=false
+    - |
+      if [ $CI_PIPELINE_SOURCE == 'merge_request_event' ]; then
+        export PKR_VAR_image_name="${PROXY_NAME}-PR-${CI_MERGE_REQUEST_IID}"
+      elif [ $CI_PIPELINE_SOURCE == 'schedule' ]; then
+        export PKR_VAR_image_name="${PROXY_NAME}-${BUILD_DATE}"
+      fi
+    # Ansible var overrides
+    - |
+      if [ -n "${PROXY_ENABLE_VAR}" ]; then
+        sed -i -E "s/(${PROXY_ENABLE_VAR}: ).*/\1true/" CRI_XCBC/group_vars/all
+      fi
+    - 'sed -i -E "s|(s3_endpoint: ).*|\1\"${S3_ENDPOINT}\"|" CRI_XCBC/group_vars/all'
+    - 'sed -i -E "s/(lts_access_key: ).*/\1\"${AWS_ACCESS_KEY_ID}\"/" CRI_XCBC/group_vars/all'
+    - 'sed -i -E "s/(lts_secret_key: ).*/\1\"${AWS_SECRET_ACCESS_KEY}\"/" CRI_XCBC/group_vars/all'
+    - 'sed -i -E "s/(s3_shibboleth_bucket_name: ).*/\1\"${S3_SHIBBOLETH_BUCKET_NAME}\"/" CRI_XCBC/group_vars/all'
+    - 'sed -i -E "s/(s3_shibboleth_object_name: ).*/\1\"${S3_SHIBBOLETH_OBJECT_NAME}\"/" CRI_XCBC/group_vars/all'
+    - 'sed -i -E "s|(ssh_pub_key: ).*|\1\"{{ lookup(''file'', ''${SSH_PUB_KEY}'') }}\"|" CRI_XCBC/group_vars/all'
+    # packer commands
+    - packer init openstack-proxy
+    - packer validate openstack-proxy
+    - packer build -machine-readable openstack-proxy | tee proxy_build.log
+    - export BUILT_PROXY_IMAGE_ID=$(grep 'Image:' proxy_build.log | awk '{print $4}')
+    - echo BUILT_PROXY_IMAGE_ID=${BUILT_PROXY_IMAGE_ID} | tee -a $CI_PROJECT_DIR/image.env
+    - openstack image set --property CRI_XCBC_prod=${CRI_XCBC_prod} --property CRI_XCBC_dev=${CRI_XCBC_dev} --property PACKER_IMAGE_HEAD=${PACKER_IMAGE_HEAD} ${BUILT_PROXY_IMAGE_ID}
+  artifacts:
+    reports:
+      dotenv: image.env
+
+build_http_proxy_image:
+  stage: build
+  environment:
+    name: $ENV
+  tags:
+    - build
+  variables:
+    PROXY_NAME: "http-proxy"
+    PROXY_ENABLE_VAR: "enable_ood_proxy"
+  <<: *build_proxy_image_template
+  rules:
+    - if: $PIPELINE_TARGET == "build" && $BUILD_TARGET
+      when: always
+
+build_ssh_proxy_image:
+  stage: build
+  environment:
+    name: $ENV
+  tags:
+    - build
+  variables:
+    PROXY_NAME: "ssh-proxy"
+    PROXY_ENABLE_VAR: "enable_sshpiper"
+  <<: *build_proxy_image_template
+  rules:
+    - if: $PIPELINE_TARGET == "build" && $BUILD_TARGET
+      when: always
+
 deploy_http_proxy_node:
   stage: deploy
   environment:

openstack-proxy/nodeimage.pkr.hcl

@@ -36,11 +36,28 @@ source "openstack" "image" {
 build {
   sources = ["source.openstack.image"]
 
+  provisioner "shell" {
+    inline = [
+      "sudo sed -i 's/^mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-*",
+      "sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*",
+      "sudo yum install -y epel-release",
+      "sudo yum install -y libselinux-python3 python3 tmux vim git bash-completion curl wget unzip",
+      "sudo python3 -m pip install --upgrade pip",
+      "sudo pip3 install s3cmd==2.3.0 ansible==4.10.0 python-openstackclient==5.8.0"
+    ]
+  }
+
   provisioner "ansible" {
     use_proxy        = false
     user             = var.ssh_username
     groups           = ["proxy"]
-    ansible_env_vars = ["ANSIBLE_TIMEOUT=60", "ANSIBLE_HOST_KEY_CHECKING=False"]
+    ansible_env_vars = [
+      "ANSIBLE_TIMEOUT=60",
+      "ANSIBLE_HOST_KEY_CHECKING=False",
+      "ANSIBLE_VERBOSITY=${var.ANSIBLE_VERBOSITY}",
+      "ANSIBLE_DEBUG=${var.ANSIBLE_DEBUG}",
+      "ANSIBLE_FORCE_COLOR=true"
+    ]
     playbook_file    = "./CRI_XCBC/proxy.yaml"
   }
 }

openstack-proxy/variables.pkr.hcl

@@ -94,3 +94,15 @@ variable "volume_size" {
   description = "The default volume size for building iamge"
 }
 
+variable "ANSIBLE_DEBUG" {
+  type        = string
+  default     = "false"
+  description = "to turn on debugging"
+}
+
+variable "ANSIBLE_VERBOSITY" {
+  type        = string
+  default     = "0"
+  description = "to increase verbosity - 0|1|2|3|4"
+}
+