Compare revisions

Eesaan Atluri · Bo-Chun Chen · Eesaan Atluri · Eesaan Atluri · Eesaan Atluri · Eesaan Atluri
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -26,10 +26,12 @@ variables:
  INSTANCE_FLAVOR: "m1.medium-ruffner"
  HTTP_PROXY_INSTANCE_NAME: "http-proxy"
  SSH_PROXY_INSTANCE_NAME: "ssh-proxy"
+  NUM_IMAGES_TO_KEEP: 5

 stages:
  - build
  - deploy
+  - cleanup

 workflow:
  rules:
@@ -78,9 +80,12 @@ workflow:
    - *update_ansible_repo
    - *get_ansible_files
    # packer vars for job env
+    - image_tag=${BUILD_TARGET}${BUILD_TAG:+-${BUILD_TAG}}-${ENV}
    - export PKR_VAR_flavor="${PROXY_BUILD_FLAVOR:-$PKR_VAR_flavor}"
    - export PKR_VAR_build_instance_name="${BUILD_TARGET}-${EXT_REPO_HEAD}"
    - export PKR_VAR_image_date_suffix=false
+    - export PKR_VAR_image_tags="[\"${image_tag}\"]"
+    - export PKR_VAR_image_name=${image_tag}-${BUILD_DATE}
    - |
      if [ $CI_PIPELINE_SOURCE == 'merge_request_event' ]; then
        export PKR_VAR_image_name="${BUILD_TARGET}-PR-${CI_MERGE_REQUEST_IID}"
@@ -104,6 +109,7 @@ workflow:
    - packer build -machine-readable openstack-proxy | tee proxy_build.log
    - export BUILT_PROXY_IMAGE_ID=$(grep 'Image:' proxy_build.log | awk '{print $4}')
    - echo BUILT_PROXY_IMAGE_ID=${BUILT_PROXY_IMAGE_ID} | tee -a $CI_PROJECT_DIR/image.env
+    - echo image_tag=${image_tag} | tee -a $CI_PROJECT_DIR/image.env
    # set image properties with repo state
    - openstack image set --property EXT_PR_SRC_REPO=${EXT_PR_SRC_REPO} --property EXT_PR_SRC_BRANCH_SHA=${EXT_PR_SRC_BRANCH_SHA} --property EXT_PR_TARGET_REPO=${EXT_PR_TARGET_REPO} --property EXT_PR_TARGET_BRANCH_SHA=${EXT_PR_TARGET_BRANCH_SHA} --property PACKER_IMAGE_HEAD=${PACKER_IMAGE_HEAD} ${BUILT_PROXY_IMAGE_ID}
  artifacts:
@@ -141,9 +147,12 @@ build_ssh_proxy_image:
    - *update_ansible_repo
    - *get_ansible_files
    # packer vars for job env
+    - image_tag=${BUILD_TARGET}${BUILD_TAG:+-${BUILD_TAG}}-${ENV}
    - export PKR_VAR_flavor="${PROXY_BUILD_FLAVOR:-$PKR_VAR_flavor}"
    - export PKR_VAR_build_instance_name="${BUILD_TARGET}-${EXT_REPO_HEAD}"
    - export PKR_VAR_image_date_suffix=false
+    - export PKR_VAR_image_tags="[\"${image_tag}\"]"
+    - export PKR_VAR_image_name=${image_tag}-${BUILD_DATE}
    - |
      if [ $CI_PIPELINE_SOURCE == 'merge_request_event' ]; then
        export PKR_VAR_image_name="${BUILD_TARGET}-PR-${CI_MERGE_REQUEST_IID}"
@@ -156,6 +165,7 @@ build_ssh_proxy_image:
    - packer build -machine-readable openstack-login | tee login_build.log
    - export BUILT_LOGIN_IMAGE_ID=$(grep 'Image:' login_build.log | awk '{print $4}')
    - echo BUILT_LOGIN_IMAGE_ID=${BUILT_LOGIN_IMAGE_ID} | tee -a $CI_PROJECT_DIR/image.env
+    - echo image_tag=${image_tag} | tee -a $CI_PROJECT_DIR/image.env
    # set image properties with repo state
    - openstack image set --property EXT_PR_SRC_REPO=${EXT_PR_SRC_REPO} --property EXT_PR_SRC_BRANCH_SHA=${EXT_PR_SRC_BRANCH_SHA} --property EXT_PR_TARGET_REPO=${EXT_PR_TARGET_REPO} --property EXT_PR_TARGET_BRANCH_SHA=${EXT_PR_TARGET_BRANCH_SHA} --property PACKER_IMAGE_HEAD=${CI_COMMIT_SHORT_SHA} ${BUILT_LOGIN_IMAGE_ID}
  artifacts:
@@ -183,10 +193,12 @@ build_ood_image:
    - *update_ansible_repo
    - *get_ansible_files
    # packer vars for job env
+    - image_tag=${BUILD_TARGET}${BUILD_TAG:+-${BUILD_TAG}}-${ENV}
    - export PKR_VAR_flavor="${OOD_BUILD_FLAVOR:-$PKR_VAR_flavor}"
    - export PKR_VAR_build_instance_name="${BUILD_TARGET}-${EXT_REPO_HEAD}"
    - export PKR_VAR_image_date_suffix=false
-    - export PKR_VAR_image_name="${BUILD_TARGET}-${BUILD_TAG:-${BUILD_DATE}}"
+    - export PKR_VAR_image_tags="[\"${image_tag}\"]"
+    - export PKR_VAR_image_name=${image_tag}-${BUILD_DATE}
    - |
      if [ $ENV = 'knightly' ] || [ $ENV = 'prod' ]; then
        curl --header "PRIVATE-TOKEN: ${ANSIBLE_VAR_TOKEN}" \
@@ -204,13 +216,14 @@ build_ood_image:
    - packer build -machine-readable openstack-ood | tee ood_build.log
    - export BUILT_OOD_IMAGE_ID=$(grep 'Image:' ood_build.log | awk '{print $4}')
    - echo BUILT_OOD_IMAGE_ID=${BUILT_OOD_IMAGE_ID} | tee -a $CI_PROJECT_DIR/image.env
+    - echo image_tag=${image_tag} | tee -a $CI_PROJECT_DIR/image.env
    # set image properties with repo state
    - openstack image set --property EXT_PR_SRC_REPO=${EXT_PR_SRC_REPO} --property EXT_PR_SRC_BRANCH_SHA=${EXT_PR_SRC_BRANCH_SHA} --property EXT_PR_TARGET_REPO=${EXT_PR_TARGET_REPO} --property EXT_PR_TARGET_BRANCH_SHA=${EXT_PR_TARGET_BRANCH_SHA} --property PACKER_IMAGE_HEAD=${CI_COMMIT_SHORT_SHA} ${BUILT_OOD_IMAGE_ID}
  artifacts:
    reports:
      dotenv: image.env
  rules:
-    - if: $PIPELINE_TARGET == "build" && $BUILD_TARGET == "ood"
+    - if: $BUILD_TARGET == "ood"
      when: always

 deploy_http_proxy_node:
@@ -373,3 +386,101 @@ deploy_login_node:
    - if: $PIPELINE_TARGET == "deploy" && $LOGIN_IMAGE_ID
      when: always

+deploy_ood_node:
+  stage: deploy
+  environment:
+    name: $ENV
+  tags:
+    - build
+  before_script:
+    - |
+      for OOD_FLOATING_IP in ${OOD_FLOATING_IP_LIST[@]}; do
+        OOD_FIXED_IP=$(
+          openstack floating ip list \
+          --floating-ip-address $OOD_FLOATING_IP -c "Fixed IP Address" -f value)
+        CURRENT_INSTANCE_ID=$(
+          openstack server list \
+          --name $OOD_INSTANCE_NAME --ip $OOD_FIXED_IP -c ID -f value)
+        openstack server remove floating ip $CURRENT_INSTANCE_ID $OOD_FLOATING_IP
+      done
+    - |
+      if [ -n "$OOD_PORT" ];then
+        openstack server remove port $CURRENT_INSTANCE_ID $OOD_PORT
+      fi
+
+  script:
+    - OOD_IMAGE_ID="${BUILT_OOD_IMAGE_ID:-$OOD_IMAGE_ID}"
+    - openstack image set --accept $OOD_IMAGE_ID || true
+    - FAILED=false
+    - |
+      cat > user_data.txt <<EOF
+      #!/bin/bash
+      cat >> /etc/NetworkManager/conf.d/90-dns-none.conf<<EEOF
+      [main]
+      dns=none
+      EEOF
+      systemctl reload NetworkManager
+      echo "$DEV_KEY" >> /root/.ssh/authorized_keys
+      ip route replace default via ${DEFAULT_GATEWAY_IP} dev eth0
+      git clone ${CI_REPOSITORY_URL} /tmp/${CI_PROJECT_NAME}
+      cd /tmp/${CI_PROJECT_NAME}
+      git checkout ${CI_COMMIT_REF_NAME}
+      cat >> ansible/hosts<<EEOF
+      [$ENV]
+      127.0.0.1
+      EEOF
+      s3cmd get --force -r --access_key=$AWS_ACCESS_KEY_ID --secret_key=$AWS_SECRET_ACCESS_KEY --host=$AWS_HOST --host-bucket=$AWS_HOST s3://cheaha-cloud-ansible-files/ /tmp/${CI_PROJECT_NAME}/ansible/files/
+      ansible-playbook -c local -i ansible/hosts --extra-vars="$EXTRA_VARS" ansible/cluster.yml | tee -a /tmp/ansible.log
+      rm -rf /tmp/${CI_PROJECT_NAME}
+      EOF
+    - |
+      export cmd="openstack server create"
+      cmd+=" -c id -f value --image $OOD_IMAGE_ID"
+      cmd+=" --flavor $INSTANCE_FLAVOR"
+      for security_group in ${SECURITY_GROUP_LIST[@]};
+      do
+        cmd+=" --security-group $security_group"
+      done
+      cmd+=" --user-data user_data.txt"
+      if [ -n "$INSTANCE_NETWORK" ];then cmd+=" --network $INSTANCE_NETWORK"; fi
+      if [ -n "$OOD_PORT" ];then cmd+=" --port $OOD_PORT"; fi
+      cmd+=" --wait $OOD_INSTANCE_NAME"
+    - export OOD_INSTANCE_ID=$(bash -c "$cmd")
+    - |
+      # Associate the floating IP(s) with the SSH Proxy instance
+      for OOD_FLOATING_IP in ${OOD_FLOATING_IP_LIST[@]};
+      do
+        echo "Associating FLOATING_IP $OOD_FLOATING_IP with OOD_INSTANCE_ID $OOD_INSTANCE_ID"
+        openstack server add floating ip $OOD_INSTANCE_ID $OOD_FLOATING_IP
+      done
+  rules:
+    - if: $DEPLOY_TARGET == "ood"
+      when: always
+
+cleanup_img:
+  stage: cleanup
+  environment:
+    name: $ENV
+  tags:
+    - build
+  script:
+    - |
+      OS_PROJECT_ID=$(
+        openstack application credential \
+        show $OS_APPLICATION_CREDENTIAL_ID -f value -c project_id
+      )
+    - |
+      IMAGES_TO_DELETE=($(
+        openstack image list --tag ${image_tag} \
+        --sort created_at:desc -f value -c ID \
+        --property owner=$OS_PROJECT_ID | tail -n +$((NUM_IMAGES_TO_KEEP+1))
+      ))
+    - |
+      for img in ${IMAGES_TO_DELETE[@]}; do
+        echo "Deleting image $img"
+        openstack image delete ${img}
+      done
+  rules:
+    - if: $NUM_IMAGES_TO_KEEP =~ /^(-\d+|0)$/
+      when: never
+    - if: $BUILD_TARGET
--- a/ansible/ansible.cfg
+++ b/ansible/ansible.cfg
+[defaults]
+# change the default callback, you can only have one 'stdout' type  enabled at a time.
+#stdout_callback = skippy
+stdout_callback = yaml
+
+## Ansible ships with some plugins that require whitelisting,
+## this is done to avoid running all of a type by default.
+## These setting lists those that you want enabled for your system.
+## Custom plugins should not need this unless plugin author specifies it.
+
+# enable callback plugins, they can output to stdout but cannot be 'stdout' type.
+callbacks_enabled = timer, debug, profile_roles, profile_tasks, minimal
+
+# Force color
+force_color = true
--- a/ansible/cluster.yml
+++ b/ansible/cluster.yml
@@ -14,3 +14,5 @@
    - { name: 'rewrite_map', tags: 'rewrite_map', when: enable_rewrite_map }
    - { name: 'fail2ban', tags: 'fail2ban', when: enable_fail2ban }
    - { name: 'install_node_exporter', tags: 'install_node_exporter', when: enable_node_exporter }
+    - { name: 'ood_config', tags: 'ood_config', when: enable_ood_config }
+
--- a/ansible/group_vars/all
+++ b/ansible/group_vars/all
@@ -89,3 +89,18 @@
  node_exporter_user: node_exporter
  node_exporter_group: node_exporter
  node_exporter_port: 9100
+
+# CentOS Repo
+  centos_base_url: "http://vault.centos.org"
+
+# ood_config
+  enable_ood_config: false
+  ood_internal_ip: OOD_INTERNAL_IP
+  ood_hostname: ood-gpfs5
+  login_hostname: login001
+  ood_domain: https://rc.uab.edu
+  account_app: account
+  account_app_port: 8000
+  account_app_bind_address: ["0.0.0.0:{{account_app_port}}"]
+  ood_user_regex: "([^!]+?)(@uab.edu)?$"
+  cluster_name: CoD
--- a/ansible/roles/fix_centos_repo/tasks/main.yaml
+++ b/ansible/roles/fix_centos_repo/tasks/main.yaml
@@ -15,6 +15,6 @@
  ansible.builtin.replace:
    path: "{{ item }}"
    regexp: '^#baseurl=http://mirror.centos.org'
-    replace: 'baseurl=http://vault.centos.org'
+    replace: 'baseurl={{ centos_base_url }}'
    backup: yes
  with_items: "{{ repo_files.stdout_lines }}"
--- a/ansible/roles/ldap_config/tasks/main.yml
+++ b/ansible/roles/ldap_config/tasks/main.yml
@@ -7,7 +7,6 @@
    - nss-pam-ldapd
    - openldap
    - openldap-clients
-    - openldap-servers
    - sssd-ldap

 - name: Update nsswitch.conf to look for ldap

--- a/ansible/roles/nfs_mounts/tasks/fstab.yml
+++ b/ansible/roles/nfs_mounts/tasks/fstab.yml
@@ -13,6 +13,6 @@
    path: "{{ item.path }}"
    opts: "{{ item.opts }}"
    state: mounted
-    fstype: nfs
+    fstype: "{{ item.fstype | default('nfs') }}"
  loop:
    "{{ mount_points }}"
--- a/ansible/roles/ood_config/files/shell-app.js
+++ b/ansible/roles/ood_config/files/shell-app.js
+const fs        = require('fs');
+const http      = require('http');
+const path      = require('path');
+const WebSocket = require('ws');
+const express   = require('express');
+const pty       = require('node-pty');
+const hbs       = require('hbs');
+const dotenv    = require('dotenv');
+const Tokens    = require('csrf');
+const url       = require('url');
+const yaml      = require('js-yaml');
+const glob      = require('glob');
+const port      = 3000;
+const host_path_rx = '/ssh/([^\\/\\?]+)([^\\?]+)?(\\?.*)?$';
+const helpers   = require('./utils/helpers');
+
+const pingInterval = 30000;
+
+
+// Read in environment variables
+dotenv.config({path: '.env.local'});
+if (process.env.NODE_ENV === 'production') {
+  dotenv.config({path: '/etc/ood/config/apps/shell/env'});
+}
+
+// Keep app backwards compatible
+if (fs.existsSync('.env')) {
+  console.warn('[DEPRECATION] The file \'.env\' is being deprecated. Please move this file to \'/etc/ood/config/apps/shell/env\'.');
+  dotenv.config({path: '.env'});
+}
+
+// Load color themes
+var color_themes = {dark: [], light: []};
+glob.sync('./color_themes/light/*').forEach(f => color_themes.light.push(require(path.resolve(f))));
+glob.sync('./color_themes/dark/*').forEach(f => color_themes.dark.push(require(path.resolve(f))));
+color_themes.json_array = JSON.stringify([...color_themes.light, ...color_themes.dark]);
+
+
+const tokens = new Tokens({});
+const secret = tokens.secretSync();
+
+// Create all your routes
+var router = express.Router();
+router.get(['/', '/ssh'], function (req, res) {
+  res.redirect(req.baseUrl + '/ssh/default');
+});
+
+router.get('/ssh*', function (req, res) {
+  var theHost, theDir;
+  [theHost, theDir] = host_and_dir_from_url(req.url);
+  res.render('index',
+    {
+      baseURI: req.baseUrl,
+      csrfToken: tokens.create(secret),
+      host: theHost,
+      dir: theDir,
+      colorThemes: color_themes,
+      siteTitle: (process.env.OOD_DASHBOARD_TITLE || "Open OnDemand"),
+    });
+});
+
+router.use(express.static(path.join(__dirname, 'public')));
+
+// Setup app
+var app = express();
+
+// Setup template engine
+app.set('view engine', 'hbs');
+app.set('views', path.join(__dirname, 'views'));
+
+// Mount the routes at the base URI
+app.use(process.env.PASSENGER_BASE_URI || '/', router);
+
+// Setup websocket server
+const server = new http.createServer(app);
+const wss = new WebSocket.Server({ noServer: true });
+
+let host_allowlist = new Set;
+if (process.env.OOD_SSHHOST_ALLOWLIST){
+  host_allowlist = new Set(process.env.OOD_SSHHOST_ALLOWLIST.split(':'));
+}
+
+let default_sshhost, first_available_host;
+glob.sync(path.join((process.env.OOD_CLUSTERS || '/etc/ood/config/clusters.d'), '*.y*ml'))
+  .map(yml => {
+    try {
+      return yaml.safeLoad(fs.readFileSync(yml));
+    } catch(err) { /** just keep going. dashboard should have an alert about it */}
+  })
+  .filter(config => (config && config.v2 && config.v2.login && config.v2.login.host) && ! (config.v2 && config.v2.metadata && config.v2.metadata.hidden))
+  .forEach((config) => {
+    let host = config.v2.login.host; //Already did checking above
+    let isDefault = config.v2.login.default;
+    host_allowlist.add(host);
+    if (isDefault) default_sshhost = host;
+    if (!first_available_host) first_available_host = host;
+  });
+
+default_sshhost = process.env.OOD_DEFAULT_SSHHOST || process.env.DEFAULT_SSHHOST || default_sshhost || first_available_host;
+if (default_sshhost) host_allowlist.add(default_sshhost);
+
+function host_and_dir_from_url(url){
+  let match = url.match(host_path_rx), 
+  hostname = null, 
+  directory = null;
+
+  if (match) {
+    hostname = match[1] === "default" ? default_sshhost : match[1];
+    directory = match[2] ? decodeURIComponent(match[2]) : null;
+  }
+  return [hostname, directory];
+}
+
+function heartbeat() {
+    this.isAlive = true;
+}
+
+wss.on('connection', function connection (ws, req) {
+  var dir,
+      term,
+      args,
+      host,
+      cmd = process.env.OOD_SSH_WRAPPER || 'ssh';
+
+  ws.isAlive = true;
+  ws.on('pong', heartbeat);
+  console.log('Connection established');
+
+  [host, dir] = host_and_dir_from_url(req.url);
+  args = dir ? [host, '-t', 'cd \'' + dir.replace(/\'/g, "'\\''") + '\' ; exec ${SHELL} -l'] : [host];
+
+  process.env.LANG = 'en_US.UTF-8'; // this patch (from b996d36) lost when removing wetty (2c8a022)
+
+  term = pty.spawn(cmd, args, {
+    name: 'xterm-16color',
+    cols: 80,
+    rows: 30
+  });
+
+  console.log('Opened terminal: ' + term.pid);
+
+  term.on('data', function (data) {
+    ws.send(data, function (error) {
+      if (error) console.log('Send error: ' + error.message);
+    });
+  });
+
+  term.on('error', function (error) {
+    ws.close();
+  });
+
+  term.on('close', function () {
+    ws.close();
+  });
+
+  ws.on('message', function (msg) {
+    msg = JSON.parse(msg);
+    if (msg.input)  term.write(msg.input);
+    if (msg.resize) term.resize(parseInt(msg.resize.cols), parseInt(msg.resize.rows));
+  });
+
+  ws.on('close', function () {
+    term.end();
+    console.log('Closed terminal: ' + term.pid);
+  });
+});
+
+const interval = setInterval(function ping() {
+  wss.clients.forEach(function each(ws) {
+    if (ws.isAlive === false) return ws.terminate();
+
+    ws.isAlive = false;
+    ws.ping();
+  });
+}, pingInterval);
+
+function custom_server_origin(default_value = null){
+  var custom_origin = null;
+
+  if(process.env.OOD_SHELL_ORIGIN_CHECK) {
+    // if ENV is set, do not use default!
+    if(process.env.OOD_SHELL_ORIGIN_CHECK.startsWith('http')){
+      custom_origin = process.env.OOD_SHELL_ORIGIN_CHECK;
+    }
+  }
+  else {
+    custom_origin = default_value;
+  }
+
+  return custom_origin;
+}
+
+function default_server_origin(headers){
+  var origin = null;
+
+  if (headers['x-forwarded-proto'] && headers['x-forwarded-host']){
+    origin = headers['x-forwarded-proto'] + "://" + headers['x-forwarded-host']
+  }
+
+  return origin;
+}
+
+server.on('upgrade', function upgrade(request, socket, head) {
+  const requestToken = new URLSearchParams(url.parse(request.url).search).get('csrf'),
+        client_origin = request.headers['origin'],
+        server_origin = custom_server_origin(default_server_origin(request.headers));
+
+  var host, dir;
+  [host, dir] = host_and_dir_from_url(request.url);
+
+  if (client_origin &&
+      client_origin.startsWith('http') &&
+      server_origin && client_origin !== server_origin) {
+    socket.write([
+      'HTTP/1.1 401 Unauthorized',
+      'Content-Type: text/html; charset=UTF-8',
+      'Content-Encoding: UTF-8',
+      'Connection: close',
+      'X-OOD-Failure-Reason: invalid origin',
+    ].join('\r\n') + '\r\n\r\n');
+
+    socket.destroy();
+  } else if (!tokens.verify(secret, requestToken)) {
+    socket.write([
+      'HTTP/1.1 401 Unauthorized',
+      'Content-Type: text/html; charset=UTF-8',
+      'Content-Encoding: UTF-8',
+      'Connection: close',
+      'X-OOD-Failure-Reason: bad csrf token',
+    ].join('\r\n') + '\r\n\r\n');
+
+    socket.destroy();
+  } else if (!helpers.hostInAllowList(host_allowlist, host)) { // host not in allowlist
+    socket.write([
+      'HTTP/1.1 401 Unauthorized',
+      'Content-Type: text/html; charset=UTF-8',
+      'Content-Encoding: UTF-8',
+      'Connection: close',
+      'X-OOD-Failure-Reason: host not specified in allowlist or cluster configs',
+    ].join('\r\n') + '\r\n\r\n');
+
+    socket.destroy();
+  } else {
+    wss.handleUpgrade(request, socket, head, function done(ws) {
+      wss.emit('connection', ws, request);
+    });
+  }
+});
+
+server.listen(port, function () {
+  console.log('Listening on ' + port);
+});
--- a/ansible/roles/ood_config/tasks/main.yml
+++ b/ansible/roles/ood_config/tasks/main.yml
+---
+- name: Config ood to run behind proxy
+  ansible.builtin.template:
+    src: ood_proxy.conf.j2
+    dest: /opt/rh/httpd24/root/etc/httpd/conf.d/ood-proxy.conf
+
+- name: Patch shell app with shell timeout fix
+  ansible.builtin.copy:
+    src: shell-app.js
+    dest: /var/www/ood/apps/sys/shell/app.js
+
+- name: Create a directory if it does not exist
+  ansible.builtin.file:
+    path: /etc/ood/config/apps/shell
+    state: directory
+    mode: "0755"
+
+- name: Shell app configuration env
+  ansible.builtin.template:
+    src: shell_app.env.j2
+    dest: /etc/ood/config/apps/shell/env
+
+- name: Point shell app to login node
+  ansible.builtin.replace:
+    path: /etc/ood/config/clusters.d/{{ cluster_name }}.yml
+    regexp: '^(\s+host:).*'
+    replace: '\1 "{{ login_hostname }}"'
+    backup: yes
+
+- name: Modify account app binding to listen to http-proxy
+  ansible.builtin.lineinfile:
+    path: /var/www/ood/register/{{account_app}}/{{account_app}}.ini
+    regexp: '^(bind\s=).*'
+    line: "bind = {{ account_app_bind_address }}"
+
+- name: Restart httpd24-httpd
+  ansible.builtin.service:
+    name: httpd24-httpd
+    state: restarted
--- a/ansible/roles/ood_config/templates/ood_proxy.conf.j2
+++ b/ansible/roles/ood_config/templates/ood_proxy.conf.j2
+#
+# Open OnDemand Portal
+#
+# Generated using ood-portal-generator version 0.8.0
+#
+# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+# !!                                    !!
+# !!     DO NOT EDIT THIS FILE          !!
+# !!                                    !!
+# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+#
+# This file is auto-generated by ood-portal-generator and will be over-written
+# in future updates.
+#
+# 1. To modify this file, first update the global configuration file:
+#
+#      /etc/ood/config/ood_portal.yml
+#
+#    You can find more information about the ood-portal-generator configuration
+#    at:
+#
+#      https://osc.github.io/ood-documentation/master/infrastructure/ood-portal-generator.html
+#
+# 2. Then build/install the updated Apache config with:
+#
+#      sudo /opt/ood/ood-portal-generator/sbin/update_ood_portal
+#
+# 3. Finally, restart Apache to have the changes take effect:
+#
+#      # For CentOS 6
+#      sudo service httpd24-httpd condrestart
+#      sudo service httpd24-htcacheclean condrestart
+#
+#      # For CentOS 7
+#      sudo systemctl try-restart httpd24-httpd.service httpd24-htcacheclean.service
+#
+
+
+
+# The Open OnDemand portal VirtualHost
+#
+<VirtualHost {{ ood_internal_ip }}:80>
+  ServerName {{ ood_hostname }}
+
+  ErrorLog  "logs/{{ ood_hostname }}_error.log"
+  CustomLog "logs/{{ ood_hostname }}_access.log" combined
+
+  RewriteEngine On
+  RewriteCond %{HTTP_HOST} !^(rc.uab.edu(:80)?)?$ [NC]
+  RewriteRule ^(.*) http://rc.uab.edu:80$1 [R=301,NE,L]
+
+
+  # Lua configuration
+  #
+  LuaRoot "/opt/ood/mod_ood_proxy/lib"
+  LogLevel lua_module:info
+
+  # Log authenticated user requests (requires min log level: info)
+  LuaHookLog logger.lua logger
+
+  # Authenticated-user to system-user mapping configuration
+  #
+  SetEnv OOD_USER_MAP_CMD "/opt/ood/ood_auth_map/bin/user_auth.py"
+  SetEnv OOD_USER_ENV "REMOTE_USER"
+  SetEnv OOD_MAP_FAIL_URI "/account"
+
+  # Per-user Nginx (PUN) configuration
+  # NB: Apache will need sudo privs to control the PUNs
+  #
+  SetEnv OOD_PUN_STAGE_CMD "sudo /opt/ood/nginx_stage/sbin/nginx_stage"
+
+  #
+  # Below is used for sub-uri's this Open OnDemand portal supports
+  #
+
+  # Serve up publicly available assets from local file system:
+  #
+  #     http://{{ ood_hostname }}:80/public/favicon.ico
+  #     #=> /var/www/ood/public/favicon.ico
+  #
+  Alias "/public" "/var/www/ood/public"
+  <Directory "/var/www/ood/public">
+    Options Indexes FollowSymLinks
+    AllowOverride None
+    Require all granted
+  </Directory>
+
+  # Reverse proxy traffic to backend webserver through IP sockets:
+  #
+  #     http://{{ ood_hostname }}:80/node/HOST/PORT/index.html
+  #     #=> http://HOST:PORT/node/HOST/PORT/index.html
+  #
+  <LocationMatch "^/node/(?<host>c\d+)/(?<port>\d+)">
+    RewriteCond %{IS_SUBREQ} ^false$
+    RewriteCond %{HTTP:Proxy-user} "([^!]+?)(@uab.edu)?$"
+    RewriteRule . - [E=REMOTE_USER:%1]
+
+    # ProxyPassReverse implementation
+    Header edit Location "^[^/]+//[^/]+" ""
+
+    # ProxyPassReverseCookieDomain implemenation
+    Header edit* Set-Cookie ";\s*(?i)Domain[^;]*" ""
+
+    # ProxyPassReverseCookiePath implementation
+    Header edit* Set-Cookie ";\s*(?i)Path[^;]*" ""
+    Header edit  Set-Cookie "^([^;]+)" "$1; Path=/node/%{MATCH_HOST}e/%{MATCH_PORT}e"
+
+    LuaHookFixups node_proxy.lua node_proxy_handler
+  </LocationMatch>
+
+  # Reverse "relative" proxy traffic to backend webserver through IP sockets:
+  #
+  #     http://{{ ood_hostname }}:80/rnode/HOST/PORT/index.html
+  #     #=> http://HOST:PORT/index.html
+  #
+  <LocationMatch "^/rnode/(?<host>c\d+)/(?<port>\d+)(?<uri>/.*|)">
+    RewriteCond %{IS_SUBREQ} ^false$
+    RewriteCond %{HTTP:Proxy-user} "([^!]+?)(@uab.edu)?$"
+    RewriteRule . - [E=REMOTE_USER:%1]
+
+    # ProxyPassReverse implementation
+    Header edit Location "^([^/]+//[^/]+)|(?=/)" "/rnode/%{MATCH_HOST}e/%{MATCH_PORT}e"
+
+    # ProxyPassReverseCookieDomain implemenation
+    Header edit* Set-Cookie ";\s*(?i)Domain[^;]*" ""
+
+    # ProxyPassReverseCookiePath implementation
+    Header edit* Set-Cookie ";\s*(?i)Path[^;]*" ""
+    Header edit  Set-Cookie "^([^;]+)" "$1; Path=/rnode/%{MATCH_HOST}e/%{MATCH_PORT}e"
+
+    LuaHookFixups node_proxy.lua node_proxy_handler
+  </LocationMatch>
+
+  # Reverse proxy traffic to backend PUNs through Unix domain sockets:
+  #
+  #     http://{{ ood_hostname }}:80/pun/dev/app/simulations/1
+  #     #=> unix:/path/to/socket|http://localhost/pun/dev/app/simulations/1
+  #
+  SetEnv OOD_PUN_URI "/pun"
+  <Location "/pun">
+    RewriteCond %{IS_SUBREQ} ^false$
+    RewriteCond %{HTTP:Proxy-user} "([^!]+?)(@uab.edu)?$"
+    RewriteRule . - [E=REMOTE_USER:%1]
+
+    ProxyPassReverse "http://localhost/pun"
+
+    # ProxyPassReverseCookieDomain implementation (strip domain)
+    Header edit* Set-Cookie ";\s*(?i)Domain[^;]*" ""
+
+    # ProxyPassReverseCookiePath implementation (less restrictive)
+    Header edit* Set-Cookie ";\s*(?i)Path\s*=(?-i)(?!\s*/pun)[^;]*" "; Path=/pun"
+
+    SetEnv OOD_PUN_SOCKET_ROOT "/var/run/ondemand-nginx"
+    SetEnv OOD_PUN_MAX_RETRIES "5"
+    LuaHookFixups pun_proxy.lua pun_proxy_handler
+
+  </Location>
+
+  # Control backend PUN for authenticated user:
+  # NB: See mod_ood_proxy for more details.
+  #
+  #    http://{{ ood_hostname }}:80/nginx/stop
+  #    #=> stops the authenticated user's PUN
+  #
+  SetEnv OOD_NGINX_URI "/nginx"
+  <Location "/nginx">
+    RewriteCond %{IS_SUBREQ} ^false$
+    RewriteCond %{HTTP:Proxy-user} "([^!]+?)(@uab.edu)?$"
+    RewriteRule . - [E=REMOTE_USER:%1]
+
+    LuaHookFixups nginx.lua nginx_handler
+  </Location>
+
+  # Redirect root URI to specified URI
+  #
+  #     http://{{ ood_hostname }}:80/
+  #     #=> http://{{ ood_hostname }}:80/pun/sys/dashboard
+  #
+  RedirectMatch ^/$ "/pun/sys/dashboard"
+
+  # Redirect logout URI to specified redirect URI
+  #
+  #     http://{{ ood_hostname }}:80/logout
+  #     #=> http://{{ ood_hostname }}:80/pun/sys/dashboard/logout
+  #
+  Redirect "/logout" "/pun/sys/dashboard/logout"
+
+
+
+
+  # Register and/or unregister the mapping of an authenticated-user to a system-user
+  # NB: This is not needed for regular expression mapping
+  #
+  #     http://{{ ood_hostname }}:80/account
+  #     #=> /var/www/ood/register/
+  #
+  Alias "/account" "/var/www/ood/register"
+  <Directory "/var/www/ood/register">
+    Options Indexes FollowSymLinks
+    AllowOverride None
+
+    RewriteCond %{IS_SUBREQ} ^false$
+    RewriteCond %{HTTP:Proxy-user} "([^!]+?)(@uab.edu)?$"
+    RewriteRule . - [E=REMOTE_USER:%1]
+  </Directory>
+</VirtualHost>
--- a/ansible/roles/ood_config/templates/shell_app.env.j2
+++ b/ansible/roles/ood_config/templates/shell_app.env.j2
+OOD_SHELL_ORIGIN_CHECK='{{ ood_domain }}'
--- a/ansible/roles/ssh_proxy_config/templates/sshpiperd.yaml.j2
+++ b/ansible/roles/ssh_proxy_config/templates/sshpiperd.yaml.j2
@@ -24,5 +24,11 @@ pipes:
    host: "{{ group.host }}"
    ignore_hostkey: true
    private_key: "{{ group.private_key }}"
+- from:
+    - username: ".*"
+      username_regex_match: true
+  to:
+    host: "{{ group.host }}"
+    ignore_hostkey: true
 {% endif %}
 {% endfor %}
--- a/openstack-ood/nodeimage.pkr.hcl
+++ b/openstack-ood/nodeimage.pkr.hcl
@@ -40,6 +40,9 @@ build {
    use_proxy     = false
    user          = var.ssh_username
    groups        = ["ood"]
+    ansible_env_vars = [
+      "ANSIBLE_CONFIG=./ansible/ansible.cfg"
+    ]
    playbook_file = "./ansible/ood.yml"
    roles_path    = "./ansible/roles"
    extra_arguments = [
@@ -51,7 +54,10 @@ build {
    use_proxy        = false
    user             = var.ssh_username
    groups           = ["ood", "knightly"]
-    ansible_env_vars = ["ANSIBLE_HOST_KEY_CHECKING=False"]
+    ansible_env_vars = [
+      "ANSIBLE_HOST_KEY_CHECKING=False",
+      "ANSIBLE_CONFIG=./CRI_XCBC/ansible.cfg"
+    ]
    playbook_file    = "./CRI_XCBC/ood-packer.yaml"
    extra_arguments  = [
      "--extra-vars", "${var.extra_vars}"
@@ -60,7 +66,7 @@ build {

  provisioner "shell" {
    inline = [
-      "sudo yum install -y libselinux-python3 python3 python3-pip tmux vim git bash-completion curl wget unzip",
+      "sudo yum install -y libselinux-python3 python3 python3-pip tmux vim git bash-completion curl wget unzip NetworkManager",
      "sudo python3 -m pip install --upgrade pip",
      "sudo pip3 install s3cmd==2.3.0 ansible==4.10.0 python-openstackclient==5.8.0"
    ]

--- a/requirements.txt
+++ b/requirements.txt
+certifi==2025.1.31
+charset-normalizer==3.4.1
+idna==3.10
+python-gitlab==5.6.0
+requests==2.32.3
+requests-toolbelt==1.0.0
+urllib3==2.4.0
--- a/utils/README.md
+++ b/utils/README.md
+### Description
+These utility scripts avoid copying each ci variable manually which is tedious.
+- The gitlab-ci-vars-reader.py reads variables from a specific project or a pipeline (depending on the options provided) and copies them into a yaml file
+- The gitlab-ci-vars-updater.py takes a yaml file containing key value pairs in yaml format as an input. It then creates/updates project variables or pipeline variables (depending on the options provided)
+
+### Prerequisites
+```
+python -m venv ~/venvs/gitlab
+source ~/venvs/gitlab/bin/activate
+pip install -r requirements
+```
+
+### Setup
+```
+cd utils
+mv gitlab.ini.example gitlab.ini
+```
+Make changes to the gitlab.ini as you require.
+[Create a personal access token](https://docs.gitlab.com/user/profile/personal_access_tokens/) via the gitlab UI and copy it to the private_token field in gitlab.ini file
+
+### Usage
+> Create an empty schedule pipeline before you try this out.
+```
+python3 gitlab-ci-vars-reader.py --config_file gitlab.ini --project_id <PROJECT_ID> --sched_pipeline_id <PIPELINE_ID> --var_file ci-variables.yaml
+
+python3 gitlab-ci-vars-updater.py --config_file gitlab.ini --project_id <PROJECT_ID> --sched_pipeline_id <NEW-PIPELINE_ID> --var_file ci-variables.yaml
+```
--- a/utils/gitlab-ci-vars-reader.py
+++ b/utils/gitlab-ci-vars-reader.py
+import argparse
+
+import gitlab
+import yaml
+
+
+# Function to fetch all CI/CD variables from a GitLab project
+def fetch_variables(project):
+    p_variables = list(project.variables.list(iterator=True))
+    variables = [var.asdict() for var in p_variables]
+
+    return variables
+
+
+def fetch_sched_variables(sched_pipeline):
+    variables = sched_pipeline.attributes["variables"]
+    return variables
+
+
+# Main function to load the config and fetch variables
+def main():
+    # Setup argument parser
+    parser = argparse.ArgumentParser(description="GitLab CI/CD Variable reader")
+    parser.add_argument(
+        "--config_file",
+        type=str,
+        default="gitlab.ini",
+        required=True,
+        help="Path to the configuration file (default: gitlab.ini)",
+    )
+    parser.add_argument(
+        "--var_file",
+        type=str,
+        default="ci-variables.yaml",
+        help="Path to the CI vars file (default: ci-variables.yaml)",
+    )
+    parser.add_argument(
+        "--project_name",
+        type=str,
+        required=True,
+        help="Gitlab project name with namespace",
+    )
+    parser.add_argument(
+        "--sched_pipeline_id",
+        type=int,
+        help="Gitlab project scheduled pipeline ID",
+    )
+
+    # Parse the arguments
+    args = parser.parse_args()
+
+    gl = gitlab.Gitlab.from_config("uabrc", [args.config_file])
+    project = gl.projects.get(args.project_name)
+
+    # Fetch project or sched pipeline variables
+    if not args.sched_pipeline_id:
+        variables = fetch_variables(project)
+    else:
+        sched_pipeline = project.pipelineschedules.get(args.sched_pipeline_id)
+        variables = fetch_sched_variables(sched_pipeline)
+
+    try:
+        with open(args.var_file, mode="wt", encoding="utf-8") as file:
+            yaml.dump(variables, file, explicit_start=True)
+    except FileNotFoundError:
+        print(f"Error: Writing File to '{args.var_file}'")
+        exit(1)
+
+
+# Run the main function
+if __name__ == "__main__":
+    main()
--- a/utils/gitlab-ci-vars-updater.py
+++ b/utils/gitlab-ci-vars-updater.py
+import argparse
+
+import gitlab
+import yaml
+
+
+def load_file(file_path):
+    try:
+        with open(file_path, mode="rt", encoding="utf-8") as file:
+            return yaml.safe_load(file)
+    except FileNotFoundError:
+        print(f"Error: Configuration file '{file_path}' not found.")
+        exit(1)
+
+
+# Function to create or update a GitLab CI/CD variable
+def create_or_update_variable(project, var_dict):
+    key = var_dict.get("key")
+    scope = var_dict.get("environment_scope", "*")
+    p_variable = None
+
+    DEFAULTS = {
+        "variable_type": "env_var",
+        "hidden": False,
+        "protected": False,
+        "masked": False,
+        "environment_scope": "*",
+        "raw": False,
+        "description": None,
+    }
+
+    # Merge defaults with var_dict
+    var_dict = {**DEFAULTS, **var_dict}
+
+    # Fetch a variable with matching key and scope
+    try:
+        all_vars = project.variables.list(get_all=True)
+        for var in all_vars:
+            if var.key == key and var.environment_scope == scope:
+                p_variable = var
+                break
+    except gitlab.exceptions.GitlabGetError:
+        print("Variable not found")
+        exit(1)
+
+    # Check if the variable exists and same as input
+    if p_variable is not None:
+        if p_variable.asdict() != var_dict:
+            # if not same update the project variable
+            print(f"Updating {p_variable.attributes['key']}")
+            p_variable.delete()
+            return project.variables.create(var_dict)
+        else:
+            print(f"variable {var_dict["key"]} already exists")
+    # Create variable if it doesn't exist in the project
+    else:
+        print(f"Creating variable {var_dict["key"]}")
+        return project.variables.create(var_dict)
+
+
+def get_pipeline_vars_by_key(sched_pipeline, key_name):
+    p_vars = sched_pipeline.attributes["variables"]
+    for p_variable in p_vars:
+        if p_variable.get("key") == key_name:
+            return p_variable
+
+
+# Function to create or update a schedule pipeline variable
+def create_or_update_sched_vars(sched_pipeline, var_dict):
+    # Check if the variable exists in the sched pipeline
+    p_variable = get_pipeline_vars_by_key(sched_pipeline, var_dict["key"])
+    if p_variable:
+        # Check if the attributes are the same
+        if p_variable != var_dict:
+            # If not update the value in the project
+            sched_pipeline.variables.delete(p_variable["key"])
+            sched_pipeline.variables.create(var_dict)
+        else:
+            print(f"variable {var_dict["key"]} already exists")
+    # Create variable if it doesn't exist in the project
+    else:
+        print(f"Creating variable {var_dict["key"]}")
+        return sched_pipeline.variables.create(var_dict)
+
+
+def main():
+    # Setup argument parser
+    parser = argparse.ArgumentParser(description="GitLab CI/CD Variables Updater")
+    parser.add_argument(
+        "--config_file",
+        type=str,
+        default="gitlab.ini",
+        required=True,
+        help="Path to the configuration file (default: gitlab.ini)",
+    )
+    parser.add_argument(
+        "--var_file",
+        type=str,
+        default="ci-variables.yaml",
+        help="Path to the CI vars file (default: ci-variables.yaml)",
+    )
+    parser.add_argument(
+        "--project_name",
+        type=str,
+        required=True,
+        help="Gitlab project name with namespace",
+    )
+    parser.add_argument(
+        "--sched_pipeline_id",
+        type=int,
+        help="Gitlab project scheduled pipeline ID",
+    )
+
+    # Parse the arguments
+    args = parser.parse_args()
+
+    gl = gitlab.Gitlab.from_config("uabrc", [args.config_file])
+    project = gl.projects.get(args.project_name)
+
+    # Load the CI vars file
+    var_list = load_file(args.var_file)
+
+    # Create or update all variables
+    for var_dict in var_list:
+        if not args.sched_pipeline_id:
+            create_or_update_variable(project, var_dict)
+        else:
+            sched_pipeline = project.pipelineschedules.get(args.sched_pipeline_id)
+            create_or_update_sched_vars(sched_pipeline, var_dict)
+
+
+if __name__ == "__main__":
+    main()
--- a/utils/gitlab.ini.example
+++ b/utils/gitlab.ini.example
+[global]
+default = uabrc
+ssl_verify = true
+timeout = 5
+per_page = 100
+
+[uabrc]
+url = https://gitlab.rc.uab.edu
+private_token =
+api_version = 4
No results found