Merge branch 'feat-hpc-factory' into 'feat-ood-image-build'

# Conflicts:
#   openstack-ood/variables.pkr.hcl

.gitlab-ci.yml

@@ -246,10 +246,12 @@ deploy_http_proxy_node:
       export cmd="openstack server create"
       cmd+=" -c id -f value --image $HTTP_PROXY_IMAGE_ID"
       cmd+=" --flavor $INSTANCE_FLAVOR"
-      cmd+=" --network $PROXY_NETWORK"
-      cmd+=" --security-group webserver_sec_group"
-      cmd+=" --security-group allow-ssh"
+      for security_group in ${SECURITY_GROUP_LIST[@]};
+      do
+        cmd+=" --security-group $security_group"
+      done
       cmd+=" --user-data user_data.txt"
+      if [ -n "$PROXY_NETWORK" ];then cmd+=" --network $PROXY_NETWORK"; fi
       if [ -n "$HTTP_PROXY_PORT" ];then cmd+=" --port $HTTP_PROXY_PORT"; fi
       cmd+=" --wait $HTTP_PROXY_INSTANCE_NAME"
     - export HTTP_PROXY_INSTANCE_ID=$(bash -c "$cmd")
@@ -297,9 +299,12 @@ deploy_ssh_proxy_node:
       export cmd="openstack server create"
       cmd+=" -c id -f value --image $SSH_PROXY_IMAGE_ID"
       cmd+=" --flavor $INSTANCE_FLAVOR"
-      cmd+=" --network $PROXY_NETWORK"
-      cmd+=" --security-group allow-ssh"
+      for security_group in ${SECURITY_GROUP_LIST[@]};
+      do
+        cmd+=" --security-group $security_group"
+      done
       cmd+=" --user-data user_data.txt"
+      if [ -n "$PROXY_NETWORK" ];then cmd+=" --network $PROXY_NETWORK"; fi
       if [ -n "$SSH_PROXY_PORT" ];then cmd+=" --port $SSH_PROXY_PORT"; fi
       cmd+=" --wait $SSH_PROXY_INSTANCE_NAME"
     - export SSH_PROXY_INSTANCE_ID=$(bash -c "$cmd")
@@ -348,9 +353,12 @@ deploy_login_node:
       export cmd="openstack server create"
       cmd+=" -c id -f value --image $LOGIN_IMAGE_ID"
       cmd+=" --flavor $INSTANCE_FLAVOR"
-      cmd+=" --network $INSTANCE_NETWORK"
-      cmd+=" --security-group allow-ssh"
+      for security_group in ${SECURITY_GROUP_LIST[@]};
+      do
+        cmd+=" --security-group $security_group"
+      done
       cmd+=" --user-data user_data.txt"
+      if [ -n "$INSTANCE_NETWORK" ];then cmd+=" --network $INSTANCE_NETWORK"; fi
       if [ -n "$LOGIN_PORT" ];then cmd+=" --port $LOGIN_PORT"; fi
       cmd+=" --wait $LOGIN_INSTANCE_NAME"
     - export LOGIN_INSTANCE_ID=$(bash -c "$cmd")

ansible/cluster.yml

@@ -12,3 +12,5 @@
     - { name: 'ssl_cert', tags: 'ssl_cert', when: enable_ssl_certs }
     - { name: 'rsyslog_config', tags: 'rsyslog_config', when: enable_rsyslog_config }
     - { name: 'rewrite_map', tags: 'rewrite_map', when: enable_rewrite_map }
+    - { name: 'fail2ban', tags: 'fail2ban', when: enable_fail2ban }
+    - { name: 'install_node_exporter', tags: 'install_node_exporter', when: enable_node_exporter }

ansible/group_vars/all

@@ -50,10 +50,9 @@
 # ssh proxy
   enable_ssh_proxy_config: false
   sshpiper_dest_dir: "/opt/sshpiper"
-  fail2ban_cidr_list: "127.0.0.1/8"
 
 # rsyslog
-  enable_rsyslog_config: false
+  enable_rsyslog_config: true
   rsyslog_target: "*.* @master:514"
 
 # ssl certs
@@ -73,3 +72,20 @@
     - {"name": "gpfs4", "host": "login001", "default": True }
     - {"name": "gpfs5", "host": "login002", "default": False }
 
+# account app
+  account_app_port: 8000
+
+# fail2ban
+  enable_fail2ban: false
+  maxretry: 1
+  findtime: 600
+  bantime: 1200
+  fail2ban_white_list: "127.0.0.1/8"
+
+# Node Exporter
+  enable_node_exporter: false
+  node_exporter_ver: "1.8.2"
+  node_exporter_filename: "node_exporter-{{ node_exporter_ver }}.linux-amd64"
+  node_exporter_user: node_exporter
+  node_exporter_group: node_exporter
+  node_exporter_port: 9100

ansible/roles/fail2ban/tasks/main.yml

+---
+
+- name: Install fail2ban
+  ansible.builtin.package:
+    name: "{{ item }}"
+    state: present
+  loop:
+    - fail2ban
+    - fail2ban-firewalld
+
+- name: Configure fail2ban
+  ansible.builtin.template:
+    src: "{{ item.src }}"
+    dest: "{{ item.dest }}"
+    backup: true
+  loop:
+    - { src: 'jail.local.j2', dest: '/etc/fail2ban/jail.local' }
+    - { src: 'sshpiperd_filter.local.j2', dest: '/etc/fail2ban/filter.d/sshpiperd.local' }
+    - { src: 'sshpiperd_jail.local.j2', dest: '/etc/fail2ban/jail.d/sshpiperd.local' }
+
+- name: Activate the firewalld support for fail2ban
+  ansible.builtin.command:
+    cmd: mv /etc/fail2ban/jail.d/00-firewalld.conf /etc/fail2ban/jail.d/00-firewalld.local
+
+- name: Configure firewalld to allow ssh and sshpiper traffic
+  ansible.posix.firewalld:
+    port: "{{ item }}"
+    zone: public
+    state: enabled
+    permanent: true
+  loop:
+    - 2222/tcp
+    - 22/tcp
+
+- name: Enable and start firewalld
+  ansible.builtin.service:
+    name: firewalld
+    enabled: true
+    state: restarted
+
+- name: Enable and start fail2ban
+  ansible.builtin.service:
+    name: fail2ban
+    enabled: true
+    state: restarted
+

ansible/roles/ssh_proxy_config/templates/jail.local.j2 → ansible/roles/fail2ban/templates/jail.local.j2

 [DEFAULT]
 banaction = firewalld
-bantime  = 1200
-ignoreip = {{ fail2ban_cidr_list }}
+bantime  = {{ bantime }}
+ignoreip = {{ fail2ban_white_list }}
 
 [sshd]
 enabled = true

ansible/roles/fail2ban/templates/sshpiperd_filter.local.j2

+# Refer to https://github.com/fail2ban/fail2ban/wiki/Developing-Regex-in-Fail2ban for developing regex using fail2ban
+#
+[INCLUDES]
+before = common.conf
+
+[DEFAULT]
+_daemon = sshpiperd
+__iso_datetime = "\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:[+-]\d{2}:\d{2}|Z)"
+__pref = time=%(__iso_datetime)s level=(?:debug|error)
+
+[Definition]
+# Define the prefix regex for the log lines
+prefregex = ^<F-MLFID>%(__prefix_line)s%(__pref)s</F-MLFID>\s+<F-CONTENT>.+</F-CONTENT>$
+
+# Failregex to match the specific failure log lines (prefregex is automatically included)
+failregex = ^msg="connection from .*failtoban: ip <HOST> too auth many failures"$
+
+ignoreregex =
+
+mode = normal
+
+maxlines = 1

ansible/roles/fail2ban/templates/sshpiperd_jail.local.j2

+# This configuration will block the remote host after {{maxretry}} failed SSH login attempts.
+[sshpiperd]
+enabled  = true
+filter   = sshpiperd
+logpath  = /var/log/messages
+port     = 22
+maxretry = {{ maxretry }}
+backend  = auto
+findtime = {{ findtime }}

ansible/roles/install_node_exporter/tasks/main.yaml

+---
+- name: Download node_exporter binary
+  ansible.builtin.get_url:
+    url: "https://github.com/prometheus/node_exporter/releases/download/v{{ node_exporter_ver }}/{{ node_exporter_filename }}.tar.gz"
+    dest: "/tmp/{{ node_exporter_filename }}.tar.gz"
+
+- name: Extract node_exporter
+  ansible.builtin.unarchive:
+    src: "/tmp/{{ node_exporter_filename }}.tar.gz"
+    dest: "/tmp"
+    remote_src: yes
+
+- name: Create system group for user account {{ node_exporter_group }}
+  ansible.builtin.group:
+    name: "{{ node_exporter_group }}"
+    system: true
+    state: present
+
+- name: Create system user account {{ node_exporter_user }}
+  ansible.builtin.user:
+    name: "{{ node_exporter_user }}"
+    comment: Prometheus node_exporter system account
+    group: "{{ node_exporter_group }}"
+    system: true
+    home: /var/lib/node_exporter
+    create_home: false
+    shell: /sbin/nologin
+    state: present
+
+- name: Copy node_exporter binary
+  ansible.builtin.copy:
+    src: "/tmp/{{ node_exporter_filename }}/node_exporter"
+    dest: /usr/local/bin/node_exporter
+    remote_src: yes
+    owner: root
+    group: root
+    mode: 0755
+
+- name: Copy systemd unit file
+  ansible.builtin.template:
+    src: node_exporter.service.j2
+    dest: /etc/systemd/system/node_exporter.service
+    owner: root
+    group: root
+    mode: '0644'
+
+- name: Clean up /tmp
+  ansible.builtin.file:
+    path: "/tmp/{{ item }}"
+    state: absent
+  loop:
+    - "{{ node_exporter_filename }}.tar.gz"
+    - "{{ node_exporter_filename }}"
+
+- name: Restart node_exporter service
+  ansible.builtin.systemd:
+    daemon_reload: yes
+    name: node_exporter
+    state: restarted
+    enabled: true
+
+- name: Collect facts about system services
+  ansible.builtin.service_facts:
+
+- name: Configure firewalld to allow prometheus
+  ansible.posix.firewalld:
+    port: "{{ node_exporter_port }}/tcp"
+    zone: public
+    state: enabled
+    permanent: true
+  when:
+    - "'firewalld.service' in ansible_facts.services"
+    - ansible_facts.services["firewalld.service"].state == "running"
+
+- name: Enable and start firewalld
+  ansible.builtin.service:
+    name: firewalld
+    enabled: true
+    state: restarted
+  when:
+    - "'firewalld.service' in ansible_facts.services"
+    - ansible_facts.services["firewalld.service"].state == "running"

ansible/roles/install_node_exporter/templates/node_exporter.service.j2

+[Unit]
+Description=Node Exporter
+After=network.target
+
+[Service]
+User={{ node_exporter_user }}
+Group={{ node_exporter_group }}
+Type=simple
+ExecStart=/usr/local/bin/node_exporter --web.listen-address=:{{ node_exporter_port }} --collector.filesystem.mount-points-exclude "^/(dev|proc|run/user/.+|run/credentials/.+|sys|var/lib/docker/.+)($|/)" --collector.filesystem.fs-types-exclude "^(autofs|binfmt_misc|bpf|cgroup|tmpfs|sunrpc|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$"
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/rewrite_map/tasks/main.yaml

@@ -15,6 +15,12 @@
       RewriteCond %{HTTP:REMOTE_USER} '([a-zA-Z0-9_.+-]+)@uab.edu$' [OR]
           RewriteCond %{HTTP:REMOTE_USER} 'urn:mace:incommon:uab.edu!https://uabgrid.uab.edu/shibboleth!(.+)$'
 
+- name: Replace account app port in Apache configuration
+  ansible.builtin.replace:
+    path: /etc/httpd/conf.d/front-end.conf
+    regexp: "account-app:8000"
+    replace: "account-app:{{ account_app_port }}"
+
 - name: Restart httpd services
   ansible.builtin.service:
     name: httpd

ansible/roles/slurm_client/tasks/main.yml

@@ -29,6 +29,19 @@
     group: root
     mode: 0400
 
+- name: Create symbolic links for Slurm config files
+  ansible.builtin.file:
+    src: "{{ item.src }}"
+    dest: "{{ item.dest }}"
+    state: link
+    force: yes  # Force the creation of the symlinks even if source files do not exist yet
+  loop:
+    - { src: "/cm/shared/apps/slurm/var/etc/cgroup.conf", dest: "/etc/slurm/cgroup.conf" }
+    - { src: "/cm/shared/apps/slurm/var/etc/gres.conf", dest: "/etc/slurm/gres.conf" }
+    - { src: "/cm/shared/apps/slurm/var/etc/slurm.conf", dest: "/etc/slurm/slurm.conf" }
+    - { src: "/cm/shared/apps/slurm/var/etc/slurmdbd.conf", dest: "/etc/slurm/slurmdbd.conf" }
+    - { src: "/cm/shared/apps/slurm/var/etc/job_submit.lua", dest: "/etc/slurm/job_submit.lua" }
+
 - name: Enable services
   ansible.builtin.service:
     name: "{{ item }}"

ansible/roles/ssh_proxy_config/tasks/main.yml

@@ -10,45 +10,3 @@
     name: sshpiperd
     enabled: true
     state: restarted
-
-- name: Install firewalld
-  ansible.builtin.package:
-    name: firewalld
-    state: present
-
-- name: Configure firewalld
-  ansible.posix.firewalld:
-    port: 2222/tcp
-    zone: public
-    state: enabled
-    permanent: true
-
-- name: Enable and start firewalld
-  ansible.builtin.service:
-    name: firewalld
-    enabled: true
-    state: restarted
-
-- name: Install fail2ban
-  ansible.builtin.package:
-    name: "{{ item }}"
-    state: present
-  loop:
-    - fail2ban
-    - fail2ban-firewalld
-
-- name: Configure fail2ban
-  ansible.builtin.template:
-    src: jail.local.j2
-    dest: "/etc/fail2ban/jail.local"
-    backup: true
-
-- name: Activate the firewall support
-  ansible.builtin.command:
-    cmd: mv /etc/fail2ban/jail.d/00-firewalld.conf /etc/fail2ban/jail.d/00-firewalld.local
-
-- name: Enable and start fail2ban
-  ansible.builtin.service:
-    name: fail2ban
-    enabled: true
-    state: restarted

openstack-login/nodeimage.pkr.hcl

@@ -61,5 +61,8 @@ build {
     groups           = ["compute"]
     ansible_env_vars = ["ANSIBLE_HOST_KEY_CHECKING=False"]
     playbook_file    = "./CRI_XCBC/compute-packer.yaml"
+    extra_arguments  = [
+      "--extra-vars", "${var.extra_vars}"
+    ]
   }
 }

openstack-login/variables.pkr.hcl

@@ -88,4 +88,10 @@ variable "volume_size" {
   type        = number
   default     = 20
   description = "The default volume size for building iamge"
-}
\ No newline at end of file
+}
+
+variable "extra_vars" {
+  type        = string
+  default     = ""
+  description = "Extra vars to pass to ansible playbook command"
+}

openstack-ood/nodeimage.pkr.hcl

@@ -53,6 +53,9 @@ build {
     groups           = ["ood", "knightly"]
     ansible_env_vars = ["ANSIBLE_HOST_KEY_CHECKING=False"]
     playbook_file    = "./CRI_XCBC/ood-packer.yaml"
+    extra_arguments  = [
+      "--extra-vars", "${var.extra_vars}"
+    ]
   }
 
   provisioner "shell" {

openstack-ood/variables.pkr.hcl

@@ -89,3 +89,9 @@ variable "volume_size" {
   default     = 20
   description = "The default volume size for building iamge"
 }
+
+variable "extra_vars" {
+  type        = string
+  default     = ""
+  description = "Extra vars to pass to ansible playbook command"
+}

openstack-proxy/nodeimage.pkr.hcl

@@ -58,5 +58,8 @@ build {
       "ANSIBLE_FORCE_COLOR=true"
     ]
     playbook_file    = "./CRI_XCBC/proxy.yaml"
+    extra_arguments  = [
+      "--extra-vars", "${var.extra_vars}"
+    ]
   }
 }

openstack-proxy/variables.pkr.hcl

@@ -106,3 +106,8 @@ variable "ANSIBLE_VERBOSITY" {
   description = "to increase verbosity - 0|1|2|3|4"
 }
 
+variable "extra_vars" {
+  type        = string
+  default     = ""
+  description = "Extra vars to pass to ansible playbook command"
+}