diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9a6a6938e6a5bac910fa0d6407e09f04a2ff6126..287365758ceaa7b1502ab4fc63cccf6c95cd9e5a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -246,10 +246,12 @@ deploy_http_proxy_node: export cmd="openstack server create" cmd+=" -c id -f value --image $HTTP_PROXY_IMAGE_ID" cmd+=" --flavor $INSTANCE_FLAVOR" - cmd+=" --network $PROXY_NETWORK" - cmd+=" --security-group webserver_sec_group" - cmd+=" --security-group allow-ssh" + for security_group in ${SECURITY_GROUP_LIST[@]}; + do + cmd+=" --security-group $security_group" + done cmd+=" --user-data user_data.txt" + if [ -n "$PROXY_NETWORK" ];then cmd+=" --network $PROXY_NETWORK"; fi if [ -n "$HTTP_PROXY_PORT" ];then cmd+=" --port $HTTP_PROXY_PORT"; fi cmd+=" --wait $HTTP_PROXY_INSTANCE_NAME" - export HTTP_PROXY_INSTANCE_ID=$(bash -c "$cmd") @@ -297,9 +299,12 @@ deploy_ssh_proxy_node: export cmd="openstack server create" cmd+=" -c id -f value --image $SSH_PROXY_IMAGE_ID" cmd+=" --flavor $INSTANCE_FLAVOR" - cmd+=" --network $PROXY_NETWORK" - cmd+=" --security-group allow-ssh" + for security_group in ${SECURITY_GROUP_LIST[@]}; + do + cmd+=" --security-group $security_group" + done cmd+=" --user-data user_data.txt" + if [ -n "$PROXY_NETWORK" ];then cmd+=" --network $PROXY_NETWORK"; fi if [ -n "$SSH_PROXY_PORT" ];then cmd+=" --port $SSH_PROXY_PORT"; fi cmd+=" --wait $SSH_PROXY_INSTANCE_NAME" - export SSH_PROXY_INSTANCE_ID=$(bash -c "$cmd") @@ -348,9 +353,12 @@ deploy_login_node: export cmd="openstack server create" cmd+=" -c id -f value --image $LOGIN_IMAGE_ID" cmd+=" --flavor $INSTANCE_FLAVOR" - cmd+=" --network $INSTANCE_NETWORK" - cmd+=" --security-group allow-ssh" + for security_group in ${SECURITY_GROUP_LIST[@]}; + do + cmd+=" --security-group $security_group" + done cmd+=" --user-data user_data.txt" + if [ -n "$INSTANCE_NETWORK" ];then cmd+=" --network $INSTANCE_NETWORK"; fi if [ -n "$LOGIN_PORT" ];then cmd+=" --port $LOGIN_PORT"; fi cmd+=" --wait $LOGIN_INSTANCE_NAME" - export LOGIN_INSTANCE_ID=$(bash -c "$cmd") diff --git a/ansible/cluster.yml b/ansible/cluster.yml index 1a2c83e24576ac7a3e8305d6d0e906442d4c606a..9664bf9e22996377bec93b4d4eb4155830a1d0d7 100644 --- a/ansible/cluster.yml +++ b/ansible/cluster.yml @@ -12,3 +12,5 @@ - { name: 'ssl_cert', tags: 'ssl_cert', when: enable_ssl_certs } - { name: 'rsyslog_config', tags: 'rsyslog_config', when: enable_rsyslog_config } - { name: 'rewrite_map', tags: 'rewrite_map', when: enable_rewrite_map } + - { name: 'fail2ban', tags: 'fail2ban', when: enable_fail2ban } + - { name: 'install_node_exporter', tags: 'install_node_exporter', when: enable_node_exporter } diff --git a/ansible/group_vars/all b/ansible/group_vars/all index eaef961815f6ce7dfb7bdd0957acca2a77f512b2..46267cd8eb7d645ece328c8f74f82fb3d536fa77 100644 --- a/ansible/group_vars/all +++ b/ansible/group_vars/all @@ -50,10 +50,9 @@ # ssh proxy enable_ssh_proxy_config: false sshpiper_dest_dir: "/opt/sshpiper" - fail2ban_cidr_list: "127.0.0.1/8" # rsyslog - enable_rsyslog_config: false + enable_rsyslog_config: true rsyslog_target: "*.* @master:514" # ssl certs @@ -73,3 +72,20 @@ - {"name": "gpfs4", "host": "login001", "default": True } - {"name": "gpfs5", "host": "login002", "default": False } +# account app + account_app_port: 8000 + +# fail2ban + enable_fail2ban: false + maxretry: 1 + findtime: 600 + bantime: 1200 + fail2ban_white_list: "127.0.0.1/8" + +# Node Exporter + enable_node_exporter: false + node_exporter_ver: "1.8.2" + node_exporter_filename: "node_exporter-{{ node_exporter_ver }}.linux-amd64" + node_exporter_user: node_exporter + node_exporter_group: node_exporter + node_exporter_port: 9100 diff --git a/ansible/roles/fail2ban/tasks/main.yml b/ansible/roles/fail2ban/tasks/main.yml new file mode 100644 index 0000000000000000000000000000000000000000..2f7d96e68acb65478a263291c9f3e3092612ff94 --- /dev/null +++ b/ansible/roles/fail2ban/tasks/main.yml @@ -0,0 +1,46 @@ +--- + +- name: Install fail2ban + ansible.builtin.package: + name: "{{ item }}" + state: present + loop: + - fail2ban + - fail2ban-firewalld + +- name: Configure fail2ban + ansible.builtin.template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + backup: true + loop: + - { src: 'jail.local.j2', dest: '/etc/fail2ban/jail.local' } + - { src: 'sshpiperd_filter.local.j2', dest: '/etc/fail2ban/filter.d/sshpiperd.local' } + - { src: 'sshpiperd_jail.local.j2', dest: '/etc/fail2ban/jail.d/sshpiperd.local' } + +- name: Activate the firewalld support for fail2ban + ansible.builtin.command: + cmd: mv /etc/fail2ban/jail.d/00-firewalld.conf /etc/fail2ban/jail.d/00-firewalld.local + +- name: Configure firewalld to allow ssh and sshpiper traffic + ansible.posix.firewalld: + port: "{{ item }}" + zone: public + state: enabled + permanent: true + loop: + - 2222/tcp + - 22/tcp + +- name: Enable and start firewalld + ansible.builtin.service: + name: firewalld + enabled: true + state: restarted + +- name: Enable and start fail2ban + ansible.builtin.service: + name: fail2ban + enabled: true + state: restarted + diff --git a/ansible/roles/fail2ban/templates/jail.local.j2 b/ansible/roles/fail2ban/templates/jail.local.j2 new file mode 100644 index 0000000000000000000000000000000000000000..87f9e4fa06f0ca75ed77d0a2361262f94859d91d --- /dev/null +++ b/ansible/roles/fail2ban/templates/jail.local.j2 @@ -0,0 +1,7 @@ +[DEFAULT] +banaction = firewalld +bantime = {{ bantime }} +ignoreip = {{ fail2ban_white_list }} + +[sshd] +enabled = true diff --git a/ansible/roles/fail2ban/templates/sshpiperd_filter.local.j2 b/ansible/roles/fail2ban/templates/sshpiperd_filter.local.j2 new file mode 100644 index 0000000000000000000000000000000000000000..f5a6081ec28c490ab24ff3f4c96c3b08bb86b9da --- /dev/null +++ b/ansible/roles/fail2ban/templates/sshpiperd_filter.local.j2 @@ -0,0 +1,22 @@ +# Refer to https://github.com/fail2ban/fail2ban/wiki/Developing-Regex-in-Fail2ban for developing regex using fail2ban +# +[INCLUDES] +before = common.conf + +[DEFAULT] +_daemon = sshpiperd +__iso_datetime = "\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:[+-]\d{2}:\d{2}|Z)" +__pref = time=%(__iso_datetime)s level=(?:debug|error) + +[Definition] +# Define the prefix regex for the log lines +prefregex = ^<F-MLFID>%(__prefix_line)s%(__pref)s</F-MLFID>\s+<F-CONTENT>.+</F-CONTENT>$ + +# Failregex to match the specific failure log lines (prefregex is automatically included) +failregex = ^msg="connection from .*failtoban: ip <HOST> too auth many failures"$ + +ignoreregex = + +mode = normal + +maxlines = 1 diff --git a/ansible/roles/fail2ban/templates/sshpiperd_jail.local.j2 b/ansible/roles/fail2ban/templates/sshpiperd_jail.local.j2 new file mode 100644 index 0000000000000000000000000000000000000000..681212ccaf59595c5dd6097e188286db85225dbd --- /dev/null +++ b/ansible/roles/fail2ban/templates/sshpiperd_jail.local.j2 @@ -0,0 +1,9 @@ +# This configuration will block the remote host after {{maxretry}} failed SSH login attempts. +[sshpiperd] +enabled = true +filter = sshpiperd +logpath = /var/log/messages +port = 22 +maxretry = {{ maxretry }} +backend = auto +findtime = {{ findtime }} diff --git a/ansible/roles/install_node_exporter/tasks/main.yaml b/ansible/roles/install_node_exporter/tasks/main.yaml new file mode 100644 index 0000000000000000000000000000000000000000..205904b956f913c8f9e7dd8a11d45cde1d1037b0 --- /dev/null +++ b/ansible/roles/install_node_exporter/tasks/main.yaml @@ -0,0 +1,82 @@ +--- +- name: Download node_exporter binary + ansible.builtin.get_url: + url: "https://github.com/prometheus/node_exporter/releases/download/v{{ node_exporter_ver }}/{{ node_exporter_filename }}.tar.gz" + dest: "/tmp/{{ node_exporter_filename }}.tar.gz" + +- name: Extract node_exporter + ansible.builtin.unarchive: + src: "/tmp/{{ node_exporter_filename }}.tar.gz" + dest: "/tmp" + remote_src: yes + +- name: Create system group for user account {{ node_exporter_group }} + ansible.builtin.group: + name: "{{ node_exporter_group }}" + system: true + state: present + +- name: Create system user account {{ node_exporter_user }} + ansible.builtin.user: + name: "{{ node_exporter_user }}" + comment: Prometheus node_exporter system account + group: "{{ node_exporter_group }}" + system: true + home: /var/lib/node_exporter + create_home: false + shell: /sbin/nologin + state: present + +- name: Copy node_exporter binary + ansible.builtin.copy: + src: "/tmp/{{ node_exporter_filename }}/node_exporter" + dest: /usr/local/bin/node_exporter + remote_src: yes + owner: root + group: root + mode: 0755 + +- name: Copy systemd unit file + ansible.builtin.template: + src: node_exporter.service.j2 + dest: /etc/systemd/system/node_exporter.service + owner: root + group: root + mode: '0644' + +- name: Clean up /tmp + ansible.builtin.file: + path: "/tmp/{{ item }}" + state: absent + loop: + - "{{ node_exporter_filename }}.tar.gz" + - "{{ node_exporter_filename }}" + +- name: Restart node_exporter service + ansible.builtin.systemd: + daemon_reload: yes + name: node_exporter + state: restarted + enabled: true + +- name: Collect facts about system services + ansible.builtin.service_facts: + +- name: Configure firewalld to allow prometheus + ansible.posix.firewalld: + port: "{{ node_exporter_port }}/tcp" + zone: public + state: enabled + permanent: true + when: + - "'firewalld.service' in ansible_facts.services" + - ansible_facts.services["firewalld.service"].state == "running" + +- name: Enable and start firewalld + ansible.builtin.service: + name: firewalld + enabled: true + state: restarted + when: + - "'firewalld.service' in ansible_facts.services" + - ansible_facts.services["firewalld.service"].state == "running" diff --git a/ansible/roles/install_node_exporter/templates/node_exporter.service.j2 b/ansible/roles/install_node_exporter/templates/node_exporter.service.j2 new file mode 100644 index 0000000000000000000000000000000000000000..fddb82d9e0f74fe4382ac62e94fbb3a25ddb9dda --- /dev/null +++ b/ansible/roles/install_node_exporter/templates/node_exporter.service.j2 @@ -0,0 +1,12 @@ +[Unit] +Description=Node Exporter +After=network.target + +[Service] +User={{ node_exporter_user }} +Group={{ node_exporter_group }} +Type=simple +ExecStart=/usr/local/bin/node_exporter --web.listen-address=:{{ node_exporter_port }} --collector.filesystem.mount-points-exclude "^/(dev|proc|run/user/.+|run/credentials/.+|sys|var/lib/docker/.+)($|/)" --collector.filesystem.fs-types-exclude "^(autofs|binfmt_misc|bpf|cgroup|tmpfs|sunrpc|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$" + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/rewrite_map/tasks/main.yaml b/ansible/roles/rewrite_map/tasks/main.yaml index d9905652064c1fe824ab69a23b6620f36e9c1004..8b08eb62de1e987816f386bb50bdd2d8aec221e5 100644 --- a/ansible/roles/rewrite_map/tasks/main.yaml +++ b/ansible/roles/rewrite_map/tasks/main.yaml @@ -15,6 +15,12 @@ RewriteCond %{HTTP:REMOTE_USER} '([a-zA-Z0-9_.+-]+)@uab.edu$' [OR] RewriteCond %{HTTP:REMOTE_USER} 'urn:mace:incommon:uab.edu!https://uabgrid.uab.edu/shibboleth!(.+)$' +- name: Replace account app port in Apache configuration + ansible.builtin.replace: + path: /etc/httpd/conf.d/front-end.conf + regexp: "account-app:8000" + replace: "account-app:{{ account_app_port }}" + - name: Restart httpd services ansible.builtin.service: name: httpd diff --git a/ansible/roles/slurm_client/tasks/main.yml b/ansible/roles/slurm_client/tasks/main.yml index e2c1d78bf064d3f3cedbad74774efd7edc7b7e64..64612ed5a5b21aee18277e03a2be8d1b41429798 100644 --- a/ansible/roles/slurm_client/tasks/main.yml +++ b/ansible/roles/slurm_client/tasks/main.yml @@ -29,6 +29,19 @@ group: root mode: 0400 +- name: Create symbolic links for Slurm config files + ansible.builtin.file: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + state: link + force: yes # Force the creation of the symlinks even if source files do not exist yet + loop: + - { src: "/cm/shared/apps/slurm/var/etc/cgroup.conf", dest: "/etc/slurm/cgroup.conf" } + - { src: "/cm/shared/apps/slurm/var/etc/gres.conf", dest: "/etc/slurm/gres.conf" } + - { src: "/cm/shared/apps/slurm/var/etc/slurm.conf", dest: "/etc/slurm/slurm.conf" } + - { src: "/cm/shared/apps/slurm/var/etc/slurmdbd.conf", dest: "/etc/slurm/slurmdbd.conf" } + - { src: "/cm/shared/apps/slurm/var/etc/job_submit.lua", dest: "/etc/slurm/job_submit.lua" } + - name: Enable services ansible.builtin.service: name: "{{ item }}" diff --git a/ansible/roles/ssh_proxy_config/tasks/main.yml b/ansible/roles/ssh_proxy_config/tasks/main.yml index fb51f9fe65cedd2993af4199eff9e04f8c1c1b2a..30bac2abbe90860eabba3b051a4c212fa4f8c6b5 100644 --- a/ansible/roles/ssh_proxy_config/tasks/main.yml +++ b/ansible/roles/ssh_proxy_config/tasks/main.yml @@ -10,45 +10,3 @@ name: sshpiperd enabled: true state: restarted - -- name: Install firewalld - ansible.builtin.package: - name: firewalld - state: present - -- name: Configure firewalld - ansible.posix.firewalld: - port: 2222/tcp - zone: public - state: enabled - permanent: true - -- name: Enable and start firewalld - ansible.builtin.service: - name: firewalld - enabled: true - state: restarted - -- name: Install fail2ban - ansible.builtin.package: - name: "{{ item }}" - state: present - loop: - - fail2ban - - fail2ban-firewalld - -- name: Configure fail2ban - ansible.builtin.template: - src: jail.local.j2 - dest: "/etc/fail2ban/jail.local" - backup: true - -- name: Activate the firewall support - ansible.builtin.command: - cmd: mv /etc/fail2ban/jail.d/00-firewalld.conf /etc/fail2ban/jail.d/00-firewalld.local - -- name: Enable and start fail2ban - ansible.builtin.service: - name: fail2ban - enabled: true - state: restarted diff --git a/ansible/roles/ssh_proxy_config/templates/jail.local.j2 b/ansible/roles/ssh_proxy_config/templates/jail.local.j2 deleted file mode 100644 index d5898e63b7cbb1046ac28d59062b1ede7d148809..0000000000000000000000000000000000000000 --- a/ansible/roles/ssh_proxy_config/templates/jail.local.j2 +++ /dev/null @@ -1,7 +0,0 @@ -[DEFAULT] -banaction = firewalld -bantime = 1200 -ignoreip = {{ fail2ban_cidr_list }} - -[sshd] -enabled = true diff --git a/openstack-login/nodeimage.pkr.hcl b/openstack-login/nodeimage.pkr.hcl index fd6b35ee3c2be27e5fd4f353f0a1a47e27d06103..e770d3e8a817ada4c17455ee218f2f432827b9ad 100644 --- a/openstack-login/nodeimage.pkr.hcl +++ b/openstack-login/nodeimage.pkr.hcl @@ -61,5 +61,8 @@ build { groups = ["compute"] ansible_env_vars = ["ANSIBLE_HOST_KEY_CHECKING=False"] playbook_file = "./CRI_XCBC/compute-packer.yaml" + extra_arguments = [ + "--extra-vars", "${var.extra_vars}" + ] } } diff --git a/openstack-login/variables.pkr.hcl b/openstack-login/variables.pkr.hcl index d3681394fe7c66990a5b9f1507f38f7dbc74d025..6cef95c13157b85446affdea4deff552d6c1d71a 100644 --- a/openstack-login/variables.pkr.hcl +++ b/openstack-login/variables.pkr.hcl @@ -88,4 +88,10 @@ variable "volume_size" { type = number default = 20 description = "The default volume size for building iamge" -} \ No newline at end of file +} + +variable "extra_vars" { + type = string + default = "" + description = "Extra vars to pass to ansible playbook command" +} diff --git a/openstack-ood/nodeimage.pkr.hcl b/openstack-ood/nodeimage.pkr.hcl index 2b516becf9ce17df6ce78cc2596d7470dbb6e80c..1a1374419cd8ed985a4cde57378154a5cf07ff3f 100644 --- a/openstack-ood/nodeimage.pkr.hcl +++ b/openstack-ood/nodeimage.pkr.hcl @@ -53,6 +53,9 @@ build { groups = ["ood", "knightly"] ansible_env_vars = ["ANSIBLE_HOST_KEY_CHECKING=False"] playbook_file = "./CRI_XCBC/ood-packer.yaml" + extra_arguments = [ + "--extra-vars", "${var.extra_vars}" + ] } provisioner "shell" { diff --git a/openstack-ood/variables.pkr.hcl b/openstack-ood/variables.pkr.hcl index a97e3277490b758dfc23ea461e702d6f5c690fa9..3e20d6dc9e6b35ccac6b7bdb03f5ec28e8d24cce 100644 --- a/openstack-ood/variables.pkr.hcl +++ b/openstack-ood/variables.pkr.hcl @@ -89,3 +89,9 @@ variable "volume_size" { default = 20 description = "The default volume size for building iamge" } + +variable "extra_vars" { + type = string + default = "" + description = "Extra vars to pass to ansible playbook command" +} diff --git a/openstack-proxy/nodeimage.pkr.hcl b/openstack-proxy/nodeimage.pkr.hcl index 1410cbe94026d962c9ec9e0069d2b97dba619cd4..b9480f29cc550654ff3017773c5e2acb60b1584a 100644 --- a/openstack-proxy/nodeimage.pkr.hcl +++ b/openstack-proxy/nodeimage.pkr.hcl @@ -58,5 +58,8 @@ build { "ANSIBLE_FORCE_COLOR=true" ] playbook_file = "./CRI_XCBC/proxy.yaml" + extra_arguments = [ + "--extra-vars", "${var.extra_vars}" + ] } } diff --git a/openstack-proxy/variables.pkr.hcl b/openstack-proxy/variables.pkr.hcl index 9215362da397119c2833a43ee976c0717515a931..6ab03ba638197c98ba7e93bb891596a74d9938c3 100644 --- a/openstack-proxy/variables.pkr.hcl +++ b/openstack-proxy/variables.pkr.hcl @@ -106,3 +106,8 @@ variable "ANSIBLE_VERBOSITY" { description = "to increase verbosity - 0|1|2|3|4" } +variable "extra_vars" { + type = string + default = "" + description = "Extra vars to pass to ansible playbook command" +}