ansible/roles/ssh_proxy_config/tasks/main.yml

@@ -10,45 +10,3 @@
     name: sshpiperd
     enabled: true
     state: restarted
-
-- name: Install firewalld
-  ansible.builtin.package:
-    name: firewalld
-    state: present
-
-- name: Configure firewalld
-  ansible.posix.firewalld:
-    port: 2222/tcp
-    zone: public
-    state: enabled
-    permanent: true
-
-- name: Enable and start firewalld
-  ansible.builtin.service:
-    name: firewalld
-    enabled: true
-    state: restarted
-
-- name: Install fail2ban
-  ansible.builtin.package:
-    name: "{{ item }}"
-    state: present
-  loop:
-    - fail2ban
-    - fail2ban-firewalld
-
-- name: Configure fail2ban
-  ansible.builtin.template:
-    src: jail.local.j2
-    dest: "/etc/fail2ban/jail.local"
-    backup: true
-
-- name: Activate the firewall support
-  ansible.builtin.command:
-    cmd: mv /etc/fail2ban/jail.d/00-firewalld.conf /etc/fail2ban/jail.d/00-firewalld.local
-
-- name: Enable and start fail2ban
-  ansible.builtin.service:
-    name: fail2ban
-    enabled: true
-    state: restarted

ansible/roles/ssh_proxy_config/templates/sshpiperd.yaml.j2

@@ -24,5 +24,11 @@ pipes:
     host: "{{ group.host }}"
     ignore_hostkey: true
     private_key: "{{ group.private_key }}"
+- from:
+    - username: ".*"
+      username_regex_match: true
+  to:
+    host: "{{ group.host }}"
+    ignore_hostkey: true
 {% endif %}
 {% endfor %}

openstack-login/nodeimage.pkr.hcl

@@ -61,5 +61,8 @@ build {
     groups           = ["compute"]
     ansible_env_vars = ["ANSIBLE_HOST_KEY_CHECKING=False"]
     playbook_file    = "./CRI_XCBC/compute-packer.yaml"
+    extra_arguments  = [
+      "--extra-vars", "${var.extra_vars}"
+    ]
   }
 }

openstack-login/variables.pkr.hcl

@@ -88,4 +88,10 @@ variable "volume_size" {
   type        = number
   default     = 20
   description = "The default volume size for building iamge"
-}
\ No newline at end of file
+}
+
+variable "extra_vars" {
+  type        = string
+  default     = ""
+  description = "Extra vars to pass to ansible playbook command"
+}

openstack-ood/nodeimage.pkr.hcl

@@ -40,6 +40,9 @@ build {
     use_proxy     = false
     user          = var.ssh_username
     groups        = ["ood"]
+    ansible_env_vars = [
+      "ANSIBLE_CONFIG=./ansible/ansible.cfg"
+    ]
     playbook_file = "./ansible/ood.yml"
     roles_path    = "./ansible/roles"
     extra_arguments = [
@@ -51,7 +54,22 @@ build {
     use_proxy        = false
     user             = var.ssh_username
     groups           = ["ood", "knightly"]
-    ansible_env_vars = ["ANSIBLE_HOST_KEY_CHECKING=False"]
+    ansible_env_vars = [
+      "ANSIBLE_HOST_KEY_CHECKING=False",
+      "ANSIBLE_CONFIG=./CRI_XCBC/ansible.cfg"
+    ]
     playbook_file    = "./CRI_XCBC/ood-packer.yaml"
+    extra_arguments  = [
+      "--extra-vars", "${var.extra_vars}"
+    ]
   }
+
+  provisioner "shell" {
+    inline = [
+      "sudo yum install -y libselinux-python3 python3 python3-pip tmux vim git bash-completion curl wget unzip NetworkManager",
+      "sudo python3 -m pip install --upgrade pip",
+      "sudo pip3 install s3cmd==2.3.0 ansible==4.10.0 python-openstackclient==5.8.0"
+    ]
+  }
+
 }

openstack-ood/variables.pkr.hcl

 variable "root_ssh_key" {
   type        = string
+  default     = ""
   description = "The root key to use for ssh"
 }
 
@@ -87,4 +88,10 @@ variable "volume_size" {
   type        = number
   default     = 20
   description = "The default volume size for building iamge"
-}
\ No newline at end of file
+}
+
+variable "extra_vars" {
+  type        = string
+  default     = ""
+  description = "Extra vars to pass to ansible playbook command"
+}

openstack-proxy/nodeimage.pkr.hcl

@@ -58,5 +58,8 @@ build {
       "ANSIBLE_FORCE_COLOR=true"
     ]
     playbook_file    = "./CRI_XCBC/proxy.yaml"
+    extra_arguments  = [
+      "--extra-vars", "${var.extra_vars}"
+    ]
   }
 }

openstack-proxy/variables.pkr.hcl

@@ -106,3 +106,8 @@ variable "ANSIBLE_VERBOSITY" {
   description = "to increase verbosity - 0|1|2|3|4"
 }
 
+variable "extra_vars" {
+  type        = string
+  default     = ""
+  description = "Extra vars to pass to ansible playbook command"
+}

requirements.txt

+certifi==2025.1.31
+charset-normalizer==3.4.1
+idna==3.10
+python-gitlab==5.6.0
+requests==2.32.3
+requests-toolbelt==1.0.0
+urllib3==2.4.0
+PyYAML==6.0.2

utils/README.md

+### Description
+
+These utility scripts avoid copying each ci variable manually which is tedious.
+
+- The gitlab-ci-vars-reader.py reads variables from a specific project or a pipeline (depending on the options provided) and copies them into a yaml file
+- The gitlab-ci-vars-updater.py takes a yaml file containing key value pairs in yaml format as an input. It then creates/updates project variables or pipeline variables (depending on the options provided)
+
+### Prerequisites
+
+```
+python -m venv ~/venvs/gitlab
+source ~/venvs/gitlab/bin/activate
+pip install -r requirements
+```
+
+### Setup
+
+```
+cd utils
+mv gitlab.ini.example gitlab.ini
+```
+
+Make changes to the gitlab.ini as you require.
+[Create a personal access token](https://docs.gitlab.com/user/profile/personal_access_tokens/) via the gitlab UI and copy it to the private_token field in gitlab.ini file
+
+### Usage
+
+Create a new pipeline schedule if it does not exist in the destination project
+
+> Replace the content in <> with values appropriate for your project setup
+
+```
+gitlab -c gitlab.ini project-pipeline-schedule create --project-id <project-id> --description <pipeline-description> --ref <branch-name> --cron '*/15 * * * *' --active <true/false> --cron-timezone America/Chicago
+```
+
+You may want to know which pipelines exist, in the project you want to read/write the variables.
+
+To list schedule pipeline IDs along with descriptions
+
+```
+python3 list_schedule_pipelines.py --project_name <project-name>
+```
+
+> In order to copy the CI/CD Variables in to your project run the following command
+
+```
+python3 gitlab-ci-vars-reader.py --config_file gitlab.ini --project_name <PROJECT_NAME> --sched_pipeline_id <COPY_FROM_PIPELINE_ID> --var_file ci-variables.yaml
+
+python3 gitlab-ci-vars-updater.py --config_file gitlab.ini --project_name <PROJECT_NAME> --sched_pipeline_id <COPY_TO_PIPELINE_ID> --var_file ci-variables.yaml
+```
+
+Once you have copied over the CI/CD variables you will need to update a few of them to have them working in your project
+
+- OS_APPLICATION_CREDENTIAL_ID (build/all)
+- OS_APPLICATION_CREDENTIAL_SECRET (build/all)
+- PKR_VAR_networks

utils/gitlab-ci-vars-reader.py

+import argparse
+
+import gitlab
+import yaml
+
+
+# Function to fetch all CI/CD variables from a GitLab project
+def fetch_variables(project):
+    p_variables = list(project.variables.list(iterator=True))
+    variables = [var.asdict() for var in p_variables]
+
+    return variables
+
+
+def fetch_sched_variables(sched_pipeline):
+    variables = sched_pipeline.attributes["variables"]
+    return variables
+
+
+# Main function to load the config and fetch variables
+def main():
+    # Setup argument parser
+    parser = argparse.ArgumentParser(description="GitLab CI/CD Variable reader")
+    parser.add_argument(
+        "--config_file",
+        type=str,
+        default="gitlab.ini",
+        required=True,
+        help="Path to the configuration file (default: gitlab.ini)",
+    )
+    parser.add_argument(
+        "--var_file",
+        type=str,
+        default="ci-variables.yaml",
+        help="Path to the CI vars file (default: ci-variables.yaml)",
+    )
+    parser.add_argument(
+        "--project_name",
+        type=str,
+        required=True,
+        help="Gitlab project name with namespace",
+    )
+    parser.add_argument(
+        "--sched_pipeline_id",
+        type=int,
+        help="Gitlab project scheduled pipeline ID",
+    )
+
+    # Parse the arguments
+    args = parser.parse_args()
+
+    gl = gitlab.Gitlab.from_config("uabrc", [args.config_file])
+    project = gl.projects.get(args.project_name)
+
+    # Fetch project or sched pipeline variables
+    if not args.sched_pipeline_id:
+        variables = fetch_variables(project)
+    else:
+        sched_pipeline = project.pipelineschedules.get(args.sched_pipeline_id)
+        variables = fetch_sched_variables(sched_pipeline)
+
+    try:
+        with open(args.var_file, mode="wt", encoding="utf-8") as file:
+            yaml.dump(variables, file, explicit_start=True)
+    except FileNotFoundError:
+        print(f"Error: Writing File to '{args.var_file}'")
+        exit(1)
+
+
+# Run the main function
+if __name__ == "__main__":
+    main()

utils/gitlab-ci-vars-updater.py

+import argparse
+
+import gitlab
+import yaml
+
+
+def load_file(file_path):
+    try:
+        with open(file_path, mode="rt", encoding="utf-8") as file:
+            return yaml.safe_load(file)
+    except FileNotFoundError:
+        print(f"Error: Configuration file '{file_path}' not found.")
+        exit(1)
+
+
+# Function to create or update a GitLab CI/CD variable
+def create_or_update_variable(project, var_dict):
+    key = var_dict.get("key")
+    scope = var_dict.get("environment_scope", "*")
+    p_variable = None
+
+    DEFAULTS = {
+        "variable_type": "env_var",
+        "hidden": False,
+        "protected": False,
+        "masked": False,
+        "environment_scope": "*",
+        "raw": False,
+        "description": None,
+    }
+
+    # Merge defaults with var_dict
+    var_dict = {**DEFAULTS, **var_dict}
+
+    # Fetch a variable with matching key and scope
+    try:
+        all_vars = project.variables.list(get_all=True)
+        for var in all_vars:
+            if var.key == key and var.environment_scope == scope:
+                p_variable = var
+                break
+    except gitlab.exceptions.GitlabGetError:
+        print("Variable not found")
+        exit(1)
+
+    # Check if the variable exists and same as input
+    if p_variable is not None:
+        if p_variable.asdict() != var_dict:
+            # if not same update the project variable
+            print(f"Updating {p_variable.attributes['key']}")
+            p_variable.delete()
+            return project.variables.create(var_dict)
+        else:
+            print(f"variable {var_dict['key']} already exists")
+    # Create variable if it doesn't exist in the project
+    else:
+        print(f"Creating variable {var_dict['key']}")
+        return project.variables.create(var_dict)
+
+
+def get_pipeline_vars_by_key(sched_pipeline, key_name):
+    p_vars = sched_pipeline.attributes["variables"]
+    for p_variable in p_vars:
+        if p_variable.get("key") == key_name:
+            return p_variable
+
+
+# Function to create or update a schedule pipeline variable
+def create_or_update_sched_vars(sched_pipeline, var_dict):
+    # Check if the variable exists in the sched pipeline
+    p_variable = get_pipeline_vars_by_key(sched_pipeline, var_dict["key"])
+    if p_variable:
+        # Check if the attributes are the same
+        if p_variable != var_dict:
+            # If not update the value in the project
+            sched_pipeline.variables.delete(p_variable["key"])
+            sched_pipeline.variables.create(var_dict)
+        else:
+            print(f"variable {var_dict['key']} already exists")
+    # Create variable if it doesn't exist in the project
+    else:
+        print(f"Creating variable {var_dict['key']}")
+        return sched_pipeline.variables.create(var_dict)
+
+
+def main():
+    # Setup argument parser
+    parser = argparse.ArgumentParser(description="GitLab CI/CD Variables Updater")
+    parser.add_argument(
+        "--config_file",
+        type=str,
+        default="gitlab.ini",
+        required=True,
+        help="Path to the configuration file (default: gitlab.ini)",
+    )
+    parser.add_argument(
+        "--var_file",
+        type=str,
+        default="ci-variables.yaml",
+        help="Path to the CI vars file (default: ci-variables.yaml)",
+    )
+    parser.add_argument(
+        "--project_name",
+        type=str,
+        required=True,
+        help="Gitlab project name with namespace",
+    )
+    parser.add_argument(
+        "--sched_pipeline_id",
+        type=int,
+        help="Gitlab project scheduled pipeline ID",
+    )
+
+    # Parse the arguments
+    args = parser.parse_args()
+
+    gl = gitlab.Gitlab.from_config("uabrc", [args.config_file])
+    project = gl.projects.get(args.project_name)
+
+    # Load the CI vars file
+    var_list = load_file(args.var_file)
+
+    # Create or update all variables
+    for var_dict in var_list:
+        if not args.sched_pipeline_id:
+            create_or_update_variable(project, var_dict)
+        else:
+            sched_pipeline = project.pipelineschedules.get(args.sched_pipeline_id)
+            create_or_update_sched_vars(sched_pipeline, var_dict)
+
+
+if __name__ == "__main__":
+    main()

utils/gitlab.ini.example

+[global]
+default = uabrc
+ssl_verify = true
+timeout = 5
+per_page = 100
+
+[uabrc]
+url = https://gitlab.rc.uab.edu
+private_token =
+api_version = 4

utils/list-sched-pipelines.py

+import argparse
+
+import gitlab
+
+
+# Main function to load the config and list sched pipelines
+def main():
+    # Setup argument parser
+    parser = argparse.ArgumentParser(description="List GitLab sched pipeline details")
+    parser.add_argument(
+        "--config_file",
+        type=str,
+        default="gitlab.ini",
+        required=True,
+        help="Path to the configuration file (default: gitlab.ini)",
+    )
+    parser.add_argument(
+        "--project_name",
+        type=str,
+        required=True,
+        help="Gitlab project name with namespace",
+    )
+
+    # Parse the arguments
+    args = parser.parse_args()
+
+    gl = gitlab.Gitlab.from_config("uabrc", [args.config_file])
+    project = gl.projects.get(args.project_name)
+
+    print(" ID \t DESCRIPTION")
+    print("----\t-------------")
+
+    # Fetch sched pipeline details
+    for sched in project.pipelineschedules.list(all=True):
+        print(f"{sched.id}\t{sched.description}")
+
+
+# Run the main function
+if __name__ == "__main__":
+    main()