.gitlab-ci.yml

@@ -85,7 +85,7 @@ workflow:
       if [ $CI_PIPELINE_SOURCE == 'merge_request_event' ]; then
         export PKR_VAR_image_name="${BUILD_TARGET}-PR-${CI_MERGE_REQUEST_IID}"
       elif [ $CI_PIPELINE_SOURCE == 'schedule' ]; then
-        export PKR_VAR_image_name="${BUILD_TARGET}-${BUILD_DATE}"
+        export PKR_VAR_image_name="${BUILD_TARGET}-${BUILD_TAG:-${BUILD_DATE}}"
       fi
     # Ansible var overrides
     - |
@@ -113,7 +113,7 @@ workflow:
 build_http_proxy_image:
   stage: build
   environment:
-    name: $ENV
+    name: build
   tags:
     - build
   variables:
@@ -126,7 +126,7 @@ build_http_proxy_image:
 build_ssh_proxy_image:
   stage: build
   environment:
-    name: $ENV
+    name: build
   tags:
     - build
   variables:
@@ -148,7 +148,7 @@ build_ssh_proxy_image:
       if [ $CI_PIPELINE_SOURCE == 'merge_request_event' ]; then
         export PKR_VAR_image_name="${BUILD_TARGET}-PR-${CI_MERGE_REQUEST_IID}"
       elif [ $CI_PIPELINE_SOURCE == 'schedule' ]; then
-        export PKR_VAR_image_name="${BUILD_TARGET}-${BUILD_DATE}"
+        export PKR_VAR_image_name="${BUILD_TARGET}-${BUILD_TAG:-${BUILD_DATE}}"
       fi
     # packer commands
     - packer init openstack-login
@@ -165,7 +165,7 @@ build_ssh_proxy_image:
 build_login_image:
   stage: build
   environment:
-    name: $ENV
+    name: build
   tags:
     - build
   <<: *build_login_image_template
@@ -173,6 +173,46 @@ build_login_image:
     - if: $PIPELINE_TARGET == "build" && $BUILD_TARGET == "login"
       when: always
 
+build_ood_image:
+  stage: build
+  environment:
+    name: build
+  tags:
+    - build
+  script:
+    - *update_ansible_repo
+    - *get_ansible_files
+    # packer vars for job env
+    - export PKR_VAR_flavor="${OOD_BUILD_FLAVOR:-$PKR_VAR_flavor}"
+    - export PKR_VAR_build_instance_name="${BUILD_TARGET}-${EXT_REPO_HEAD}"
+    - export PKR_VAR_image_date_suffix=false
+    - export PKR_VAR_image_name="${BUILD_TARGET}-${BUILD_TAG:-${BUILD_DATE}}"
+    - |
+      if [ $ENV = 'knightly' ] || [ $ENV = 'prod' ]; then
+        curl --header "PRIVATE-TOKEN: ${ANSIBLE_VAR_TOKEN}" \
+        "${CI_API_V4_URL}/projects/2836/repository/files/$ENV/raw?ref=main" \
+        -o CRI_XCBC/group_vars/$ENV
+        sed -i -E "s/(lts_access_key: ).*/\1\"${AWS_ACCESS_KEY_ID}\"/" CRI_XCBC/group_vars/$ENV
+        sed -i -E "s/(lts_secret_key: ).*/\1\"${AWS_SECRET_ACCESS_KEY}\"/" CRI_XCBC/group_vars/$ENV
+        sed -i -E "s/(user_register_app_key: ).*/\1\"${SELF_REG_APP_KEY}\"/" CRI_XCBC/group_vars/$ENV
+        sed -i -E "s/(celery_user_password: ).*/\1\"${CELERY_PASSWD}\"/" CRI_XCBC/group_vars/$ENV
+        sed -i -E "s|(ssh_pub_key: ).*|\1\"{{ lookup('file', '${SSH_PUB_KEY}') }}\"|" CRI_XCBC/group_vars/$ENV
+      fi
+    # packer commands
+    - packer init openstack-ood
+    - packer validate openstack-ood
+    - packer build -machine-readable openstack-ood | tee ood_build.log
+    - export BUILT_OOD_IMAGE_ID=$(grep 'Image:' ood_build.log | awk '{print $4}')
+    - echo BUILT_OOD_IMAGE_ID=${BUILT_OOD_IMAGE_ID} | tee -a $CI_PROJECT_DIR/image.env
+    # set image properties with repo state
+    - openstack image set --property EXT_PR_SRC_REPO=${EXT_PR_SRC_REPO} --property EXT_PR_SRC_BRANCH_SHA=${EXT_PR_SRC_BRANCH_SHA} --property EXT_PR_TARGET_REPO=${EXT_PR_TARGET_REPO} --property EXT_PR_TARGET_BRANCH_SHA=${EXT_PR_TARGET_BRANCH_SHA} --property PACKER_IMAGE_HEAD=${CI_COMMIT_SHORT_SHA} ${BUILT_OOD_IMAGE_ID}
+  artifacts:
+    reports:
+      dotenv: image.env
+  rules:
+    - if: $PIPELINE_TARGET == "build" && $BUILD_TARGET == "ood"
+      when: always
+
 deploy_http_proxy_node:
   stage: deploy
   environment:
@@ -206,12 +246,12 @@ deploy_http_proxy_node:
       export cmd="openstack server create"
       cmd+=" -c id -f value --image $HTTP_PROXY_IMAGE_ID"
       cmd+=" --flavor $INSTANCE_FLAVOR"
-      cmd+=" --network $PROXY_NETWORK"
       for security_group in ${SECURITY_GROUP_LIST[@]};
       do
         cmd+=" --security-group $security_group"
       done
       cmd+=" --user-data user_data.txt"
+      if [ -n "$PROXY_NETWORK" ];then cmd+=" --network $PROXY_NETWORK"; fi
       if [ -n "$HTTP_PROXY_PORT" ];then cmd+=" --port $HTTP_PROXY_PORT"; fi
       cmd+=" --wait $HTTP_PROXY_INSTANCE_NAME"
     - export HTTP_PROXY_INSTANCE_ID=$(bash -c "$cmd")
@@ -259,12 +299,12 @@ deploy_ssh_proxy_node:
       export cmd="openstack server create"
       cmd+=" -c id -f value --image $SSH_PROXY_IMAGE_ID"
       cmd+=" --flavor $INSTANCE_FLAVOR"
-      cmd+=" --network $PROXY_NETWORK"
       for security_group in ${SECURITY_GROUP_LIST[@]};
       do
         cmd+=" --security-group $security_group"
       done
       cmd+=" --user-data user_data.txt"
+      if [ -n "$PROXY_NETWORK" ];then cmd+=" --network $PROXY_NETWORK"; fi
       if [ -n "$SSH_PROXY_PORT" ];then cmd+=" --port $SSH_PROXY_PORT"; fi
       cmd+=" --wait $SSH_PROXY_INSTANCE_NAME"
     - export SSH_PROXY_INSTANCE_ID=$(bash -c "$cmd")
@@ -313,12 +353,12 @@ deploy_login_node:
       export cmd="openstack server create"
       cmd+=" -c id -f value --image $LOGIN_IMAGE_ID"
       cmd+=" --flavor $INSTANCE_FLAVOR"
-      cmd+=" --network $INSTANCE_NETWORK"
       for security_group in ${SECURITY_GROUP_LIST[@]};
       do
         cmd+=" --security-group $security_group"
       done
       cmd+=" --user-data user_data.txt"
+      if [ -n "$INSTANCE_NETWORK" ];then cmd+=" --network $INSTANCE_NETWORK"; fi
       if [ -n "$LOGIN_PORT" ];then cmd+=" --port $LOGIN_PORT"; fi
       cmd+=" --wait $LOGIN_INSTANCE_NAME"
     - export LOGIN_INSTANCE_ID=$(bash -c "$cmd")
@@ -333,3 +373,56 @@ deploy_login_node:
     - if: $PIPELINE_TARGET == "deploy" && $LOGIN_IMAGE_ID
       when: always
 
+deploy_ood_node:
+  stage: deploy
+  environment:
+    name: $ENV
+  tags:
+    - build
+  script:
+    - openstack image set --accept $OOD_IMAGE_ID || true
+    - FAILED=false
+    - |
+      cat > user_data.txt <<EOF
+      #!/bin/bash
+      cat >> /etc/NetworkManager/conf.d/90-dns-none.conf<<EEOF
+      [main]
+      dns=none
+      EEOF
+      systemctl reload NetworkManager
+      echo "$DEV_KEY" >> /root/.ssh/authorized_keys
+      ip route replace default via ${DEFAULT_GATEWAY_IP} dev eth0
+      git clone ${CI_REPOSITORY_URL} /tmp/${CI_PROJECT_NAME}
+      cd /tmp/${CI_PROJECT_NAME}
+      git checkout ${CI_COMMIT_REF_NAME}
+      cat >> ansible/hosts<<EEOF
+      [$ENV]
+      127.0.0.1
+      EEOF
+      s3cmd get --force -r --access_key=$AWS_ACCESS_KEY_ID --secret_key=$AWS_SECRET_ACCESS_KEY --host=$AWS_HOST --host-bucket=$AWS_HOST s3://cheaha-cloud-ansible-files/ /tmp/${CI_PROJECT_NAME}/ansible/files/
+      ansible-playbook -c local -i ansible/hosts --extra-vars="$EXTRA_VARS" ansible/cluster.yml | tee -a /tmp/ansible.log
+      rm -rf /tmp/${CI_PROJECT_NAME}
+      EOF
+    - |
+      export cmd="openstack server create"
+      cmd+=" -c id -f value --image $OOD_IMAGE_ID"
+      cmd+=" --flavor $INSTANCE_FLAVOR"
+      for security_group in ${SECURITY_GROUP_LIST[@]};
+      do
+        cmd+=" --security-group $security_group"
+      done
+      cmd+=" --user-data user_data.txt"
+      if [ -n "$INSTANCE_NETWORK" ];then cmd+=" --network $INSTANCE_NETWORK"; fi
+      if [ -n "$OOD_PORT" ];then cmd+=" --port $OOD_PORT"; fi
+      cmd+=" --wait $OOD_INSTANCE_NAME"
+    - export OOD_INSTANCE_ID=$(bash -c "$cmd")
+    - |
+      # Associate the floating IP(s) with the SSH Proxy instance
+      for OOD_FLOATING_IP in ${OOD_FLOATING_IP_LIST[@]};
+      do
+        echo "Associating FLOATING_IP $OOD_FLOATING_IP with OOD_INSTANCE_ID $OOD_INSTANCE_ID"
+        openstack server add floating ip $OOD_INSTANCE_ID $OOD_FLOATING_IP
+      done
+  rules:
+    - if: $PIPELINE_TARGET == "deploy" && $OOD_IMAGE_ID
+      when: always

ansible/group_vars/all

@@ -52,7 +52,7 @@
   sshpiper_dest_dir: "/opt/sshpiper"
 
 # rsyslog
-  enable_rsyslog_config: false
+  enable_rsyslog_config: true
   rsyslog_target: "*.* @master:514"
 
 # ssl certs

ansible/ood.yml

@@ -6,6 +6,3 @@
     - { name: 'fix_centos_repo', tags: 'fix_centos_repo' }
     - { name: 'install_packages', tags: 'install_packages' }
     - { name: 'install_zsh', tags: 'install_zsh' }
-
-- name: Setup node for use as a virtual cheaha node
-  ansible.builtin.import_playbook: cheaha.yml

ansible/roles/install_node_exporter/tasks/main.yaml

@@ -71,3 +71,12 @@
   when:
     - "'firewalld.service' in ansible_facts.services"
     - ansible_facts.services["firewalld.service"].state == "running"
+
+- name: Enable and start firewalld
+  ansible.builtin.service:
+    name: firewalld
+    enabled: true
+    state: restarted
+  when:
+    - "'firewalld.service' in ansible_facts.services"
+    - ansible_facts.services["firewalld.service"].state == "running"

openstack-ood/nodeimage.pkr.hcl

@@ -57,4 +57,13 @@ build {
       "--extra-vars", "${var.extra_vars}"
     ]
   }
+
+  provisioner "shell" {
+    inline = [
+      "sudo yum install -y libselinux-python3 python3 python3-pip tmux vim git bash-completion curl wget unzip",
+      "sudo python3 -m pip install --upgrade pip",
+      "sudo pip3 install s3cmd==2.3.0 ansible==4.10.0 python-openstackclient==5.8.0"
+    ]
+  }
+
 }

openstack-ood/variables.pkr.hcl

 variable "root_ssh_key" {
   type        = string
+  default     = ""
   description = "The root key to use for ssh"
 }
 

requirements.txt

+certifi==2025.1.31
+charset-normalizer==3.4.1
+idna==3.10
+python-gitlab==5.6.0
+requests==2.32.3
+requests-toolbelt==1.0.0
+urllib3==2.4.0

utils/README.md

+### Description
+These utility scripts avoid copying each ci variable manually which is tedious.
+- The gitlab-ci-vars-reader.py reads variables from a specific project or a pipeline (depending on the options provided) and copies them into a yaml file
+- The gitlab-ci-vars-updater.py takes a yaml file containing key value pairs in yaml format as an input. It then creates/updates project variables or pipeline variables (depending on the options provided)
+
+### Prerequisites
+```
+python -m venv ~/venvs/gitlab
+source ~/venvs/gitlab/bin/activate
+pip install -r requirements
+```
+
+### Setup
+```
+cd utils
+mv gitlab.ini.example gitlab.ini
+```
+Make changes to the gitlab.ini as you require.
+[Create a personal access token](https://docs.gitlab.com/user/profile/personal_access_tokens/) via the gitlab UI and copy it to the private_token field in gitlab.ini file
+
+### Usage
+> Create an empty schedule pipeline before you try this out.
+```
+python3 gitlab-ci-vars-reader.py --config_file gitlab.ini --project_id <PROJECT_ID> --sched_pipeline_id <PIPELINE_ID> --var_file ci-variables.yaml
+
+python3 gitlab-ci-vars-updater.py --config_file gitlab.ini --project_id <PROJECT_ID> --sched_pipeline_id <NEW-PIPELINE_ID> --var_file ci-variables.yaml
+```

utils/gitlab-ci-vars-reader.py

+import argparse
+
+import gitlab
+import yaml
+
+
+# Function to fetch all CI/CD variables from a GitLab project
+def fetch_variables(project):
+    p_variables = list(project.variables.list(iterator=True))
+    variables = [var.asdict() for var in p_variables]
+
+    return variables
+
+
+def fetch_sched_variables(sched_pipeline):
+    variables = sched_pipeline.attributes["variables"]
+    return variables
+
+
+# Main function to load the config and fetch variables
+def main():
+    # Setup argument parser
+    parser = argparse.ArgumentParser(description="GitLab CI/CD Variable reader")
+    parser.add_argument(
+        "--config_file",
+        type=str,
+        default="gitlab.ini",
+        required=True,
+        help="Path to the configuration file (default: gitlab.ini)",
+    )
+    parser.add_argument(
+        "--var_file",
+        type=str,
+        default="ci-variables.yaml",
+        help="Path to the CI vars file (default: ci-variables.yaml)",
+    )
+    parser.add_argument(
+        "--project_name",
+        type=str,
+        required=True,
+        help="Gitlab project name with namespace",
+    )
+    parser.add_argument(
+        "--sched_pipeline_id",
+        type=int,
+        help="Gitlab project scheduled pipeline ID",
+    )
+
+    # Parse the arguments
+    args = parser.parse_args()
+
+    gl = gitlab.Gitlab.from_config("uabrc", [args.config_file])
+    project = gl.projects.get(args.project_name)
+
+    # Fetch project or sched pipeline variables
+    if not args.sched_pipeline_id:
+        variables = fetch_variables(project)
+    else:
+        sched_pipeline = project.pipelineschedules.get(args.sched_pipeline_id)
+        variables = fetch_sched_variables(sched_pipeline)
+
+    try:
+        with open(args.var_file, mode="wt", encoding="utf-8") as file:
+            yaml.dump(variables, file, explicit_start=True)
+    except FileNotFoundError:
+        print(f"Error: Writing File to '{args.var_file}'")
+        exit(1)
+
+
+# Run the main function
+if __name__ == "__main__":
+    main()

utils/gitlab-ci-vars-updater.py

+import argparse
+
+import gitlab
+import yaml
+
+
+def load_file(file_path):
+    try:
+        with open(file_path, mode="rt", encoding="utf-8") as file:
+            return yaml.safe_load(file)
+    except FileNotFoundError:
+        print(f"Error: Configuration file '{file_path}' not found.")
+        exit(1)
+
+
+# Function to create or update a GitLab CI/CD variable
+def create_or_update_variable(project, var_dict):
+    key = var_dict.get("key")
+    scope = var_dict.get("environment_scope", "*")
+    p_variable = None
+
+    DEFAULTS = {
+        "variable_type": "env_var",
+        "hidden": False,
+        "protected": False,
+        "masked": False,
+        "environment_scope": "*",
+        "raw": False,
+        "description": None,
+    }
+
+    # Merge defaults with var_dict
+    var_dict = {**DEFAULTS, **var_dict}
+
+    # Fetch a variable with matching key and scope
+    try:
+        all_vars = project.variables.list(get_all=True)
+        for var in all_vars:
+            if var.key == key and var.environment_scope == scope:
+                p_variable = var
+                break
+    except gitlab.exceptions.GitlabGetError:
+        print("Variable not found")
+        exit(1)
+
+    # Check if the variable exists and same as input
+    if p_variable is not None:
+        if p_variable.asdict() != var_dict:
+            # if not same update the project variable
+            print(f"Updating {p_variable.attributes['key']}")
+            p_variable.delete()
+            return project.variables.create(var_dict)
+        else:
+            print(f"variable {var_dict["key"]} already exists")
+    # Create variable if it doesn't exist in the project
+    else:
+        print(f"Creating variable {var_dict["key"]}")
+        return project.variables.create(var_dict)
+
+
+def get_pipeline_vars_by_key(sched_pipeline, key_name):
+    p_vars = sched_pipeline.attributes["variables"]
+    for p_variable in p_vars:
+        if p_variable.get("key") == key_name:
+            return p_variable
+
+
+# Function to create or update a schedule pipeline variable
+def create_or_update_sched_vars(sched_pipeline, var_dict):
+    # Check if the variable exists in the sched pipeline
+    p_variable = get_pipeline_vars_by_key(sched_pipeline, var_dict["key"])
+    if p_variable:
+        # Check if the attributes are the same
+        if p_variable != var_dict:
+            # If not update the value in the project
+            sched_pipeline.variables.delete(p_variable["key"])
+            sched_pipeline.variables.create(var_dict)
+        else:
+            print(f"variable {var_dict["key"]} already exists")
+    # Create variable if it doesn't exist in the project
+    else:
+        print(f"Creating variable {var_dict["key"]}")
+        return sched_pipeline.variables.create(var_dict)
+
+
+def main():
+    # Setup argument parser
+    parser = argparse.ArgumentParser(description="GitLab CI/CD Variables Updater")
+    parser.add_argument(
+        "--config_file",
+        type=str,
+        default="gitlab.ini",
+        required=True,
+        help="Path to the configuration file (default: gitlab.ini)",
+    )
+    parser.add_argument(
+        "--var_file",
+        type=str,
+        default="ci-variables.yaml",
+        help="Path to the CI vars file (default: ci-variables.yaml)",
+    )
+    parser.add_argument(
+        "--project_name",
+        type=str,
+        required=True,
+        help="Gitlab project name with namespace",
+    )
+    parser.add_argument(
+        "--sched_pipeline_id",
+        type=int,
+        help="Gitlab project scheduled pipeline ID",
+    )
+
+    # Parse the arguments
+    args = parser.parse_args()
+
+    gl = gitlab.Gitlab.from_config("uabrc", [args.config_file])
+    project = gl.projects.get(args.project_name)
+
+    # Load the CI vars file
+    var_list = load_file(args.var_file)
+
+    # Create or update all variables
+    for var_dict in var_list:
+        if not args.sched_pipeline_id:
+            create_or_update_variable(project, var_dict)
+        else:
+            sched_pipeline = project.pipelineschedules.get(args.sched_pipeline_id)
+            create_or_update_sched_vars(sched_pipeline, var_dict)
+
+
+if __name__ == "__main__":
+    main()

utils/gitlab.ini.example

+[global]
+default = uabrc
+ssl_verify = true
+timeout = 5
+per_page = 100
+
+[uabrc]
+url = https://gitlab.rc.uab.edu
+private_token =
+api_version = 4