Skip to content
Snippets Groups Projects
Commit a43040d4 authored by Eesaan Atluri's avatar Eesaan Atluri
Browse files

Merge branch 'feat-gl-cicd-var-scripts' into 'main'

Feat gl cicd var scripts

Closes #232

See merge request rc/hpc-factory!196
parents b9ed554a 190fe18e
No related branches found
No related tags found
No related merge requests found
certifi==2025.1.31
charset-normalizer==3.4.1
idna==3.10
python-gitlab==5.6.0
requests==2.32.3
requests-toolbelt==1.0.0
urllib3==2.4.0
### Description
These utility scripts avoid copying each ci variable manually which is tedious.
- The gitlab-ci-vars-reader.py reads variables from a specific project or a pipeline (depending on the options provided) and copies them into a yaml file
- The gitlab-ci-vars-updater.py takes a yaml file containing key value pairs in yaml format as an input. It then creates/updates project variables or pipeline variables (depending on the options provided)
### Prerequisites
```
python -m venv ~/venvs/gitlab
source ~/venvs/gitlab/bin/activate
pip install -r requirements
```
### Setup
```
cd utils
mv gitlab.ini.example gitlab.ini
```
Make changes to the gitlab.ini as you require.
[Create a personal access token](https://docs.gitlab.com/user/profile/personal_access_tokens/) via the gitlab UI and copy it to the private_token field in gitlab.ini file
### Usage
> Create an empty schedule pipeline before you try this out.
```
python3 gitlab-ci-vars-reader.py --config_file gitlab.ini --project_id <PROJECT_ID> --sched_pipeline_id <PIPELINE_ID> --var_file ci-variables.yaml
python3 gitlab-ci-vars-updater.py --config_file gitlab.ini --project_id <PROJECT_ID> --sched_pipeline_id <NEW-PIPELINE_ID> --var_file ci-variables.yaml
```
import argparse
import gitlab
import yaml
# Function to fetch all CI/CD variables from a GitLab project
def fetch_variables(project):
p_variables = list(project.variables.list(iterator=True))
variables = [var.asdict() for var in p_variables]
return variables
def fetch_sched_variables(sched_pipeline):
variables = sched_pipeline.attributes["variables"]
return variables
# Main function to load the config and fetch variables
def main():
# Setup argument parser
parser = argparse.ArgumentParser(description="GitLab CI/CD Variable reader")
parser.add_argument(
"--config_file",
type=str,
default="gitlab.ini",
required=True,
help="Path to the configuration file (default: gitlab.ini)",
)
parser.add_argument(
"--var_file",
type=str,
default="ci-variables.yaml",
help="Path to the CI vars file (default: ci-variables.yaml)",
)
parser.add_argument(
"--project_name",
type=str,
required=True,
help="Gitlab project name with namespace",
)
parser.add_argument(
"--sched_pipeline_id",
type=int,
help="Gitlab project scheduled pipeline ID",
)
# Parse the arguments
args = parser.parse_args()
gl = gitlab.Gitlab.from_config("uabrc", [args.config_file])
project = gl.projects.get(args.project_name)
# Fetch project or sched pipeline variables
if not args.sched_pipeline_id:
variables = fetch_variables(project)
else:
sched_pipeline = project.pipelineschedules.get(args.sched_pipeline_id)
variables = fetch_sched_variables(sched_pipeline)
try:
with open(args.var_file, mode="wt", encoding="utf-8") as file:
yaml.dump(variables, file, explicit_start=True)
except FileNotFoundError:
print(f"Error: Writing File to '{args.var_file}'")
exit(1)
# Run the main function
if __name__ == "__main__":
main()
import argparse
import gitlab
import yaml
def load_file(file_path):
try:
with open(file_path, mode="rt", encoding="utf-8") as file:
return yaml.safe_load(file)
except FileNotFoundError:
print(f"Error: Configuration file '{file_path}' not found.")
exit(1)
# Function to create or update a GitLab CI/CD variable
def create_or_update_variable(project, var_dict):
key = var_dict.get("key")
scope = var_dict.get("environment_scope", "*")
p_variable = None
DEFAULTS = {
"variable_type": "env_var",
"hidden": False,
"protected": False,
"masked": False,
"environment_scope": "*",
"raw": False,
"description": None,
}
# Merge defaults with var_dict
var_dict = {**DEFAULTS, **var_dict}
# Fetch a variable with matching key and scope
try:
all_vars = project.variables.list(get_all=True)
for var in all_vars:
if var.key == key and var.environment_scope == scope:
p_variable = var
break
except gitlab.exceptions.GitlabGetError:
print("Variable not found")
exit(1)
# Check if the variable exists and same as input
if p_variable is not None:
if p_variable.asdict() != var_dict:
# if not same update the project variable
print(f"Updating {p_variable.attributes['key']}")
p_variable.delete()
return project.variables.create(var_dict)
else:
print(f"variable {var_dict["key"]} already exists")
# Create variable if it doesn't exist in the project
else:
print(f"Creating variable {var_dict["key"]}")
return project.variables.create(var_dict)
def get_pipeline_vars_by_key(sched_pipeline, key_name):
p_vars = sched_pipeline.attributes["variables"]
for p_variable in p_vars:
if p_variable.get("key") == key_name:
return p_variable
# Function to create or update a schedule pipeline variable
def create_or_update_sched_vars(sched_pipeline, var_dict):
# Check if the variable exists in the sched pipeline
p_variable = get_pipeline_vars_by_key(sched_pipeline, var_dict["key"])
if p_variable:
# Check if the attributes are the same
if p_variable != var_dict:
# If not update the value in the project
sched_pipeline.variables.delete(p_variable["key"])
sched_pipeline.variables.create(var_dict)
else:
print(f"variable {var_dict["key"]} already exists")
# Create variable if it doesn't exist in the project
else:
print(f"Creating variable {var_dict["key"]}")
return sched_pipeline.variables.create(var_dict)
def main():
# Setup argument parser
parser = argparse.ArgumentParser(description="GitLab CI/CD Variables Updater")
parser.add_argument(
"--config_file",
type=str,
default="gitlab.ini",
required=True,
help="Path to the configuration file (default: gitlab.ini)",
)
parser.add_argument(
"--var_file",
type=str,
default="ci-variables.yaml",
help="Path to the CI vars file (default: ci-variables.yaml)",
)
parser.add_argument(
"--project_name",
type=str,
required=True,
help="Gitlab project name with namespace",
)
parser.add_argument(
"--sched_pipeline_id",
type=int,
help="Gitlab project scheduled pipeline ID",
)
# Parse the arguments
args = parser.parse_args()
gl = gitlab.Gitlab.from_config("uabrc", [args.config_file])
project = gl.projects.get(args.project_name)
# Load the CI vars file
var_list = load_file(args.var_file)
# Create or update all variables
for var_dict in var_list:
if not args.sched_pipeline_id:
create_or_update_variable(project, var_dict)
else:
sched_pipeline = project.pipelineschedules.get(args.sched_pipeline_id)
create_or_update_sched_vars(sched_pipeline, var_dict)
if __name__ == "__main__":
main()
[global]
default = uabrc
ssl_verify = true
timeout = 5
per_page = 100
[uabrc]
url = https://gitlab.rc.uab.edu
private_token =
api_version = 4
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment