From aa196497b41c9d8bda155e2b3f369010fd52d5f1 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Mon, 28 Oct 2024 16:49:13 -0400
Subject: [PATCH 01/18] feat: Add scripts to read and write the cicd vars
---
config.json | 5 +++
gitlab-ci-vars-reader.py | 71 +++++++++++++++++++++++++++++++++
gitlab-ci-vars-updater.py | 84 +++++++++++++++++++++++++++++++++++++++
3 files changed, 160 insertions(+)
create mode 100644 config.json
create mode 100755 gitlab-ci-vars-reader.py
create mode 100644 gitlab-ci-vars-updater.py
diff --git a/config.json b/config.json
new file mode 100644
index 0000000..9965336
--- /dev/null
+++ b/config.json
@@ -0,0 +1,5 @@
+{
+ "gitlab_url": "",
+ "private_token": "",
+ "project_id": ""
+}
\ No newline at end of file
diff --git a/gitlab-ci-vars-reader.py b/gitlab-ci-vars-reader.py
new file mode 100755
index 0000000..8894dd3
--- /dev/null
+++ b/gitlab-ci-vars-reader.py
@@ -0,0 +1,71 @@
+import requests
+import json
+import argparse
+
+# Load configuration from JSON file
+def load_config(file_path):
+ try:
+ with open(file_path, 'r') as file:
+ return json.load(file)
+ except FileNotFoundError:
+ print(f"Error: Configuration file '{file_path}' not found.")
+ exit(1)
+
+# Function to fetch all CI/CD variables from a GitLab project
+def fetch_variables(config):
+ gitlab_url = config["gitlab_url"]
+ project_id = config["project_id"]
+ private_token = config["private_token"]
+
+ headers = {
+ "PRIVATE-TOKEN": private_token,
+ "Content-Type": "application/json"
+ }
+
+ url = f"{gitlab_url}/api/v4/projects/{project_id}/variables"
+ response = requests.get(url, headers=headers)
+
+ if response.status_code == 200:
+ variables = response.json()
+ return variables
+ else:
+ print(f"Failed to fetch variables. Status code: {response.status_code}")
+ return []
+
+# Function to format the fetched variables for input into the config file
+def format_variables_for_input(variables):
+ ci_variables = {var['key']: var['value'] for var in variables}
+ return ci_variables
+
+# Main function to load the config and fetch variables
+def main():
+ # Setup argument parser
+ parser = argparse.ArgumentParser(description="GitLab CI/CD Variables Reader")
+ parser.add_argument(
+ "--config",
+ type=str,
+ default="config.json",
+ help="Path to the configuration file (default: config.json)"
+ )
+
+ # Parse the arguments
+ args = parser.parse_args()
+
+ # Load the configuration file
+ config = load_config(args.config)
+
+ # Fetch and print the variables
+ variables = fetch_variables(config)
+
+ if variables:
+ print("Fetched variables:")
+ formatted_variables = format_variables_for_input(variables)
+ # Print the formatted variables dictionary in JSON format
+ print(json.dumps(formatted_variables, indent=4))
+ else:
+ print("No variables found.")
+
+# Run the main function
+if __name__ == "__main__":
+ main()
+
diff --git a/gitlab-ci-vars-updater.py b/gitlab-ci-vars-updater.py
new file mode 100644
index 0000000..d400669
--- /dev/null
+++ b/gitlab-ci-vars-updater.py
@@ -0,0 +1,84 @@
+import requests
+import json
+import argparse
+
+# Load configuration from JSON file
+def load_config(file_path):
+ try:
+ with open(file_path, 'r') as file:
+ return json.load(file)
+ except FileNotFoundError:
+ print(f"Error: Configuration file '{file_path}' not found.")
+ exit(1)
+
+# Function to create or update a GitLab CI/CD variable
+def create_or_update_variable(config, var_name, var_value):
+ gitlab_url = config["gitlab_url"]
+ project_id = config["project_id"]
+ private_token = config["private_token"]
+
+ headers = {
+ "PRIVATE-TOKEN": private_token,
+ "Content-Type": "application/json"
+ }
+
+ url = f"{gitlab_url}/api/v4/projects/{project_id}/variables/{var_name}"
+
+ # Check if the variable already exists
+ response = requests.get(url, headers=headers)
+
+ if response.status_code == 404:
+ print(f"Variable '{var_name}' does not exist. Creating new variable...")
+ create_url = f"{gitlab_url}/api/v4/projects/{project_id}/variables"
+ data = {
+ "key": var_name,
+ "value": var_value,
+ "variable_type": "env_var", # Can be 'file' for file variables
+ "protected": False, # Set to True if you want the variable protected
+ "masked": False # Set to True if you want the variable masked
+ }
+ create_response = requests.post(create_url, headers=headers, json=data)
+
+ if create_response.status_code == 201:
+ print(f"Variable '{var_name}' created successfully.")
+ else:
+ print(f"Failed to create variable '{var_name}'. Status code: {create_response.status_code}")
+ elif response.status_code == 200:
+ print(f"Variable '{var_name}' already exists. Updating variable...")
+ data = {"value": var_value}
+ update_response = requests.put(url, headers=headers, json=data)
+
+ if update_response.status_code == 200:
+ print(f"Variable '{var_name}' updated successfully.")
+ else:
+ print(f"Failed to update variable '{var_name}'. Status code: {update_response.status_code}")
+ else:
+ print(f"Error checking variable '{var_name}'. Status code: {response.status_code}")
+
+# Main function to load the config and apply variables
+def main():
+ # Setup argument parser
+ parser = argparse.ArgumentParser(description="GitLab CI/CD Variables Updater")
+ parser.add_argument(
+ "--config",
+ type=str,
+ default="config.json",
+ help="Path to the configuration file (default: config.json)"
+ )
+
+ # Parse the arguments
+ args = parser.parse_args()
+
+ # Load the configuration file
+ config = load_config(args.config)
+
+ ci_variables = config["ci_variables"]
+
+ # Loop through and create/update all variables
+ for var_name, var_value in ci_variables.items():
+ create_or_update_variable(config, var_name, var_value)
+
+# Run the main function
+if __name__ == "__main__":
+ main()
+
--
GitLab
From 51aed5aa787a649305cfb9f668aa40c9936a763c Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Mon, 28 Oct 2024 23:33:34 -0400
Subject: [PATCH 02/18] feat: Separate auth config and vars file
---
gitlab-ci-vars-reader.py | 12 ++++++------
gitlab-ci-vars-updater.py | 19 +++++++++++++------
2 files changed, 19 insertions(+), 12 deletions(-)
diff --git a/gitlab-ci-vars-reader.py b/gitlab-ci-vars-reader.py
index 8894dd3..6722606 100755
--- a/gitlab-ci-vars-reader.py
+++ b/gitlab-ci-vars-reader.py
@@ -14,7 +14,7 @@ def load_config(file_path):
# Function to fetch all CI/CD variables from a GitLab project
def fetch_variables(config):
gitlab_url = config["gitlab_url"]
- project_id = config["project_id"]
+ project_id = config["src_project_id"]
private_token = config["private_token"]
headers = {
@@ -22,7 +22,7 @@ def fetch_variables(config):
"Content-Type": "application/json"
}
- url = f"{gitlab_url}/api/v4/projects/{project_id}/variables"
+ url = f"{gitlab_url}/api/v4/projects/{project_id}/variables?per_page=100"
response = requests.get(url, headers=headers)
if response.status_code == 200:
@@ -42,9 +42,9 @@ def main():
# Setup argument parser
parser = argparse.ArgumentParser(description="GitLab CI/CD Variables Reader")
parser.add_argument(
- "--config",
+ "--auth_config",
type=str,
- default="config.json",
+ default="auth-config.json",
help="Path to the configuration file (default: config.json)"
)
@@ -52,13 +52,13 @@ def main():
args = parser.parse_args()
# Load the configuration file
- config = load_config(args.config)
+ config = load_config(args.auth_config)
# Fetch and print the variables
variables = fetch_variables(config)
if variables:
- print("Fetched variables:")
+ print("ci_variables:")
formatted_variables = format_variables_for_input(variables)
# Print the formatted variables dictionary in JSON format
print(json.dumps(formatted_variables, indent=4))
diff --git a/gitlab-ci-vars-updater.py b/gitlab-ci-vars-updater.py
index d400669..5e51a25 100644
--- a/gitlab-ci-vars-updater.py
+++ b/gitlab-ci-vars-updater.py
@@ -14,7 +14,7 @@ def load_config(file_path):
# Function to create or update a GitLab CI/CD variable
def create_or_update_variable(config, var_name, var_value):
gitlab_url = config["gitlab_url"]
- project_id = config["project_id"]
+ project_id = config["dest_project_id"]
private_token = config["private_token"]
headers = {
@@ -55,14 +55,20 @@ def create_or_update_variable(config, var_name, var_value):
else:
print(f"Error checking variable '{var_name}'. Status code: {response.status_code}")
-# Main function to load the config and apply variables
+# Main function to load the auth config and apply variables
def main():
# Setup argument parser
parser = argparse.ArgumentParser(description="GitLab CI/CD Variables Updater")
parser.add_argument(
- "--config",
+ "--auth_config",
type=str,
- default="config.json",
+ default="auth-config.json",
+ help="Path to the configuration file (default: config.json)"
+ )
+ parser.add_argument(
+ "--vars",
+ type=str,
+ default="ci-variables.json",
help="Path to the configuration file (default: config.json)"
)
@@ -70,9 +76,10 @@ def main():
args = parser.parse_args()
# Load the configuration file
- config = load_config(args.config)
+ config = load_config(args.auth_config)
+ vars_dict = load_config(args.vars)
- ci_variables = config["ci_variables"]
+ ci_variables = vars_dict["ci_variables"]
# Loop through and create/update all variables
for var_name, var_value in ci_variables.items():
--
GitLab
From 14abc939459baa94de0818a0e00b257d576b0b20 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Mon, 28 Oct 2024 23:38:06 -0400
Subject: [PATCH 03/18] feat: Move files to a dedicated dir
---
gitlab-ci-vars-reader.py => scripts/gitlab-ci-vars-reader.py | 0
gitlab-ci-vars-updater.py => scripts/gitlab-ci-vars-updater.py | 0
2 files changed, 0 insertions(+), 0 deletions(-)
rename gitlab-ci-vars-reader.py => scripts/gitlab-ci-vars-reader.py (100%)
rename gitlab-ci-vars-updater.py => scripts/gitlab-ci-vars-updater.py (100%)
diff --git a/gitlab-ci-vars-reader.py b/scripts/gitlab-ci-vars-reader.py
similarity index 100%
rename from gitlab-ci-vars-reader.py
rename to scripts/gitlab-ci-vars-reader.py
diff --git a/gitlab-ci-vars-updater.py b/scripts/gitlab-ci-vars-updater.py
similarity index 100%
rename from gitlab-ci-vars-updater.py
rename to scripts/gitlab-ci-vars-updater.py
--
GitLab
From fac0002a75bde3356dfae4330b66fa96a4a57777 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Mon, 28 Apr 2025 19:54:00 -0400
Subject: [PATCH 04/18] feat: Use gitlab module vs requests to modify cicd vars
---
config.json | 5 --
scripts/gitlab-ci-vars-reader.py | 94 ++++++++++----------
scripts/gitlab-ci-vars-updater.py | 142 +++++++++++++++++-------------
3 files changed, 131 insertions(+), 110 deletions(-)
delete mode 100644 config.json
diff --git a/config.json b/config.json
deleted file mode 100644
index 9965336..0000000
--- a/config.json
+++ /dev/null
@@ -1,5 +0,0 @@
-{
- "gitlab_url": "",
- "private_token": "",
- "project_id": ""
-}
\ No newline at end of file
diff --git a/scripts/gitlab-ci-vars-reader.py b/scripts/gitlab-ci-vars-reader.py
index 6722606..9e17ae9 100755
--- a/scripts/gitlab-ci-vars-reader.py
+++ b/scripts/gitlab-ci-vars-reader.py
@@ -1,71 +1,73 @@
-import requests
-import json
import argparse
+import json
+
+import gitlab
-# Load configuration from JSON file
-def load_config(file_path):
- try:
- with open(file_path, 'r') as file:
- return json.load(file)
- except FileNotFoundError:
- print(f"Error: Configuration file '{file_path}' not found.")
- exit(1)
# Function to fetch all CI/CD variables from a GitLab project
-def fetch_variables(config):
- gitlab_url = config["gitlab_url"]
- project_id = config["src_project_id"]
- private_token = config["private_token"]
+def fetch_variables(project):
+ p_variables = list(project.variables.list(iterator=True))
+ variables = [var.asdict() for var in p_variables]
- headers = {
- "PRIVATE-TOKEN": private_token,
- "Content-Type": "application/json"
- }
+ return variables
- url = f"{gitlab_url}/api/v4/projects/{project_id}/variables?per_page=100"
- response = requests.get(url, headers=headers)
- if response.status_code == 200:
- variables = response.json()
- return variables
- else:
- print(f"Failed to fetch variables. Status code: {response.status_code}")
- return []
+def fetch_sched_variables(sched_pipeline):
+ variables = sched_pipeline.attributes["variables"]
+ return variables
-# Function to format the fetched variables for input into the config file
-def format_variables_for_input(variables):
- ci_variables = {var['key']: var['value'] for var in variables}
- return ci_variables
# Main function to load the config and fetch variables
def main():
# Setup argument parser
- parser = argparse.ArgumentParser(description="GitLab CI/CD Variables Reader")
+ parser = argparse.ArgumentParser(description="GitLab CI/CD Variable reader")
parser.add_argument(
- "--auth_config",
+ "--config_file",
type=str,
- default="auth-config.json",
- help="Path to the configuration file (default: config.json)"
+ default="gitlab.ini",
+ required=True,
+ help="Path to the configuration file (default: gitlab.ini)",
+ )
+ parser.add_argument(
+ "--var_file",
+ type=str,
+ default="ci-variables.json",
+ help="Path to the CI vars file (default: ci-variables.json)",
+ )
+ parser.add_argument(
+ "--project_id",
+ type=int,
+ default="",
+ required=True,
+ help="Gitlab project ID to read variables from",
+ )
+ parser.add_argument(
+ "--sched_pipeline_id",
+ type=int,
+ help="Gitlab project scheduled pipeline ID",
)
# Parse the arguments
args = parser.parse_args()
- # Load the configuration file
- config = load_config(args.auth_config)
+ gl = gitlab.Gitlab.from_config("uabrc", [args.config_file])
+ project = gl.projects.get(args.project_id)
- # Fetch and print the variables
- variables = fetch_variables(config)
-
- if variables:
- print("ci_variables:")
- formatted_variables = format_variables_for_input(variables)
- # Print the formatted variables dictionary in JSON format
- print(json.dumps(formatted_variables, indent=4))
+ # Fetch project or sched pipeline variables
+ if not args.sched_pipeline_id:
+ variables = fetch_variables(project)
else:
- print("No variables found.")
+ sched_pipeline = project.pipelineschedules.get(args.sched_pipeline_id)
+ variables = fetch_sched_variables(sched_pipeline)
+
+ try:
+ with open(args.var_file, "w") as file:
+ json.dump(variables, file, indent=2)
+ except FileNotFoundError:
+ print(f"Error: Writing File to '{args.var_file}'")
+ exit(1)
+
# Run the main function
if __name__ == "__main__":
main()
-
diff --git a/scripts/gitlab-ci-vars-updater.py b/scripts/gitlab-ci-vars-updater.py
index 5e51a25..1ed43c2 100644
--- a/scripts/gitlab-ci-vars-updater.py
+++ b/scripts/gitlab-ci-vars-updater.py
@@ -1,91 +1,115 @@
-import requests
-import json
import argparse
+import json
+
+import gitlab
-# Load configuration from JSON file
-def load_config(file_path):
+
+def load_file(file_path):
try:
- with open(file_path, 'r') as file:
+ with open(file_path, "r") as file:
return json.load(file)
except FileNotFoundError:
print(f"Error: Configuration file '{file_path}' not found.")
exit(1)
+
# Function to create or update a GitLab CI/CD variable
-def create_or_update_variable(config, var_name, var_value):
- gitlab_url = config["gitlab_url"]
- project_id = config["dest_project_id"]
- private_token = config["private_token"]
-
- headers = {
- "PRIVATE-TOKEN": private_token,
- "Content-Type": "application/json"
- }
-
- url = f"{gitlab_url}/api/v4/projects/{project_id}/variables/{var_name}"
-
- # Check if the variable already exists
- response = requests.get(url, headers=headers)
-
- if response.status_code == 404:
- print(f"Variable '{var_name}' does not exist. Creating new variable...")
- create_url = f"{gitlab_url}/api/v4/projects/{project_id}/variables"
- data = {
- "key": var_name,
- "value": var_value,
- "variable_type": "env_var", # Can be 'file' for file variables
- "protected": False, # Set to True if you want the variable protected
- "masked": False # Set to True if you want the variable masked
- }
- create_response = requests.post(create_url, headers=headers, json=data)
-
- if create_response.status_code == 201:
- print(f"Variable '{var_name}' created successfully.")
+def create_or_update_variable(project, var_dict):
+ # Check if the variable exists in the project
+ p_variable = project.variables.get(var_dict["key"]).asdict()
+ if p_variable:
+ if p_variable != var_dict:
+ # Check if the attributes are the same
+ for k, v in var_dict.items():
+ if p_variable[k] != v:
+ # If not update the value in the project
+ print(f"Updating key {k} value")
+ project.variables.update(k, {"value": v})
else:
- print(f"Failed to create variable '{var_name}'. Status code: {create_response.status_code}")
- elif response.status_code == 200:
- print(f"Variable '{var_name}' already exists. Updating variable...")
- data = {"value": var_value}
- update_response = requests.put(url, headers=headers, json=data)
-
- if update_response.status_code == 200:
- print(f"Variable '{var_name}' updated successfully.")
+ print(f"variable {var_dict["key"]} already exists")
+ # Create variable if it doesn't exist in the project
+ else:
+ for k, v in var_dict.items():
+ print(f"Creating variable {var_dict["key"]}")
+ project.variables.create(var_dict)
+
+
+def get_pipeline_vars_by_key(sched_pipeline, key_name):
+ p_vars = sched_pipeline.attributes["variables"]
+ # p_vars.sort(key=lambda x: x["key"])
+ for p_variable in p_vars:
+ if p_variable.get("key") == key_name:
+ return p_variable
+
+
+# Function to create or update a schedule pipeline variable
+def create_or_update_sched_vars(sched_pipeline, var_dict):
+ # Check if the variable exists in the sched pipeline
+ p_variable = get_pipeline_vars_by_key(sched_pipeline, var_dict["key"])
+ if p_variable:
+ if p_variable != var_dict:
+ # Check if the attributes are the same
+ for k, v in var_dict.items():
+ if p_variable[k] != v:
+ # If not update the value in the project
+ print(f"Updating key {k} value")
+ sched_pipeline.variables.delete(p_variable["key"])
+ sched_pipeline.variables.create(var_dict)
else:
- print(f"Failed to update variable '{var_name}'. Status code: {update_response.status_code}")
+ print(f"variable {var_dict["key"]} already exists")
+ # Create variable if it doesn't exist in the project
else:
- print(f"Error checking variable '{var_name}'. Status code: {response.status_code}")
+ for k, v in var_dict.items():
+ print(f"Creating variable {var_dict["key"]}")
+ return sched_pipeline.variables.create(var_dict)
+
-# Main function to load the auth config and apply variables
def main():
# Setup argument parser
parser = argparse.ArgumentParser(description="GitLab CI/CD Variables Updater")
parser.add_argument(
- "--auth_config",
+ "--config_file",
type=str,
- default="auth-config.json",
- help="Path to the configuration file (default: config.json)"
+ default="gitlab.ini",
+ required=True,
+ help="Path to the configuration file (default: gitlab.ini)",
)
parser.add_argument(
- "--vars",
+ "--var_file",
type=str,
default="ci-variables.json",
- help="Path to the configuration file (default: config.json)"
+ help="Path to the CI vars file (default: ci-variables.json)",
+ )
+ parser.add_argument(
+ "--project_id",
+ type=int,
+ required=True,
+ help="Gitlab project ID for target variables",
+ )
+ parser.add_argument(
+ "--sched_pipeline_id",
+ type=int,
+ help="Gitlab project scheduled pipeline ID",
)
# Parse the arguments
args = parser.parse_args()
- # Load the configuration file
- config = load_config(args.auth_config)
- vars_dict = load_config(args.vars)
+ gl = gitlab.Gitlab.from_config("uabrc", [args.config_file])
+ project = gl.projects.get(args.project_id)
+
+ # Load the CI vars file
+ var_list = load_file(args.var_file)
+ # var_list.sort(key=lambda x: x["key"])
- ci_variables = vars_dict["ci_variables"]
+ # Create or update all variables
+ for var_dict in var_list:
+ if not args.sched_pipeline_id:
+ create_or_update_variable(project, var_dict)
+ else:
+ sched_pipeline = project.pipelineschedules.get(args.sched_pipeline_id)
+ create_or_update_sched_vars(sched_pipeline, var_dict)
- # Loop through and create/update all variables
- for var_name, var_value in ci_variables.items():
- create_or_update_variable(config, var_name, var_value)
-# Run the main function
if __name__ == "__main__":
main()
-
--
GitLab
From 1f33598ad07f895cfbd3b03cbc42e4ec557f9b12 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Wed, 30 Apr 2025 17:21:24 -0400
Subject: [PATCH 05/18] Add an example config file to access gitlab api
---
scripts/gitlab.ini.example | 10 ++++++++++
1 file changed, 10 insertions(+)
create mode 100644 scripts/gitlab.ini.example
diff --git a/scripts/gitlab.ini.example b/scripts/gitlab.ini.example
new file mode 100644
index 0000000..6b21153
--- /dev/null
+++ b/scripts/gitlab.ini.example
@@ -0,0 +1,10 @@
+[global]
+default = uabrc
+ssl_verify = true
+timeout = 5
+per_page = 100
+
+[uabrc]
+url = https://gitlab.rc.uab.edu
+private_token =
+api_version = 4
--
GitLab
From 05fc1163c1dff5ca7cb9d912bf9ffae411fb398c Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Thu, 1 May 2025 14:18:36 -0400
Subject: [PATCH 06/18] Add requirements file for pip env installs
---
requirements.txt | 7 +++++++
1 file changed, 7 insertions(+)
create mode 100644 requirements.txt
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..5149467
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,7 @@
+certifi==2025.1.31
+charset-normalizer==3.4.1
+idna==3.10
+python-gitlab==5.6.0
+requests==2.32.3
+requests-toolbelt==1.0.0
+urllib3==2.4.0
--
GitLab
From ef30db8fcc74b88f9ddad11e166c5958d113ac78 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Thu, 1 May 2025 14:47:21 -0400
Subject: [PATCH 07/18] Add yaml support for ci-variables files
The yaml format is for user convenience. It is converted into JSON
for performance. Processing is still done in JSON because it is
faster data exchange format over http.
---
scripts/gitlab-ci-vars-reader.py | 10 +++++-----
scripts/gitlab-ci-vars-updater.py | 10 +++++-----
2 files changed, 10 insertions(+), 10 deletions(-)
diff --git a/scripts/gitlab-ci-vars-reader.py b/scripts/gitlab-ci-vars-reader.py
index 9e17ae9..0e8e4f9 100755
--- a/scripts/gitlab-ci-vars-reader.py
+++ b/scripts/gitlab-ci-vars-reader.py
@@ -1,7 +1,7 @@
import argparse
-import json
import gitlab
+import yaml
# Function to fetch all CI/CD variables from a GitLab project
@@ -31,8 +31,8 @@ def main():
parser.add_argument(
"--var_file",
type=str,
- default="ci-variables.json",
- help="Path to the CI vars file (default: ci-variables.json)",
+ default="ci-variables.yaml",
+ help="Path to the CI vars file (default: ci-variables.yaml)",
)
parser.add_argument(
"--project_id",
@@ -61,8 +61,8 @@ def main():
variables = fetch_sched_variables(sched_pipeline)
try:
- with open(args.var_file, "w") as file:
- json.dump(variables, file, indent=2)
+ with open(args.var_file, mode="wt", encoding="utf-8") as file:
+ yaml.dump(variables, file, explicit_start=True)
except FileNotFoundError:
print(f"Error: Writing File to '{args.var_file}'")
exit(1)
diff --git a/scripts/gitlab-ci-vars-updater.py b/scripts/gitlab-ci-vars-updater.py
index 1ed43c2..1221d83 100644
--- a/scripts/gitlab-ci-vars-updater.py
+++ b/scripts/gitlab-ci-vars-updater.py
@@ -1,13 +1,13 @@
import argparse
-import json
import gitlab
+import yaml
def load_file(file_path):
try:
- with open(file_path, "r") as file:
- return json.load(file)
+ with open(file_path, mode="rt", encoding="utf-8") as file:
+ return yaml.safe_load(file)
except FileNotFoundError:
print(f"Error: Configuration file '{file_path}' not found.")
exit(1)
@@ -77,8 +77,8 @@ def main():
parser.add_argument(
"--var_file",
type=str,
- default="ci-variables.json",
- help="Path to the CI vars file (default: ci-variables.json)",
+ default="ci-variables.yaml",
+ help="Path to the CI vars file (default: ci-variables.yaml)",
)
parser.add_argument(
"--project_id",
--
GitLab
From 3e01c15af6ea89ef0eec8caebdc6c67de0fd2eac Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Thu, 1 May 2025 14:58:27 -0400
Subject: [PATCH 08/18] Use project name instead of ID for input
---
scripts/gitlab-ci-vars-reader.py | 9 ++++-----
scripts/gitlab-ci-vars-updater.py | 8 ++++----
2 files changed, 8 insertions(+), 9 deletions(-)
diff --git a/scripts/gitlab-ci-vars-reader.py b/scripts/gitlab-ci-vars-reader.py
index 0e8e4f9..c573a56 100755
--- a/scripts/gitlab-ci-vars-reader.py
+++ b/scripts/gitlab-ci-vars-reader.py
@@ -35,11 +35,10 @@ def main():
help="Path to the CI vars file (default: ci-variables.yaml)",
)
parser.add_argument(
- "--project_id",
- type=int,
- default="",
+ "--project_name",
+ type=str,
required=True,
- help="Gitlab project ID to read variables from",
+ help="Gitlab project name with namespace",
)
parser.add_argument(
"--sched_pipeline_id",
@@ -51,7 +50,7 @@ def main():
args = parser.parse_args()
gl = gitlab.Gitlab.from_config("uabrc", [args.config_file])
- project = gl.projects.get(args.project_id)
+ project = gl.projects.get(args.project_name)
# Fetch project or sched pipeline variables
if not args.sched_pipeline_id:
diff --git a/scripts/gitlab-ci-vars-updater.py b/scripts/gitlab-ci-vars-updater.py
index 1221d83..df46972 100644
--- a/scripts/gitlab-ci-vars-updater.py
+++ b/scripts/gitlab-ci-vars-updater.py
@@ -81,10 +81,10 @@ def main():
help="Path to the CI vars file (default: ci-variables.yaml)",
)
parser.add_argument(
- "--project_id",
- type=int,
+ "--project_name",
+ type=str,
required=True,
- help="Gitlab project ID for target variables",
+ help="Gitlab project name with namespace",
)
parser.add_argument(
"--sched_pipeline_id",
@@ -96,7 +96,7 @@ def main():
args = parser.parse_args()
gl = gitlab.Gitlab.from_config("uabrc", [args.config_file])
- project = gl.projects.get(args.project_id)
+ project = gl.projects.get(args.project_name)
# Load the CI vars file
var_list = load_file(args.var_file)
--
GitLab
From 0a8efb1f978b4db1eaeb1e7fff4f9d8f0d7fd5d8 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Thu, 1 May 2025 16:19:12 -0400
Subject: [PATCH 09/18] Handle the exception when variable is not found
---
scripts/gitlab-ci-vars-updater.py | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/scripts/gitlab-ci-vars-updater.py b/scripts/gitlab-ci-vars-updater.py
index df46972..c09e68a 100644
--- a/scripts/gitlab-ci-vars-updater.py
+++ b/scripts/gitlab-ci-vars-updater.py
@@ -16,7 +16,11 @@ def load_file(file_path):
# Function to create or update a GitLab CI/CD variable
def create_or_update_variable(project, var_dict):
# Check if the variable exists in the project
- p_variable = project.variables.get(var_dict["key"]).asdict()
+ try:
+ p_variable = project.variables.get(var_dict["key"]).asdict()
+ except gitlab.exceptions.GitlabGetError:
+ print("Variable not found")
+ p_variable = False
if p_variable:
if p_variable != var_dict:
# Check if the attributes are the same
--
GitLab
From e5e2c2b78c47fcf64e567244b4f2d6a8a2362700 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Thu, 1 May 2025 16:25:52 -0400
Subject: [PATCH 10/18] Add missing func return after creating var
---
scripts/gitlab-ci-vars-updater.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/scripts/gitlab-ci-vars-updater.py b/scripts/gitlab-ci-vars-updater.py
index c09e68a..e80d1f7 100644
--- a/scripts/gitlab-ci-vars-updater.py
+++ b/scripts/gitlab-ci-vars-updater.py
@@ -35,7 +35,7 @@ def create_or_update_variable(project, var_dict):
else:
for k, v in var_dict.items():
print(f"Creating variable {var_dict["key"]}")
- project.variables.create(var_dict)
+ return project.variables.create(var_dict)
def get_pipeline_vars_by_key(sched_pipeline, key_name):
--
GitLab
From f6b5cf3e6a2e3dfdd37e67994010aac4799d3958 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Fri, 2 May 2025 17:16:31 -0400
Subject: [PATCH 11/18] Add support for same key with different scopes
API doesn't support filtering the key based on env scope so we
do it manually.
---
scripts/gitlab-ci-vars-updater.py | 26 +++++++++++++++++++++++---
1 file changed, 23 insertions(+), 3 deletions(-)
diff --git a/scripts/gitlab-ci-vars-updater.py b/scripts/gitlab-ci-vars-updater.py
index e80d1f7..b67c63e 100644
--- a/scripts/gitlab-ci-vars-updater.py
+++ b/scripts/gitlab-ci-vars-updater.py
@@ -15,12 +15,19 @@ def load_file(file_path):
# Function to create or update a GitLab CI/CD variable
def create_or_update_variable(project, var_dict):
+ key = var_dict.get("key")
+ scope = var_dict.get("environment_scope", "*")
+ p_variable = None
# Check if the variable exists in the project
try:
- p_variable = project.variables.get(var_dict["key"]).asdict()
+ all_vars = project.variables.list(get_all=True)
+ for var in all_vars:
+ if var.key == key and var.environment_scope == scope:
+ p_variable = var.asdict()
+ break
except gitlab.exceptions.GitlabGetError:
print("Variable not found")
- p_variable = False
+ exit(1)
if p_variable:
if p_variable != var_dict:
# Check if the attributes are the same
@@ -28,7 +35,20 @@ def create_or_update_variable(project, var_dict):
if p_variable[k] != v:
# If not update the value in the project
print(f"Updating key {k} value")
- project.variables.update(k, {"value": v})
+ # project.variables.update(k, {"value": v})
+ project.variables.update(
+ key,
+ {
+ "value": var_dict["value"],
+ "environment_scope": scope,
+ "variable_type": var_dict.get("variable_type", "env_var"),
+ "masked": var_dict.get("masked", False),
+ "protected": var_dict.get("protected", False),
+ "raw": var_dict.get("raw", False),
+ "description": var_dict.get("description", None),
+ },
+ )
+ break
else:
print(f"variable {var_dict["key"]} already exists")
# Create variable if it doesn't exist in the project
--
GitLab
From 4ed47f073a0451641616a612c0237acfbfd7d887 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Fri, 2 May 2025 17:31:48 -0400
Subject: [PATCH 12/18] Misc updates to commented sections
---
scripts/gitlab-ci-vars-updater.py | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/scripts/gitlab-ci-vars-updater.py b/scripts/gitlab-ci-vars-updater.py
index b67c63e..922369a 100644
--- a/scripts/gitlab-ci-vars-updater.py
+++ b/scripts/gitlab-ci-vars-updater.py
@@ -18,7 +18,7 @@ def create_or_update_variable(project, var_dict):
key = var_dict.get("key")
scope = var_dict.get("environment_scope", "*")
p_variable = None
- # Check if the variable exists in the project
+ # Fetch a variable with matching key and scope
try:
all_vars = project.variables.list(get_all=True)
for var in all_vars:
@@ -28,6 +28,7 @@ def create_or_update_variable(project, var_dict):
except gitlab.exceptions.GitlabGetError:
print("Variable not found")
exit(1)
+ # Check if the variable exists in the project
if p_variable:
if p_variable != var_dict:
# Check if the attributes are the same
@@ -35,7 +36,6 @@ def create_or_update_variable(project, var_dict):
if p_variable[k] != v:
# If not update the value in the project
print(f"Updating key {k} value")
- # project.variables.update(k, {"value": v})
project.variables.update(
key,
{
@@ -60,7 +60,6 @@ def create_or_update_variable(project, var_dict):
def get_pipeline_vars_by_key(sched_pipeline, key_name):
p_vars = sched_pipeline.attributes["variables"]
- # p_vars.sort(key=lambda x: x["key"])
for p_variable in p_vars:
if p_variable.get("key") == key_name:
return p_variable
@@ -124,7 +123,6 @@ def main():
# Load the CI vars file
var_list = load_file(args.var_file)
- # var_list.sort(key=lambda x: x["key"])
# Create or update all variables
for var_dict in var_list:
--
GitLab
From 46928010227ff0fb0dfd849ea5f547f3d0b742f1 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Sat, 3 May 2025 03:31:07 -0400
Subject: [PATCH 13/18] Remove unnecessary loop as we update only once
---
scripts/gitlab-ci-vars-updater.py | 21 +++++++++------------
1 file changed, 9 insertions(+), 12 deletions(-)
diff --git a/scripts/gitlab-ci-vars-updater.py b/scripts/gitlab-ci-vars-updater.py
index 922369a..7318de6 100644
--- a/scripts/gitlab-ci-vars-updater.py
+++ b/scripts/gitlab-ci-vars-updater.py
@@ -23,19 +23,11 @@ def create_or_update_variable(project, var_dict):
all_vars = project.variables.list(get_all=True)
for var in all_vars:
if var.key == key and var.environment_scope == scope:
- p_variable = var.asdict()
+ p_variable = var
break
except gitlab.exceptions.GitlabGetError:
print("Variable not found")
exit(1)
- # Check if the variable exists in the project
- if p_variable:
- if p_variable != var_dict:
- # Check if the attributes are the same
- for k, v in var_dict.items():
- if p_variable[k] != v:
- # If not update the value in the project
- print(f"Updating key {k} value")
project.variables.update(
key,
{
@@ -49,13 +41,18 @@ def create_or_update_variable(project, var_dict):
},
)
break
+
+ # Check if the variable exists and same as input
+ if p_variable is not None:
+ if p_variable.asdict() != var_dict:
+ # if not same update the project variable
+ print(f"Updating {p_variable.attributes['key']}")
else:
print(f"variable {var_dict["key"]} already exists")
# Create variable if it doesn't exist in the project
else:
- for k, v in var_dict.items():
- print(f"Creating variable {var_dict["key"]}")
- return project.variables.create(var_dict)
+ print(f"Creating variable {var_dict["key"]}")
+ return project.variables.create(var_dict)
def get_pipeline_vars_by_key(sched_pipeline, key_name):
--
GitLab
From 7fc47b62846c02112b9737c050641b152a45b660 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Sat, 3 May 2025 03:40:30 -0400
Subject: [PATCH 14/18] Use delete + create rather than update function
Update doesn't support scoped variables. So we manually filter scoped
variable and use delete and create to update the scoped variable.
This will avoid the http 409 error due to conflict.
---
scripts/gitlab-ci-vars-updater.py | 15 ++-------------
1 file changed, 2 insertions(+), 13 deletions(-)
diff --git a/scripts/gitlab-ci-vars-updater.py b/scripts/gitlab-ci-vars-updater.py
index 7318de6..458e4b8 100644
--- a/scripts/gitlab-ci-vars-updater.py
+++ b/scripts/gitlab-ci-vars-updater.py
@@ -28,25 +28,14 @@ def create_or_update_variable(project, var_dict):
except gitlab.exceptions.GitlabGetError:
print("Variable not found")
exit(1)
- project.variables.update(
- key,
- {
- "value": var_dict["value"],
- "environment_scope": scope,
- "variable_type": var_dict.get("variable_type", "env_var"),
- "masked": var_dict.get("masked", False),
- "protected": var_dict.get("protected", False),
- "raw": var_dict.get("raw", False),
- "description": var_dict.get("description", None),
- },
- )
- break
# Check if the variable exists and same as input
if p_variable is not None:
if p_variable.asdict() != var_dict:
# if not same update the project variable
print(f"Updating {p_variable.attributes['key']}")
+ p_variable.delete()
+ return project.variables.create(var_dict)
else:
print(f"variable {var_dict["key"]} already exists")
# Create variable if it doesn't exist in the project
--
GitLab
From 2579d8f6a69b3236e847eff40c7cf6b02c6f8233 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Sat, 3 May 2025 03:47:44 -0400
Subject: [PATCH 15/18] Add default fields if the user provides only key and
value
This provides flexibility for the user to define just the value for a
key. More fields can be added for a key which will override the defaults
---
scripts/gitlab-ci-vars-updater.py | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/scripts/gitlab-ci-vars-updater.py b/scripts/gitlab-ci-vars-updater.py
index 458e4b8..e66b88f 100644
--- a/scripts/gitlab-ci-vars-updater.py
+++ b/scripts/gitlab-ci-vars-updater.py
@@ -18,6 +18,20 @@ def create_or_update_variable(project, var_dict):
key = var_dict.get("key")
scope = var_dict.get("environment_scope", "*")
p_variable = None
+
+ DEFAULTS = {
+ "variable_type": "env_var",
+ "hidden": False,
+ "protected": False,
+ "masked": False,
+ "environment_scope": "*",
+ "raw": False,
+ "description": None,
+ }
+
+ # Merge defaults with var_dict
+ var_dict = {**DEFAULTS, **var_dict}
+
# Fetch a variable with matching key and scope
try:
all_vars = project.variables.list(get_all=True)
--
GitLab
From 9c457bfbcc708aac432ecc88c2c9b0c6b1efbafc Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Sat, 3 May 2025 04:02:08 -0400
Subject: [PATCH 16/18] Remove unnecessary loop as we update only once
---
scripts/gitlab-ci-vars-updater.py | 16 ++++++----------
1 file changed, 6 insertions(+), 10 deletions(-)
diff --git a/scripts/gitlab-ci-vars-updater.py b/scripts/gitlab-ci-vars-updater.py
index e66b88f..5da49d5 100644
--- a/scripts/gitlab-ci-vars-updater.py
+++ b/scripts/gitlab-ci-vars-updater.py
@@ -70,21 +70,17 @@ def create_or_update_sched_vars(sched_pipeline, var_dict):
# Check if the variable exists in the sched pipeline
p_variable = get_pipeline_vars_by_key(sched_pipeline, var_dict["key"])
if p_variable:
+ # Check if the attributes are the same
if p_variable != var_dict:
- # Check if the attributes are the same
- for k, v in var_dict.items():
- if p_variable[k] != v:
- # If not update the value in the project
- print(f"Updating key {k} value")
- sched_pipeline.variables.delete(p_variable["key"])
- sched_pipeline.variables.create(var_dict)
+ # If not update the value in the project
+ sched_pipeline.variables.delete(p_variable["key"])
+ sched_pipeline.variables.create(var_dict)
else:
print(f"variable {var_dict["key"]} already exists")
# Create variable if it doesn't exist in the project
else:
- for k, v in var_dict.items():
- print(f"Creating variable {var_dict["key"]}")
- return sched_pipeline.variables.create(var_dict)
+ print(f"Creating variable {var_dict["key"]}")
+ return sched_pipeline.variables.create(var_dict)
def main():
--
GitLab
From ad4ad4b170ae6541ddfe43a4bf3c159f51efb8db Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Tue, 6 May 2025 10:31:36 -0400
Subject: [PATCH 17/18] Add Readme file for utility scripts
---
scripts/README.md | 20 ++++++++++++++++++++
1 file changed, 20 insertions(+)
create mode 100644 scripts/README.md
diff --git a/scripts/README.md b/scripts/README.md
new file mode 100644
index 0000000..c1ad5e3
--- /dev/null
+++ b/scripts/README.md
@@ -0,0 +1,20 @@
+**Prerequisites**
+python -m venv ~/venvs/gitlab
+source ~/venvs/gitlab/bin/activate
+pip install -r requirements
+
+**Clone the repo**
+git clone -b feat-gl-cicd-var-scripts --single-branch https://gitlab.rc.uab.edu/atlurie/hpc-factory.git tmp-hpc-factory
+cd tmp-hpc-factory/scripts
+mv gitlab.ini.example gitlab.ini
+
+Make changes to the gitlab.ini as you require.
+[Create a personal access token](https://docs.gitlab.com/user/profile/personal_access_tokens/) via the gitlab UI and copy it to the private_token field in gitlab.ini file
+
+**Usage**
+> Create an empty schedule pipeline before you try this out.
+```
+python3 gitlab-ci-vars-reader.py --config_file gitlab.ini --project_id <PROJECT_ID> --sched_pipeline_id <PIPELINE_ID> --var_file ci-variables.json
+
+python3 gitlab-ci-vars-updater.py --config_file gitlab.ini --project_id <PROJECT_ID> --sched_pipeline_id <NEW-PIPELINE_ID> --var_file ci-variables.json
+```
--
GitLab
From 0d34d32784e752ffba5ce8ae059234264ee329ba Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Tue, 6 May 2025 13:25:30 -0400
Subject: [PATCH 18/18] Rename scripts dir to utils
---
{scripts => utils}/README.md | 23 +++++++++++++-------
{scripts => utils}/gitlab-ci-vars-reader.py | 0
{scripts => utils}/gitlab-ci-vars-updater.py | 0
{scripts => utils}/gitlab.ini.example | 0
4 files changed, 15 insertions(+), 8 deletions(-)
rename {scripts => utils}/README.md (54%)
rename {scripts => utils}/gitlab-ci-vars-reader.py (100%)
rename {scripts => utils}/gitlab-ci-vars-updater.py (100%)
rename {scripts => utils}/gitlab.ini.example (100%)
diff --git a/scripts/README.md b/utils/README.md
similarity index 54%
rename from scripts/README.md
rename to utils/README.md
index c1ad5e3..6a669e0 100644
--- a/scripts/README.md
+++ b/utils/README.md
@@ -1,20 +1,27 @@
-**Prerequisites**
+### Description
+These utility scripts avoid copying each ci variable manually which is tedious.
+- The gitlab-ci-vars-reader.py reads variables from a specific project or a pipeline (depending on the options provided) and copies them into a yaml file
+- The gitlab-ci-vars-updater.py takes a yaml file containing key value pairs in yaml format as an input. It then creates/updates project variables or pipeline variables (depending on the options provided)
+
+### Prerequisites
+```
python -m venv ~/venvs/gitlab
source ~/venvs/gitlab/bin/activate
pip install -r requirements
+```
-**Clone the repo**
-git clone -b feat-gl-cicd-var-scripts --single-branch https://gitlab.rc.uab.edu/atlurie/hpc-factory.git tmp-hpc-factory
-cd tmp-hpc-factory/scripts
+### Setup
+```
+cd utils
mv gitlab.ini.example gitlab.ini
-
+```
Make changes to the gitlab.ini as you require.
[Create a personal access token](https://docs.gitlab.com/user/profile/personal_access_tokens/) via the gitlab UI and copy it to the private_token field in gitlab.ini file
-**Usage**
+### Usage
> Create an empty schedule pipeline before you try this out.
```
-python3 gitlab-ci-vars-reader.py --config_file gitlab.ini --project_id <PROJECT_ID> --sched_pipeline_id <PIPELINE_ID> --var_file ci-variables.json
+python3 gitlab-ci-vars-reader.py --config_file gitlab.ini --project_id <PROJECT_ID> --sched_pipeline_id <PIPELINE_ID> --var_file ci-variables.yaml
-python3 gitlab-ci-vars-updater.py --config_file gitlab.ini --project_id <PROJECT_ID> --sched_pipeline_id <NEW-PIPELINE_ID> --var_file ci-variables.json
+python3 gitlab-ci-vars-updater.py --config_file gitlab.ini --project_id <PROJECT_ID> --sched_pipeline_id <NEW-PIPELINE_ID> --var_file ci-variables.yaml
```
diff --git a/scripts/gitlab-ci-vars-reader.py b/utils/gitlab-ci-vars-reader.py
similarity index 100%
rename from scripts/gitlab-ci-vars-reader.py
rename to utils/gitlab-ci-vars-reader.py
diff --git a/scripts/gitlab-ci-vars-updater.py b/utils/gitlab-ci-vars-updater.py
similarity index 100%
rename from scripts/gitlab-ci-vars-updater.py
rename to utils/gitlab-ci-vars-updater.py
diff --git a/scripts/gitlab.ini.example b/utils/gitlab.ini.example
similarity index 100%
rename from scripts/gitlab.ini.example
rename to utils/gitlab.ini.example
--
GitLab