From ec03fe3699e6c7616bd84cae9f02ba20030a8182 Mon Sep 17 00:00:00 2001
From: Krish Moodbidri <krish94@uab.edu>
Date: Fri, 22 Nov 2024 09:47:12 -0600
Subject: [PATCH 01/29] Update file .gitlab-ci.yml
---
.gitlab-ci.yml | 24 ++++++++++++++++++++++++
1 file changed, 24 insertions(+)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 18e0edb..5a3722a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -110,6 +110,30 @@ workflow:
reports:
dotenv: image.env
+build_compute_image:
+ stage: build
+ needs: [build_base_image]
+ tags:
+ - build
+ script:
+ - *update_ansible_repo
+ - *get_ansible_files
+ - export PKR_VAR_source_image=${BUILT_BASE_IMAGE_ID}
+ - export REPO_HEAD=$(git rev-parse --short HEAD)
+ - export PKR_VAR_flavor="${COMPUTE_BUILD_FLAVOR:-$PKR_VAR_flavor}"
+ - export PKR_VAR_build_instance_name="compute-${REPO_HEAD}"
+ - export PKR_VAR_image_date_suffix=false
+ - |
+ if [ $CI_PIPELINE_SOURCE == 'merge_request_event' ]; then
+ export PKR_VAR_image_name="compute-PR-${CI_MERGE_REQUEST_IID}"
+ elif [ $CI_PIPELINE_SOURCE == 'schedule' ]; then
+ export PKR_VAR_image_name="compute-${BUILD_DATE}"
+ fi
+ - packer init openstack-compute
+ - packer validate openstack-compute
+ - packer build -machine-readable openstack-compute | tee compute_build.log
+
+
build_http_proxy_image:
stage: build
environment:
--
GitLab
From 53d68ff4d948b1f0abf63ef4729373fd8ee9634b Mon Sep 17 00:00:00 2001
From: Krish Moodbidri <krish94@uab.edu>
Date: Tue, 3 Dec 2024 10:32:25 -0600
Subject: [PATCH 02/29] compute node deploy
---
.gitlab-ci.yml | 36 ++++++++++++++++++++++++++++++++++++
1 file changed, 36 insertions(+)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5a3722a..65ebe23 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -160,6 +160,42 @@ build_ssh_proxy_image:
- if: $PIPELINE_TARGET == "build" && $BUILD_TARGET == "ssh-proxy"
when: always
+deploy_compute_node:
+ stage: deploy
+ environment:
+ name: $ENV
+ tags:
+ - build
+ script:
+ - openstack image set --accept $compute-${BUILD_DATE} || true
+ - FAILED=false
+ - |
+ cat > user_data.txt <<EOF
+ #!/bin/bash
+ cat >> /etc/NetworkManager/conf.d/90-dns-none.conf<<EEOF
+ [main]
+ dns=none
+ EEOF
+ systemctl reload NetworkManager
+ echo "$DEV_KEY" >> /root/.ssh/authorized_keys
+ ip route replace default via ${DEFAULT_GATEWAY_IP} dev eth0
+ git clone ${CI_REPOSITORY_URL} /tmp/${CI_PROJECT_NAME}
+ cd /tmp/${CI_PROJECT_NAME}
+ git checkout ${CI_COMMIT_REF_NAME}
+ cat >> ansible/hosts<<EEOF
+ [$ENV]
+ 127.0.0.1
+ EEOF
+ ansible-playbook -c local -i ansible/hosts --extra-vars="$EXTRA_VARS" ansible/cluster.yml | tee -a /tmp/ansible.log
+ rm -rf /tmp/${CI_PROJECT_NAME}
+ EOF
+ - |
+
+ rules:
+ - if: $PIPELINE_TARGET == "deploy"
+ when: always
+
+
deploy_http_proxy_node:
stage: deploy
environment:
--
GitLab
From 02d6e526594116e997f31be0de40c3b60e09d870 Mon Sep 17 00:00:00 2001
From: Krish Moodbidri <krish94@uab.edu>
Date: Tue, 3 Dec 2024 14:01:36 -0600
Subject: [PATCH 03/29] fixed yaml syntax for build and added rule block
---
.gitlab-ci.yml | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 65ebe23..3de3f00 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -132,6 +132,11 @@ build_compute_image:
- packer init openstack-compute
- packer validate openstack-compute
- packer build -machine-readable openstack-compute | tee compute_build.log
+ - |
+
+ rules:
+ - if: $PIPELINE_TARGET == "build"
+ when: always
build_http_proxy_image:
--
GitLab
From 048380ccb072ac9bcb9771377619e82e5dcc3af6 Mon Sep 17 00:00:00 2001
From: Krish Moodbidri <krish94@uab.edu>
Date: Wed, 4 Dec 2024 09:16:30 -0600
Subject: [PATCH 04/29] removed needs condition from build_compute_image
---
.gitlab-ci.yml | 1 -
1 file changed, 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3de3f00..05c0734 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -112,7 +112,6 @@ workflow:
build_compute_image:
stage: build
- needs: [build_base_image]
tags:
- build
script:
--
GitLab
From 986e692e5fd10c39a3662a4a8538806581170356 Mon Sep 17 00:00:00 2001
From: Krish Moodbidri <krish94@uab.edu>
Date: Wed, 4 Dec 2024 09:34:47 -0600
Subject: [PATCH 05/29] changed compute build structure
---
.gitlab-ci.yml | 12 +++---------
1 file changed, 3 insertions(+), 9 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 05c0734..f51049c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -117,21 +117,15 @@ build_compute_image:
script:
- *update_ansible_repo
- *get_ansible_files
- - export PKR_VAR_source_image=${BUILT_BASE_IMAGE_ID}
- - export REPO_HEAD=$(git rev-parse --short HEAD)
- export PKR_VAR_flavor="${COMPUTE_BUILD_FLAVOR:-$PKR_VAR_flavor}"
- - export PKR_VAR_build_instance_name="compute-${REPO_HEAD}"
+ - export PKR_VAR_build_instance_name="${BUILD_TARGET}-${COMPUTE_XCBC_HEAD}"
- export PKR_VAR_image_date_suffix=false
- |
if [ $CI_PIPELINE_SOURCE == 'merge_request_event' ]; then
- export PKR_VAR_image_name="compute-PR-${CI_MERGE_REQUEST_IID}"
+ export PKR_VAR_image_name="${BUILD_TARGET}-PR-${CI_MERGE_REQUEST_IID}"
elif [ $CI_PIPELINE_SOURCE == 'schedule' ]; then
- export PKR_VAR_image_name="compute-${BUILD_DATE}"
+ export PKR_VAR_image_name="${BUILD_TARGET}-${BUILD_DATE}"
fi
- - packer init openstack-compute
- - packer validate openstack-compute
- - packer build -machine-readable openstack-compute | tee compute_build.log
- - |
rules:
- if: $PIPELINE_TARGET == "build"
--
GitLab
From 835958f501117f976236795b884b633bacfa34b1 Mon Sep 17 00:00:00 2001
From: Krish Moodbidri <krish94@uab.edu>
Date: Wed, 4 Dec 2024 10:08:24 -0600
Subject: [PATCH 06/29] test compute build
---
.gitlab-ci.yml | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f51049c..b336b44 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -118,7 +118,7 @@ build_compute_image:
- *update_ansible_repo
- *get_ansible_files
- export PKR_VAR_flavor="${COMPUTE_BUILD_FLAVOR:-$PKR_VAR_flavor}"
- - export PKR_VAR_build_instance_name="${BUILD_TARGET}-${COMPUTE_XCBC_HEAD}"
+ - export PKR_VAR_build_instance_name="${BUILD_TARGET}-${CRI_XCBC_HEAD}"
- export PKR_VAR_image_date_suffix=false
- |
if [ $CI_PIPELINE_SOURCE == 'merge_request_event' ]; then
@@ -126,7 +126,13 @@ build_compute_image:
elif [ $CI_PIPELINE_SOURCE == 'schedule' ]; then
export PKR_VAR_image_name="${BUILD_TARGET}-${BUILD_DATE}"
fi
-
+ # packer commands
+ - packer init openstack-compute
+ - packer validate openstack-compute
+ - packer build -machine-readable openstack-compute | tee compute_build.log
+ - export BUILT_COMPUTE_IMAGE_ID=$(grep 'Image:' compute_build.log | awk '{print $4}')
+ - echo BUILT_COMPUTE_IMAGE_ID=${BUILT_COMPUTE_IMAGE_ID} | tee -a $CI_PROJECT_DIR/image.env
+ - openstack image set --property CRI_XCBC_prod=${CRI_XCBC_prod} --property CRI_XCBC_dev=${CRI_XCBC_dev} --property PACKER_IMAGE_HEAD=${PACKER_IMAGE_HEAD} ${BUILT_COMPUTE_IMAGE_ID}
rules:
- if: $PIPELINE_TARGET == "build"
when: always
--
GitLab
From c6cb2bcf5969967650a30c0489a6c2eed1ce5c60 Mon Sep 17 00:00:00 2001
From: Krish Moodbidri <krish94@uab.edu>
Date: Wed, 4 Dec 2024 12:58:51 -0600
Subject: [PATCH 07/29] changed compute built to be similar to proxy node
builds
---
.gitlab-ci.yml | 40 +++++++++++++++++++++++++++-------------
1 file changed, 27 insertions(+), 13 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b336b44..00d3c20 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -110,15 +110,13 @@ workflow:
reports:
dotenv: image.env
-build_compute_image:
- stage: build
- tags:
- - build
+.build_compute_image_template: &build_compute_image_template
script:
- *update_ansible_repo
- *get_ansible_files
- - export PKR_VAR_flavor="${COMPUTE_BUILD_FLAVOR:-$PKR_VAR_flavor}"
- - export PKR_VAR_build_instance_name="${BUILD_TARGET}-${CRI_XCBC_HEAD}"
+ # packer vars for job env
+ - export PKR_VAR_flavor="${PROXY_BUILD_FLAVOR:-$PKR_VAR_flavor}"
+ - export PKR_VAR_build_instance_name="${BUILD_TARGET}-${EXT_REPO_HEAD}"
- export PKR_VAR_image_date_suffix=false
- |
if [ $CI_PIPELINE_SOURCE == 'merge_request_event' ]; then
@@ -126,18 +124,34 @@ build_compute_image:
elif [ $CI_PIPELINE_SOURCE == 'schedule' ]; then
export PKR_VAR_image_name="${BUILD_TARGET}-${BUILD_DATE}"
fi
+ # Ansible var overrides
+ - |
# packer commands
- - packer init openstack-compute
- - packer validate openstack-compute
- - packer build -machine-readable openstack-compute | tee compute_build.log
- - export BUILT_COMPUTE_IMAGE_ID=$(grep 'Image:' compute_build.log | awk '{print $4}')
+ - packer init openstack-proxy
+ - packer validate openstack-proxy
+ - packer build -machine-readable openstack-proxy | tee proxy_build.log
+ - export BUILT_COMPUTE_IMAGE_ID=$(grep 'Image:' proxy_build.log | awk '{print $4}')
- echo BUILT_COMPUTE_IMAGE_ID=${BUILT_COMPUTE_IMAGE_ID} | tee -a $CI_PROJECT_DIR/image.env
- - openstack image set --property CRI_XCBC_prod=${CRI_XCBC_prod} --property CRI_XCBC_dev=${CRI_XCBC_dev} --property PACKER_IMAGE_HEAD=${PACKER_IMAGE_HEAD} ${BUILT_COMPUTE_IMAGE_ID}
+ # set image properties with repo state
+ - openstack image set --property EXT_PR_SRC_REPO=${EXT_PR_SRC_REPO} --property EXT_PR_SRC_BRANCH_SHA=${EXT_PR_SRC_BRANCH_SHA} --property EXT_PR_TARGET_REPO=${EXT_PR_TARGET_REPO} --property EXT_PR_TARGET_BRANCH_SHA=${EXT_PR_TARGET_BRANCH_SHA} --property PACKER_IMAGE_HEAD=${PACKER_IMAGE_HEAD} ${BUILT_COMPUTE_IMAGE_ID}
+ artifacts:
+ reports:
+ dotenv: image.env
+
+
+build_compute_image:
+ stage: build
+ environment:
+ name: $ENV
+ tags:
+ - build
+ variables:
+ PROXY_ENABLE_VAR: "enable_compute"
+ <<: *build_compute_image_template
rules:
- - if: $PIPELINE_TARGET == "build"
+ - if: $PIPELINE_TARGET == "build" && $BUILD_TARGET == "http-proxy"
when: always
-
build_http_proxy_image:
stage: build
environment:
--
GitLab
From 608fa35cdb99e099ea4eaf62ebdb807b436e5fce Mon Sep 17 00:00:00 2001
From: Krish Moodbidri <krish94@uab.edu>
Date: Wed, 4 Dec 2024 13:17:22 -0600
Subject: [PATCH 08/29] fixed packer commands for compute build
---
.gitlab-ci.yml | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 00d3c20..5e82c95 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -127,10 +127,10 @@ workflow:
# Ansible var overrides
- |
# packer commands
- - packer init openstack-proxy
- - packer validate openstack-proxy
- - packer build -machine-readable openstack-proxy | tee proxy_build.log
- - export BUILT_COMPUTE_IMAGE_ID=$(grep 'Image:' proxy_build.log | awk '{print $4}')
+ - packer init openstack-compute
+ - packer validate openstack-compute
+ - packer build -machine-readable openstack-compute | tee compute_build.log
+ - export BUILT_COMPUTE_IMAGE_ID=$(grep 'Image:' compute_build.log | awk '{print $4}')
- echo BUILT_COMPUTE_IMAGE_ID=${BUILT_COMPUTE_IMAGE_ID} | tee -a $CI_PROJECT_DIR/image.env
# set image properties with repo state
- openstack image set --property EXT_PR_SRC_REPO=${EXT_PR_SRC_REPO} --property EXT_PR_SRC_BRANCH_SHA=${EXT_PR_SRC_BRANCH_SHA} --property EXT_PR_TARGET_REPO=${EXT_PR_TARGET_REPO} --property EXT_PR_TARGET_BRANCH_SHA=${EXT_PR_TARGET_BRANCH_SHA} --property PACKER_IMAGE_HEAD=${PACKER_IMAGE_HEAD} ${BUILT_COMPUTE_IMAGE_ID}
--
GitLab
From 8e7b925258e8669103ce9dfa95a1636380042737 Mon Sep 17 00:00:00 2001
From: Krish Moodbidri <krish94@uab.edu>
Date: Wed, 4 Dec 2024 13:23:27 -0600
Subject: [PATCH 09/29] removed extra line
---
.gitlab-ci.yml | 1 -
1 file changed, 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5e82c95..3e770f0 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -138,7 +138,6 @@ workflow:
reports:
dotenv: image.env
-
build_compute_image:
stage: build
environment:
--
GitLab
From b860545f66de8c31504d95281c0a4d5081b64184 Mon Sep 17 00:00:00 2001
From: Krish Moodbidri <krish94@uab.edu>
Date: Wed, 4 Dec 2024 13:23:49 -0600
Subject: [PATCH 10/29] removed extra line
---
.gitlab-ci.yml | 1 -
1 file changed, 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3e770f0..5abd357 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -212,7 +212,6 @@ deploy_compute_node:
- if: $PIPELINE_TARGET == "deploy"
when: always
-
deploy_http_proxy_node:
stage: deploy
environment:
--
GitLab
From 1b9f779b8a90431f5850f38d43fdce2b92f54d19 Mon Sep 17 00:00:00 2001
From: Krish Moodbidri <krish94@uab.edu>
Date: Wed, 4 Dec 2024 13:38:22 -0600
Subject: [PATCH 11/29] removed extra line
---
.gitlab-ci.yml | 2 --
1 file changed, 2 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5abd357..ebbfeef 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -124,8 +124,6 @@ workflow:
elif [ $CI_PIPELINE_SOURCE == 'schedule' ]; then
export PKR_VAR_image_name="${BUILD_TARGET}-${BUILD_DATE}"
fi
- # Ansible var overrides
- - |
# packer commands
- packer init openstack-compute
- packer validate openstack-compute
--
GitLab
From 805b49b69880b60b008adeddf0a21ab18ad2d8ec Mon Sep 17 00:00:00 2001
From: Krish Moodbidri <krish94@uab.edu>
Date: Wed, 4 Dec 2024 13:41:44 -0600
Subject: [PATCH 12/29] added root_ssh_key for compute node build variable
---
openstack-compute/variables.pkr.hcl | 1 +
1 file changed, 1 insertion(+)
diff --git a/openstack-compute/variables.pkr.hcl b/openstack-compute/variables.pkr.hcl
index 20efd64..d368139 100644
--- a/openstack-compute/variables.pkr.hcl
+++ b/openstack-compute/variables.pkr.hcl
@@ -1,6 +1,7 @@
variable "root_ssh_key" {
type = string
description = "The root key to use for ssh"
+ default = ""
}
variable "image_name" {
--
GitLab
From b4db1b33485c15eb156172e0efbe407e150ab8b0 Mon Sep 17 00:00:00 2001
From: Krish Moodbidri <krish94@uab.edu>
Date: Wed, 4 Dec 2024 13:47:57 -0600
Subject: [PATCH 13/29] set build target to compute
---
.gitlab-ci.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ebbfeef..af25bc5 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -146,7 +146,7 @@ build_compute_image:
PROXY_ENABLE_VAR: "enable_compute"
<<: *build_compute_image_template
rules:
- - if: $PIPELINE_TARGET == "build" && $BUILD_TARGET == "http-proxy"
+ - if: $PIPELINE_TARGET == "build" && $BUILD_TARGET == "compute"
when: always
build_http_proxy_image:
--
GitLab
From feee0a731dbfeb95d5b0035f8602099a67390044 Mon Sep 17 00:00:00 2001
From: Krish Moodbidri <krish94@uab.edu>
Date: Wed, 4 Dec 2024 14:07:13 -0600
Subject: [PATCH 14/29] changed network id
---
.gitlab-ci.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index af25bc5..63e179b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -20,7 +20,7 @@ variables:
PKR_VAR_security_groups: '["allow-ssh"]'
PKR_VAR_skip_create_image: "false"
PKR_VAR_ssh_username: "centos"
- PKR_VAR_networks: '["8cf2f12e-905d-46d9-bc70-b0897c65f75a"]'
+ PKR_VAR_networks: '["0295b9ca-ba14-4d08-8c57-78a4c07345eb"]'
GIT_AUTHOR_NAME: "Gitlab runner"
GIT_AUTHOR_EMAIL: "gitlab@runner"
INSTANCE_FLAVOR: "m1.medium-ruffner"
--
GitLab
From b9ee4761387d05965a21f7e928d422d4d03f0f9a Mon Sep 17 00:00:00 2001
From: Krish Moodbidri <krish94@uab.edu>
Date: Wed, 4 Dec 2024 14:22:52 -0600
Subject: [PATCH 15/29] chnaged image set for compute to match that of proxy
---
.gitlab-ci.yml | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 63e179b..d821f5d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -130,8 +130,7 @@ workflow:
- packer build -machine-readable openstack-compute | tee compute_build.log
- export BUILT_COMPUTE_IMAGE_ID=$(grep 'Image:' compute_build.log | awk '{print $4}')
- echo BUILT_COMPUTE_IMAGE_ID=${BUILT_COMPUTE_IMAGE_ID} | tee -a $CI_PROJECT_DIR/image.env
- # set image properties with repo state
- - openstack image set --property EXT_PR_SRC_REPO=${EXT_PR_SRC_REPO} --property EXT_PR_SRC_BRANCH_SHA=${EXT_PR_SRC_BRANCH_SHA} --property EXT_PR_TARGET_REPO=${EXT_PR_TARGET_REPO} --property EXT_PR_TARGET_BRANCH_SHA=${EXT_PR_TARGET_BRANCH_SHA} --property PACKER_IMAGE_HEAD=${PACKER_IMAGE_HEAD} ${BUILT_COMPUTE_IMAGE_ID}
+ - openstack image set --property CRI_XCBC_prod=${CRI_XCBC_prod} --property CRI_XCBC_dev=${CRI_XCBC_dev} --property PACKER_IMAGE_HEAD=${PACKER_IMAGE_HEAD} ${BUILT_PROXY_IMAGE_ID}
artifacts:
reports:
dotenv: image.env
--
GitLab
From 4fad8aec334f5c4d0f94256396e1f84f5ab16966 Mon Sep 17 00:00:00 2001
From: Krish Moodbidri <krish94@uab.edu>
Date: Wed, 4 Dec 2024 14:27:34 -0600
Subject: [PATCH 16/29] fixed typo in BUILT_COMPUTE_IMAGE_ID
---
.gitlab-ci.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d821f5d..8703f6d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -130,7 +130,7 @@ workflow:
- packer build -machine-readable openstack-compute | tee compute_build.log
- export BUILT_COMPUTE_IMAGE_ID=$(grep 'Image:' compute_build.log | awk '{print $4}')
- echo BUILT_COMPUTE_IMAGE_ID=${BUILT_COMPUTE_IMAGE_ID} | tee -a $CI_PROJECT_DIR/image.env
- - openstack image set --property CRI_XCBC_prod=${CRI_XCBC_prod} --property CRI_XCBC_dev=${CRI_XCBC_dev} --property PACKER_IMAGE_HEAD=${PACKER_IMAGE_HEAD} ${BUILT_PROXY_IMAGE_ID}
+ - openstack image set --property CRI_XCBC_prod=${CRI_XCBC_prod} --property CRI_XCBC_dev=${CRI_XCBC_dev} --property PACKER_IMAGE_HEAD=${PACKER_IMAGE_HEAD} ${BUILT_COMPUTE_IMAGE_ID}
artifacts:
reports:
dotenv: image.env
--
GitLab
From 21068773d6f932900526301702ce2452837d7569 Mon Sep 17 00:00:00 2001
From: Krish Moodbidri <krish94@uab.edu>
Date: Wed, 4 Dec 2024 14:39:02 -0600
Subject: [PATCH 17/29] split playbook compute.ymp to build and deploy
---
ansible/compute.yml | 2 --
1 file changed, 2 deletions(-)
diff --git a/ansible/compute.yml b/ansible/compute.yml
index 2907d08..4009929 100644
--- a/ansible/compute.yml
+++ b/ansible/compute.yml
@@ -8,5 +8,3 @@
- { name: 'pam_slurm_adopt', tags: 'pam_slurm_adopt' }
- { name: 'install_nhc', tags: 'install_nhc'}
-- name: Setup node for use as a virtual cheaha node
- ansible.builtin.import_playbook: cheaha.yml
--
GitLab
From 9a868453ea3907dfdaca2d6e604439985514cab0 Mon Sep 17 00:00:00 2001
From: Krish Moodbidri <krish94@uab.edu>
Date: Wed, 4 Dec 2024 14:53:03 -0600
Subject: [PATCH 18/29] added playbook file base.yml in compute node build
---
openstack-compute/nodeimage.pkr.hcl | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/openstack-compute/nodeimage.pkr.hcl b/openstack-compute/nodeimage.pkr.hcl
index 15941ba..0df7158 100644
--- a/openstack-compute/nodeimage.pkr.hcl
+++ b/openstack-compute/nodeimage.pkr.hcl
@@ -36,6 +36,17 @@ source "openstack" "image" {
build {
sources = ["source.openstack.image"]
+ provisioner "ansible" {
+ use_proxy = false
+ user = var.ssh_username
+ groups = ["base"]
+ playbook_file = "./ansible/base.yml"
+ roles_path = "./ansible/roles"
+ extra_arguments = [
+ "--extra-vars", "root_ssh_key='${var.root_ssh_key}'"
+ ]
+ }
+
provisioner "ansible" {
use_proxy = false
user = var.ssh_username
--
GitLab
From 2adc41dfa81c7b9d080ca15b4f2e532196366106 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Fri, 6 Dec 2024 19:01:35 -0500
Subject: [PATCH 19/29] feat: Remove unnecessary var def
---
.gitlab-ci.yml | 2 --
1 file changed, 2 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8703f6d..edb70c9 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -167,8 +167,6 @@ build_ssh_proxy_image:
name: $ENV
tags:
- build
- variables:
- PROXY_ENABLE_VAR: "enable_ssh_proxy"
<<: *build_proxy_image_template
rules:
- if: $PIPELINE_TARGET == "build" && $BUILD_TARGET == "ssh-proxy"
--
GitLab
From 1987ab3e3a917a4015b414d6212511a7a52cdc86 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Fri, 6 Dec 2024 19:03:52 -0500
Subject: [PATCH 20/29] feat: Rearrange code for readability
---
.gitlab-ci.yml | 119 +++++++++++++++++++++++++------------------------
1 file changed, 60 insertions(+), 59 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index edb70c9..340976d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -110,6 +110,30 @@ workflow:
reports:
dotenv: image.env
+build_http_proxy_image:
+ stage: build
+ environment:
+ name: $ENV
+ tags:
+ - build
+ variables:
+ PROXY_ENABLE_VAR: "enable_http_proxy"
+ <<: *build_proxy_image_template
+ rules:
+ - if: $PIPELINE_TARGET == "build" && $BUILD_TARGET == "http-proxy"
+ when: always
+
+build_ssh_proxy_image:
+ stage: build
+ environment:
+ name: $ENV
+ tags:
+ - build
+ <<: *build_proxy_image_template
+ rules:
+ - if: $PIPELINE_TARGET == "build" && $BUILD_TARGET == "ssh-proxy"
+ when: always
+
.build_compute_image_template: &build_compute_image_template
script:
- *update_ansible_repo
@@ -148,65 +172,6 @@ build_compute_image:
- if: $PIPELINE_TARGET == "build" && $BUILD_TARGET == "compute"
when: always
-build_http_proxy_image:
- stage: build
- environment:
- name: $ENV
- tags:
- - build
- variables:
- PROXY_ENABLE_VAR: "enable_http_proxy"
- <<: *build_proxy_image_template
- rules:
- - if: $PIPELINE_TARGET == "build" && $BUILD_TARGET == "http-proxy"
- when: always
-
-build_ssh_proxy_image:
- stage: build
- environment:
- name: $ENV
- tags:
- - build
- <<: *build_proxy_image_template
- rules:
- - if: $PIPELINE_TARGET == "build" && $BUILD_TARGET == "ssh-proxy"
- when: always
-
-deploy_compute_node:
- stage: deploy
- environment:
- name: $ENV
- tags:
- - build
- script:
- - openstack image set --accept $compute-${BUILD_DATE} || true
- - FAILED=false
- - |
- cat > user_data.txt <<EOF
- #!/bin/bash
- cat >> /etc/NetworkManager/conf.d/90-dns-none.conf<<EEOF
- [main]
- dns=none
- EEOF
- systemctl reload NetworkManager
- echo "$DEV_KEY" >> /root/.ssh/authorized_keys
- ip route replace default via ${DEFAULT_GATEWAY_IP} dev eth0
- git clone ${CI_REPOSITORY_URL} /tmp/${CI_PROJECT_NAME}
- cd /tmp/${CI_PROJECT_NAME}
- git checkout ${CI_COMMIT_REF_NAME}
- cat >> ansible/hosts<<EEOF
- [$ENV]
- 127.0.0.1
- EEOF
- ansible-playbook -c local -i ansible/hosts --extra-vars="$EXTRA_VARS" ansible/cluster.yml | tee -a /tmp/ansible.log
- rm -rf /tmp/${CI_PROJECT_NAME}
- EOF
- - |
-
- rules:
- - if: $PIPELINE_TARGET == "deploy"
- when: always
-
deploy_http_proxy_node:
stage: deploy
environment:
@@ -307,3 +272,39 @@ deploy_ssh_proxy_node:
rules:
- if: $PIPELINE_TARGET == "deploy" && $SSH_PROXY_IMAGE_ID
when: always
+
+deploy_compute_node:
+ stage: deploy
+ environment:
+ name: $ENV
+ tags:
+ - build
+ script:
+ - openstack image set --accept $compute-${BUILD_DATE} || true
+ - FAILED=false
+ - |
+ cat > user_data.txt <<EOF
+ #!/bin/bash
+ cat >> /etc/NetworkManager/conf.d/90-dns-none.conf<<EEOF
+ [main]
+ dns=none
+ EEOF
+ systemctl reload NetworkManager
+ echo "$DEV_KEY" >> /root/.ssh/authorized_keys
+ ip route replace default via ${DEFAULT_GATEWAY_IP} dev eth0
+ git clone ${CI_REPOSITORY_URL} /tmp/${CI_PROJECT_NAME}
+ cd /tmp/${CI_PROJECT_NAME}
+ git checkout ${CI_COMMIT_REF_NAME}
+ cat >> ansible/hosts<<EEOF
+ [$ENV]
+ 127.0.0.1
+ EEOF
+ ansible-playbook -c local -i ansible/hosts --extra-vars="$EXTRA_VARS" ansible/cluster.yml | tee -a /tmp/ansible.log
+ rm -rf /tmp/${CI_PROJECT_NAME}
+ EOF
+ - |
+
+ rules:
+ - if: $PIPELINE_TARGET == "deploy"
+ when: always
+
--
GitLab
From 66b2959f8de92193e0bd8e816fd4f11eb301c545 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Tue, 10 Dec 2024 16:18:06 -0500
Subject: [PATCH 21/29] Change compute reference to login in build
---
.gitlab-ci.yml | 15 +++++++--------
1 file changed, 7 insertions(+), 8 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 340976d..34fe32f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -134,7 +134,7 @@ build_ssh_proxy_image:
- if: $PIPELINE_TARGET == "build" && $BUILD_TARGET == "ssh-proxy"
when: always
-.build_compute_image_template: &build_compute_image_template
+.build_login_image_template: &build_login_image_template
script:
- *update_ansible_repo
- *get_ansible_files
@@ -151,15 +151,14 @@ build_ssh_proxy_image:
# packer commands
- packer init openstack-compute
- packer validate openstack-compute
- - packer build -machine-readable openstack-compute | tee compute_build.log
- - export BUILT_COMPUTE_IMAGE_ID=$(grep 'Image:' compute_build.log | awk '{print $4}')
- - echo BUILT_COMPUTE_IMAGE_ID=${BUILT_COMPUTE_IMAGE_ID} | tee -a $CI_PROJECT_DIR/image.env
- - openstack image set --property CRI_XCBC_prod=${CRI_XCBC_prod} --property CRI_XCBC_dev=${CRI_XCBC_dev} --property PACKER_IMAGE_HEAD=${PACKER_IMAGE_HEAD} ${BUILT_COMPUTE_IMAGE_ID}
+ - packer build -machine-readable openstack-compute | tee login_build.log
+ - export BUILT_LOGIN_IMAGE_ID=$(grep 'Image:' login_build.log | awk '{print $4}')
+ - echo BUILT_LOGIN_IMAGE_ID=${BUILT_LOGIN_IMAGE_ID} | tee -a $CI_PROJECT_DIR/image.env
artifacts:
reports:
dotenv: image.env
-build_compute_image:
+build_login_image:
stage: build
environment:
name: $ENV
@@ -167,9 +166,9 @@ build_compute_image:
- build
variables:
PROXY_ENABLE_VAR: "enable_compute"
- <<: *build_compute_image_template
+ <<: *build_login_image_template
rules:
- - if: $PIPELINE_TARGET == "build" && $BUILD_TARGET == "compute"
+ - if: $PIPELINE_TARGET == "build" && $BUILD_TARGET == "login"
when: always
deploy_http_proxy_node:
--
GitLab
From 35b6013d4afee084c1fd006c9825ad87ed3daddf Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Tue, 10 Dec 2024 16:19:17 -0500
Subject: [PATCH 22/29] Stick to template used in other builds for image
properties
---
.gitlab-ci.yml | 2 ++
1 file changed, 2 insertions(+)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 34fe32f..59a7976 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -154,6 +154,8 @@ build_ssh_proxy_image:
- packer build -machine-readable openstack-compute | tee login_build.log
- export BUILT_LOGIN_IMAGE_ID=$(grep 'Image:' login_build.log | awk '{print $4}')
- echo BUILT_LOGIN_IMAGE_ID=${BUILT_LOGIN_IMAGE_ID} | tee -a $CI_PROJECT_DIR/image.env
+ # set image properties with repo state
+ - openstack image set --property EXT_PR_SRC_REPO=${EXT_PR_SRC_REPO} --property EXT_PR_SRC_BRANCH_SHA=${EXT_PR_SRC_BRANCH_SHA} --property EXT_PR_TARGET_REPO=${EXT_PR_TARGET_REPO} --property EXT_PR_TARGET_BRANCH_SHA=${EXT_PR_TARGET_BRANCH_SHA} --property PACKER_IMAGE_HEAD=${CI_COMMIT_SHORT_SHA} ${BUILT_LOGIN_IMAGE_ID}
artifacts:
reports:
dotenv: image.env
--
GitLab
From 3012ed8f31e6a6cef9113599f121e738575c3b09 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Tue, 10 Dec 2024 16:19:47 -0500
Subject: [PATCH 23/29] Remove unncessary vars
---
.gitlab-ci.yml | 2 --
1 file changed, 2 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 59a7976..e0d114d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -166,8 +166,6 @@ build_login_image:
name: $ENV
tags:
- build
- variables:
- PROXY_ENABLE_VAR: "enable_compute"
<<: *build_login_image_template
rules:
- if: $PIPELINE_TARGET == "build" && $BUILD_TARGET == "login"
--
GitLab
From a8a1da95cc7f3a83572fc584b2b9986dbc6b3b20 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Tue, 10 Dec 2024 16:37:04 -0500
Subject: [PATCH 24/29] Change compute references to login in deploy
---
.gitlab-ci.yml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e0d114d..ffa84d3 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -272,14 +272,14 @@ deploy_ssh_proxy_node:
- if: $PIPELINE_TARGET == "deploy" && $SSH_PROXY_IMAGE_ID
when: always
-deploy_compute_node:
+deploy_login_node:
stage: deploy
environment:
name: $ENV
tags:
- build
script:
- - openstack image set --accept $compute-${BUILD_DATE} || true
+ - openstack image set --accept $login-${BUILD_DATE} || true
- FAILED=false
- |
cat > user_data.txt <<EOF
--
GitLab
From 73748a6cbb0a205e220f71e9bcc01cef228cbd6a Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Tue, 10 Dec 2024 16:39:41 -0500
Subject: [PATCH 25/29] Add missing code to create instance and associate FIP
---
.gitlab-ci.yml | 19 +++++++++++++++++--
1 file changed, 17 insertions(+), 2 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ffa84d3..5b15e07 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -302,8 +302,23 @@ deploy_login_node:
rm -rf /tmp/${CI_PROJECT_NAME}
EOF
- |
-
+ export cmd="openstack server create"
+ cmd+=" -c id -f value --image $LOGIN_IMAGE_ID"
+ cmd+=" --flavor $INSTANCE_FLAVOR"
+ cmd+=" --network $INSTANCE_NETWORK"
+ cmd+=" --security-group allow-ssh"
+ cmd+=" --user-data user_data.txt"
+ if [ -n "$LOGIN_PORT" ];then cmd+=" --port $LOGIN_PORT"; fi
+ cmd+=" --wait $LOGIN_INSTANCE_NAME"
+ - export LOGIN_INSTANCE_ID=$(bash -c "$cmd")
+ - |
+ # Associate the floating IP(s) with the SSH Proxy instance
+ for LOGIN_FLOATING_IP in ${LOGIN_FLOATING_IP_LIST[@]};
+ do
+ echo "Associating FLOATING_IP $LOGIN_FLOATING_IP with LOGIN_INSTANCE_ID $LOGIN_INSTANCE_ID"
+ openstack server add floating ip $LOGIN_INSTANCE_ID $LOGIN_FLOATING_IP
+ done
rules:
- - if: $PIPELINE_TARGET == "deploy"
+ - if: $PIPELINE_TARGET == "deploy" && $LOGIN_IMAGE_ID
when: always
--
GitLab
From ac15dd91da9ff920cc2e917ee55bafc34aedd1da Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Wed, 11 Dec 2024 08:25:41 -0500
Subject: [PATCH 26/29] Revert "changed network id"
This reverts commit 7cd67e96ad6b6f9f9b7f12f4776678dfa04ec8d3.
---
.gitlab-ci.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5b15e07..43b39ae 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -20,7 +20,7 @@ variables:
PKR_VAR_security_groups: '["allow-ssh"]'
PKR_VAR_skip_create_image: "false"
PKR_VAR_ssh_username: "centos"
- PKR_VAR_networks: '["0295b9ca-ba14-4d08-8c57-78a4c07345eb"]'
+ PKR_VAR_networks: '["8cf2f12e-905d-46d9-bc70-b0897c65f75a"]'
GIT_AUTHOR_NAME: "Gitlab runner"
GIT_AUTHOR_EMAIL: "gitlab@runner"
INSTANCE_FLAVOR: "m1.medium-ruffner"
--
GitLab
From 111da23fcb2ff4e2897f660048b081d3f3054f71 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Wed, 11 Dec 2024 08:30:02 -0500
Subject: [PATCH 27/29] Avoid building base instead use base img as src img
This will save time during image build. If you need base image build it
separtely and use the new image as src for build.
---
openstack-compute/nodeimage.pkr.hcl | 11 -----------
1 file changed, 11 deletions(-)
diff --git a/openstack-compute/nodeimage.pkr.hcl b/openstack-compute/nodeimage.pkr.hcl
index 0df7158..15941ba 100644
--- a/openstack-compute/nodeimage.pkr.hcl
+++ b/openstack-compute/nodeimage.pkr.hcl
@@ -36,17 +36,6 @@ source "openstack" "image" {
build {
sources = ["source.openstack.image"]
- provisioner "ansible" {
- use_proxy = false
- user = var.ssh_username
- groups = ["base"]
- playbook_file = "./ansible/base.yml"
- roles_path = "./ansible/roles"
- extra_arguments = [
- "--extra-vars", "root_ssh_key='${var.root_ssh_key}'"
- ]
- }
-
provisioner "ansible" {
use_proxy = false
user = var.ssh_username
--
GitLab
From 7d1d9820ede05ed37caacbcbf1d9fa0be7d7cba3 Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Wed, 11 Dec 2024 08:52:54 -0500
Subject: [PATCH 28/29] Rename packer template for compute to login
Because compute and login nodes are basically the same except login node
does not need slurmd and we don't use compute image anywhere. So use it
to build login image and change its references in CI script
---
.gitlab-ci.yml | 6 +++---
{openstack-compute => openstack-login}/README.md | 0
{openstack-compute => openstack-login}/nodeimage.pkr.hcl | 0
{openstack-compute => openstack-login}/variables.pkr.hcl | 0
4 files changed, 3 insertions(+), 3 deletions(-)
rename {openstack-compute => openstack-login}/README.md (100%)
rename {openstack-compute => openstack-login}/nodeimage.pkr.hcl (100%)
rename {openstack-compute => openstack-login}/variables.pkr.hcl (100%)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 43b39ae..29ec2e1 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -149,9 +149,9 @@ build_ssh_proxy_image:
export PKR_VAR_image_name="${BUILD_TARGET}-${BUILD_DATE}"
fi
# packer commands
- - packer init openstack-compute
- - packer validate openstack-compute
- - packer build -machine-readable openstack-compute | tee login_build.log
+ - packer init openstack-login
+ - packer validate openstack-login
+ - packer build -machine-readable openstack-login | tee login_build.log
- export BUILT_LOGIN_IMAGE_ID=$(grep 'Image:' login_build.log | awk '{print $4}')
- echo BUILT_LOGIN_IMAGE_ID=${BUILT_LOGIN_IMAGE_ID} | tee -a $CI_PROJECT_DIR/image.env
# set image properties with repo state
diff --git a/openstack-compute/README.md b/openstack-login/README.md
similarity index 100%
rename from openstack-compute/README.md
rename to openstack-login/README.md
diff --git a/openstack-compute/nodeimage.pkr.hcl b/openstack-login/nodeimage.pkr.hcl
similarity index 100%
rename from openstack-compute/nodeimage.pkr.hcl
rename to openstack-login/nodeimage.pkr.hcl
diff --git a/openstack-compute/variables.pkr.hcl b/openstack-login/variables.pkr.hcl
similarity index 100%
rename from openstack-compute/variables.pkr.hcl
rename to openstack-login/variables.pkr.hcl
--
GitLab
From 16d9eead335fb07fc7de3368d8cbbaa314e8a4bf Mon Sep 17 00:00:00 2001
From: Eesaan Atluri <atlurie@uab.edu>
Date: Wed, 11 Dec 2024 12:55:50 -0500
Subject: [PATCH 29/29] Install prerequisite packages
---
openstack-login/nodeimage.pkr.hcl | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/openstack-login/nodeimage.pkr.hcl b/openstack-login/nodeimage.pkr.hcl
index 15941ba..008b768 100644
--- a/openstack-login/nodeimage.pkr.hcl
+++ b/openstack-login/nodeimage.pkr.hcl
@@ -36,6 +36,14 @@ source "openstack" "image" {
build {
sources = ["source.openstack.image"]
+ provisioner "shell" {
+ inline = [
+ "sudo yum install -y libselinux-python3 python3 python3-pip tmux vim git bash-completion curl wget unzip",
+ "sudo python3 -m pip install --upgrade pip",
+ "sudo pip3 install s3cmd==2.3.0 ansible==4.10.0 python-openstackclient==5.8.0"
+ ]
+ }
+
provisioner "ansible" {
use_proxy = false
user = var.ssh_username
--
GitLab