Feat cleanup job

Move the servers and images deletion steps to a separate job

Closes #105 (closed) #111 (closed)

.gitlab-ci.yml

@@ -26,6 +26,7 @@ variables:
   GIT_AUTHOR_EMAIL: "gitlab@runner"
   NUM_SERVER_TO_KEEP: 1
   NUM_IMAGE_TO_KEEP: 30
+  TIMESTAMP_REGEXP: '[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{6}'
   PKR_VAR_root_ssh_key: "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAFqqWgmYpEaGtHBeTu27ntVJpYjwq/x5aBefrvfhk8Z9lE3cuZ26vJ9n/9tGE4Zn2Pew1mpZgi6PzfJ3vMt8yA= root@master"
   DEV_KEY: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCpncAcYosVHt7HsUcE2XOYDuCi4HQnmFJv279LOcpZgXtZ6o0BM1fe5FgJS0X1ohBXQUFRuYJuJSW/GSmC1K8T+wCrKjZLJdMbqrubHV27diUZfdoVkoJy1vcAQF5nEcoTC7MpAFbBomdn2rsrpgQe8DGiURV7+soqybXV1OsIR3FFf6npnUaskHYT/oVtG9eBOnscyBxoVgbxzlmyoBLXED/sHKFw4nQSF/glYKEFiDu6TRTsBBEGvv23Qo/66QpQiFJ6TNfApNiyY9L1X+Dy8EWU6lozmNgwGDjXQ70Lr6xHnA0QGVALJlHXa6QjpgtpC5Nefsdvtf1hpfFo2VutpbSB+aq9jk3gWNN+XkhrWN5PiwP7YYJNw/WozyfL+IhwjfHZGxkuws+wGR6ZKxlX9W9Vrsq9ncYNKuhy2SdsR6s2XECQtrEQ6ZlX5jRt6Yh5M9ls5fMsWEqknDPmr1Ui6wV7NxprYngo9fLSdYO/ETIO3S6PB0aEHOZOyGitGaM06EmNpvjQn/QkkaVgt/O8wKL1o1AVzXhDMAFvtG6ejppV6kuTUHXFgSGZF6N9fnP91HuytyzC09F+NMWcmnRdrgXlHapjuuL3zzi+XLCQvk8+aYTzBKx1nU2FPMDRZ9sInGmqdTuM002E7qVbaCy4OxcWaAS/L2UVhGnHr+egYw== louistw@uab.edu"
 
@@ -34,6 +35,7 @@ stages:
   - build
   - test
   - deploy
+  - cleanup
 
 workflow:
   rules:
@@ -361,15 +363,6 @@ deploy_knightly:
         openstack server add floating ip $NEW_INSTANCE_ID $CAMPUS_IP
         openstack server add floating ip $NEW_INSTANCE_ID $CHEAHA_IP
       fi
-    - |
-      SERVER_TO_BE_DELETE=($(openstack server list --name $OOD_INSTANCE_NAME --sort-column Image --sort-descending -f value -c ID | sed -n $(($NUM_SERVER_TO_KEEP+1))',$p'))
-      IMAGE_TO_BE_DELETE=($(openstack image list --sort-column Name --sort-descending -f value -c Name -c ID | grep -P ' ood-\d{8}$' | sed -n $(($NUM_IMAGE_TO_KEEP+1))',$p' | awk '{print $1}'))
-      for svr in ${SERVER_TO_BE_DELETE[@]}; do
-        openstack server delete ${svr}
-      done
-      for img in ${IMAGE_TO_BE_DELETE[@]}; do
-        openstack image delete ${img}
-      done
   only:
     - schedules
 
@@ -384,3 +377,75 @@ deploy_cheaha:
   when: manual
   only:
     - main
+
+cleanup_knightly:
+  stage: cleanup
+  environment:
+    name: knightly
+  tags:
+    - build
+  script:
+    - >
+      SERVER_TO_BE_DELETE=($(openstack server list --name $OOD_INSTANCE_NAME --sort-column Image --sort-descending -f value -c ID
+      | awk -v NSTK=$NUM_SERVER_TO_KEEP '{count++}
+      {if (count>NSTK) print}'))
+    - openstack image list --sort-column Name --sort-descending -f value -c Name -c ID > images.txt
+    - >
+      OOD_IMAGE_TO_BE_DELETE=($(cat images.txt
+      | awk -v NITK=$NUM_IMAGE_TO_KEEP -v REGEX=ood-$TIMESTAMP_REGEX
+      '{if ($0 ~ REGEX) result[count++] = $1}
+      END {for(i=NITK;i<count;i++) print result[i]}'))
+    - >
+      BASE_IMAGE_TO_BE_DELETE=($(cat images.txt
+      | awk -v NITK=$NUM_IMAGE_TO_KEEP -v REGEX=base-$TIMESTAMP_REGEX
+      '{if ($0 ~ REGEX) result[count++] = $1}
+      END {for(i=NITK;i<count;i++) print result[i]}'))
+    - >
+      COMPUTE_IMAGE_TO_BE_DELETE=($(cat images.txt
+      | awk -v NITK=$NUM_IMAGE_TO_KEEP -v REGEX=compute-$TIMESTAMP_REGEX
+      '{if ($0 ~ REGEX) result[count++] = $1}
+      END {for(i=NITK;i<count;i++) print result[i]}'))
+    - >
+      GPU_IMAGE_TO_BE_DELETE=($(cat images.txt
+      | awk -v NITK=$NUM_IMAGE_TO_KEEP -v REGEX=gpu-$TIMESTAMP_REGEX
+      '{if ($0 ~ REGEX) result[count++] = $1}
+      END {for(i=NITK;i<count;i++) print result[i]}'))
+    - |
+      for svr in ${SERVER_TO_BE_DELETE[@]}; do
+        openstack server delete ${svr}
+      done
+    - |
+      for img in ${OOD_IMAGE_TO_BE_DELETE[@]}; do
+        openstack image delete ${img}
+      done
+    - |
+      for img in ${BASE_IMAGE_TO_BE_DELETE[@]}; do
+        openstack image delete ${img}
+      done
+    - |
+      for img in ${COMPUTE_IMAGE_TO_BE_DELETE[@]}; do
+        openstack image delete ${img}
+      done
+    - |
+      for img in ${GPU_IMAGE_TO_BE_DELETE[@]}; do
+        openstack image delete ${img}
+      done
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+      when: always
+
+cleanup_mr:
+  stage: cleanup
+  tags:
+    - build
+  script:
+    - >
+      IMAGE_TO_BE_DELETE=($(openstack image list --sort-column Name --sort-descending -f value -c Name -c ID
+      | awk -v REGEX="(ood|base|compute|gpu)-PR-$CI_MERGE_REQUEST_IID" '{if ($0 ~ REGEX) print $1}'))
+    - |
+      for img in ${IMAGE_TO_BE_DELETE[@]}; do
+        openstack image delete ${img}
+      done
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      when: always