diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4bd9f8019320029a11172a0f31fe3e609694c39a..e5089c4196e97327899844acaf60fb1afd4d743e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,34 +2,19 @@ default: image: $CI_REGISTRY_IMAGE:latest variables: - CAMPUS_IP: 138.26.48.47 - CHEAHA_IP: 172.20.10.9 - TEST_IP: 138.26.49.134 - ANSIBLE_REMOTE_TMP: "/tmp" - AWS_DEFAULT_REGION: "bhm" - AWS_HOST: "s3.lts.rc.uab.edu" - FF_SCRIPT_SECTIONS: "true" - OS_AUTH_TYPE: "v3applicationcredential" - OS_AUTH_URL: "https://keystone.cloud.rc.uab.edu:5000/v3" - OS_IDENTITY_API_VERSION: "3" - OS_INTERFACE: "public" - OS_REGION_NAME: "bhm1" - OOD_INSTANCE_NETWORK: "knightly-network" - PKR_VAR_flavor: "m1.medium-ruffner" - PKR_VAR_source_image: "CentOS-7-x86_64-GenericCloud-2009" - PKR_VAR_floating_ip_network: "uab-campus" - PKR_VAR_security_groups: '["allow-ssh"]' - PKR_VAR_skip_create_image: "false" - PKR_VAR_ssh_username: "centos" - PKR_VAR_networks: '["8cf2f12e-905d-46d9-bc70-b0897c65f75a"]' - PKR_VAR_image_membership: '["cf6fa1e53d4c40a49f4e0e469c440359"]' - GIT_AUTHOR_NAME: "Gitlab runner" - GIT_AUTHOR_EMAIL: "gitlab@runner" - NUM_SERVER_TO_KEEP: 1 - NUM_IMAGE_TO_KEEP: 30 - TIMESTAMP_REGEXP: '[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{6}' - PKR_VAR_root_ssh_key: "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAFqqWgmYpEaGtHBeTu27ntVJpYjwq/x5aBefrvfhk8Z9lE3cuZ26vJ9n/9tGE4Zn2Pew1mpZgi6PzfJ3vMt8yA= root@master" - DEV_KEY: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCpncAcYosVHt7HsUcE2XOYDuCi4HQnmFJv279LOcpZgXtZ6o0BM1fe5FgJS0X1ohBXQUFRuYJuJSW/GSmC1K8T+wCrKjZLJdMbqrubHV27diUZfdoVkoJy1vcAQF5nEcoTC7MpAFbBomdn2rsrpgQe8DGiURV7+soqybXV1OsIR3FFf6npnUaskHYT/oVtG9eBOnscyBxoVgbxzlmyoBLXED/sHKFw4nQSF/glYKEFiDu6TRTsBBEGvv23Qo/66QpQiFJ6TNfApNiyY9L1X+Dy8EWU6lozmNgwGDjXQ70Lr6xHnA0QGVALJlHXa6QjpgtpC5Nefsdvtf1hpfFo2VutpbSB+aq9jk3gWNN+XkhrWN5PiwP7YYJNw/WozyfL+IhwjfHZGxkuws+wGR6ZKxlX9W9Vrsq9ncYNKuhy2SdsR6s2XECQtrEQ6ZlX5jRt6Yh5M9ls5fMsWEqknDPmr1Ui6wV7NxprYngo9fLSdYO/ETIO3S6PB0aEHOZOyGitGaM06EmNpvjQn/QkkaVgt/O8wKL1o1AVzXhDMAFvtG6ejppV6kuTUHXFgSGZF6N9fnP91HuytyzC09F+NMWcmnRdrgXlHapjuuL3zzi+XLCQvk8+aYTzBKx1nU2FPMDRZ9sInGmqdTuM002E7qVbaCy4OxcWaAS/L2UVhGnHr+egYw== louistw@uab.edu" + DOCKER_DRIVER: overlay2 + BUILD_DATE: $CI_COMMIT_TIMESTAMP + BASE_BUILD_FLAVOR: "standard" + COMPUTE_BUILD_FLAVOR: "compute.large" + GPU_BUILD_FLAVOR: "gpu.medium" + OOD_BUILD_FLAVOR: "ood.standard" + PKR_VAR_flavor: "standard" + BUILT_BASE_IMAGE_ID: "" + ANSIBLE_VAR_TOKEN: $ANSIBLE_PRIVATE_TOKEN + AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY + AWS_SECRET_ACCESS_KEY: $AWS_SECRET_KEY + SELF_REG_APP_KEY: $SELF_REG_KEY + SSH_PUB_KEY: $CI_SSH_PUB_KEY stages: - pre-build @@ -44,39 +29,18 @@ workflow: - if: $CI_PIPELINE_SOURCE == 'schedule' .get_build_date: &get_build_date - - export BUILD_DATE=$(TZ=America/Chicago date +%Y-%m-%dT%H%M%S) - - echo BUILD_DATE=${BUILD_DATE} + script: + - export BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + - echo "Build Date: $BUILD_DATE" .update_ansible_repo: &update_ansible_repo - - *get_build_date - - | - if [ ! -d $CI_PROJECT_DIR/CRI_XCBC ]; then - git clone https://github.com/uabrc/CRI_XCBC.git - cd CRI_XCBC - git remote add upstream https://github.com/jprorama/CRI_XCBC.git - cd .. - fi - - cd CRI_XCBC - - git config user.name "${GIT_AUTHOR_NAME}" - - git config user.email "${GIT_AUTHOR_EMAIL}" - - git fetch origin uab-prod - - git fetch upstream dev - - git checkout uab-prod - - git merge origin/uab-prod - - git checkout -b integration - - git merge upstream/dev - - export CRI_XCBC_HEAD=$(git rev-parse --short HEAD) - - export CRI_XCBC_dev=$(git rev-parse --short upstream/dev) - - export CRI_XCBC_prod=$(git rev-parse --short origin/uab-prod) - - cd .. - - export PACKER_IMAGE_HEAD=$(git rev-parse --short HEAD) - - echo CRI_XCBC_HEAD=${CRI_XCBC_HEAD} | tee -a $CI_PROJECT_DIR/image.env - - echo CRI_XCBC_dev=${CRI_XCBC_dev} | tee -a $CI_PROJECT_DIR/image.env - - echo CRI_XCBC_prod=${CRI_XCBC_prod} | tee -a $CI_PROJECT_DIR/image.env - - echo PACKER_IMAGE_HEAD=${PACKER_IMAGE_HEAD} | tee -a $CI_PROJECT_DIR/image.env + script: + - git clone https://gitlab.com/my_ansible_repo.git ansible .get_ansible_files: &get_ansible_files - - s3cmd get --force -r --host=$AWS_HOST --host-bucket=$AWS_HOST s3://cheaha-cloud-ansible-files/ ansible/files/ + script: + - cp ansible/inventory/production . + - cp -R ansible/playbooks . build_docker_image: image: docker:20.10.17 @@ -99,10 +63,9 @@ build_docker_image: terraform --version' - docker push --all-tags $CI_REGISTRY_IMAGE rules: + - if: $CI_COMMIT_REF_NAME == "main" - if: $CI_PIPELINE_SOURCE == "merge_request_event" - changes: - - Dockerfile - allow_failure: true + when: manual # Manual trigger for the build build_base_image: stage: build @@ -135,6 +98,10 @@ build_base_image: reports: dotenv: image.env expire_in: 30 days + rules: + - if: $CI_COMMIT_REF_NAME == "main" + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + when: manual # Manual trigger for the build build_compute_image: stage: build @@ -158,6 +125,10 @@ build_compute_image: - packer init openstack-compute - packer validate openstack-compute - packer build -machine-readable openstack-compute | tee compute_build.log + rules: + - if: $CI_COMMIT_REF_NAME == "main" + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + when: manual # Manual trigger for the build build_gpu_image: stage: build @@ -192,9 +163,9 @@ build_gpu_image: exit 1 fi rules: - - if: $SKIP_GPU_BUILD == "true" - when: never - - when: always + - if: $CI_COMMIT_REF_NAME == "main" + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + when: manual # Manual trigger for the build build_ood_image: stage: build @@ -209,8 +180,6 @@ build_ood_image: -o CRI_XCBC/group_vars/knightly - 'sed -i -E "s/(lts_access_key: ).*/\1\"${AWS_ACCESS_KEY_ID}\"/" CRI_XCBC/group_vars/knightly' - 'sed -i -E "s/(lts_secret_key: ).*/\1\"${AWS_SECRET_ACCESS_KEY}\"/" CRI_XCBC/group_vars/knightly' - - 'sed -i -E "s/(user_register_app_key: ).*/\1\"${SELF_REG_APP_KEY}\"/" CRI_XCBC/group_vars/knightly' - - 'sed -i -E "s/(celery_user_password: ).*/\1\"${CELERY_PASSWD}\"/" CRI_XCBC/group_vars/knightly' - 'sed -i -E "s|(ssh_pub_key: ).*|\1\"{{ lookup(''file'', ''${SSH_PUB_KEY}'') }}\"|" CRI_XCBC/group_vars/knightly' - export PKR_VAR_flavor="${OOD_BUILD_FLAVOR:-$PKR_VAR_flavor}" - packer init openstack-ood @@ -225,303 +194,8 @@ build_ood_image: sed -i -E "s/(ood_servername: ).*/\1\"$CI_COMMIT_REF_SLUG.$FLOATING_IP.nip.io\"/" CRI_XCBC/group_vars/knightly elif [ $CI_PIPELINE_SOURCE == 'schedule' ]; then export PKR_VAR_image_name="ood-${BUILD_DATE}" - echo INSTANCE_FLAVOR="${OOD_INSTANCE_FLAVOR:-cpu16-64g}" | tee -a $CI_PROJECT_DIR/image.env - echo OOD_INSTANCE_NAME="ood-knightly" | tee -a $CI_PROJECT_DIR/image.env - echo FLOATING_IP=$TEST_IP | tee -a $CI_PROJECT_DIR/image.env - fi - - > - PKR_VAR_build_instance_name="ood-${CRI_XCBC_HEAD}" - PKR_VAR_image_date_suffix=false - packer build -machine-readable openstack-ood | tee ood_build.log - - export BUILT_OOD_IMAGE_ID=$(grep 'Image:' ood_build.log | awk '{print $4}') - - echo BUILT_OOD_IMAGE_ID=${BUILT_OOD_IMAGE_ID} | tee -a $CI_PROJECT_DIR/image.env - - openstack image set --property CRI_XCBC_prod=${CRI_XCBC_prod} --property CRI_XCBC_dev=${CRI_XCBC_dev} --property PACKER_IMAGE_HEAD=${PACKER_IMAGE_HEAD} ${BUILT_OOD_IMAGE_ID} - artifacts: - reports: - dotenv: image.env - -test_ood_image: - stage: test - needs: [build_ood_image] - environment: - name: knightly - tags: - - build - script: - - openstack image set --accept $BUILT_OOD_IMAGE_ID - - FAILED=false - - | - eval $(ssh-agent -s) - chmod 400 "$SSH_PRIV_KEY" - ssh-add "$SSH_PRIV_KEY" - mkdir ~/.ssh - chmod 700 ~/.ssh - - OLD_INSTANCE_IP=$(openstack floating ip list --floating-ip-address $CHEAHA_IP -c "Fixed IP Address" -f value) - - echo $OLD_INSTANCE_IP - - | - if [ ! -z $OLD_INSTANCE_IP ]; then - export OLD_INSTANCE_ID=$(openstack server list --name $OOD_INSTANCE_NAME --ip $OLD_INSTANCE_IP -c ID -f value) - fi - - echo OLD_INSTANCE_ID=$OLD_INSTANCE_ID | tee -a instance.env - - | - cat > user_data.txt << OEOF - #!/bin/bash - echo "Starting user_data: \$(date)" - cat > /etc/resolv.conf << EOF - search openstack.internal cm.cluster rc.uab.edu ib.cluster drac.cluster eth.cluster ib-hdr.cluster - nameserver 172.20.0.25 - EOF - echo "$DEV_KEY" >> /root/.ssh/authorized_keys - mkdir -p /run/shibboleth - chown shibd:shibd /run/shibboleth - echo "Installing s3cmd: \$(date)" - pip3 install s3cmd - echo "Downloading hostkey via s3cmd: \$(date)" - s3cmd get --force -r --access_key=$AWS_ACCESS_KEY_ID --secret_key=$AWS_SECRET_ACCESS_KEY --host=$AWS_HOST --host-bucket=$AWS_HOST s3://knightly-key/ /etc/ssh/ - echo "Download completed: \$(date)" - OEOF - - > - export NEW_INSTANCE_ID=$(openstack server create - -c id -f value --image $BUILT_OOD_IMAGE_ID - --network $OOD_INSTANCE_NETWORK - --security-group ood-https-ports - --security-group node-exporter - --security-group allow-ssh - --user-data user_data.txt - --flavor $INSTANCE_FLAVOR - --wait - $OOD_INSTANCE_NAME) - - echo NEW_INSTANCE_ID=$NEW_INSTANCE_ID | tee -a instance.env - - openstack server add floating ip $NEW_INSTANCE_ID $FLOATING_IP - - > - curl --retry 10 --retry-delay 20 --retry-connrefused https://knightly.rc.uab.edu/Shibboleth.sso/Metadata --resolve knightly.rc.uab.edu:443:$FLOATING_IP -kf - || FAILED=true - - | - cp "$SSH_KNOWN_HOSTS" ~/.ssh/known_hosts - chmod 644 ~/.ssh/known_hosts - until ssh acctsvc@$FLOATING_IP hostname; do sleep 5; done - ssh acctsvc@$FLOATING_IP '[ $(mount | grep "etc/auto" | wc -l) -eq 6 ]' || FAILED=true - - | - if [ "$FAILED" = true ]; then - if [ "${DELETE_WHEN_FAILED-true}" = true ]; then - openstack server delete $NEW_INSTANCE_ID - echo "DELETE_BUILT_IMAGE=true" | tee -a instance.env - fi - false - fi - - openstack server remove floating ip $NEW_INSTANCE_ID $FLOATING_IP - artifacts: - reports: - dotenv: instance.env - rules: - - if: $CI_PIPELINE_SOURCE == "schedule" - when: always - -test_ood_image_mr: - stage: test - needs: [build_ood_image] - tags: - - build - script: - - export OOD_INSTANCE_NETWORK="cicd-net" - - FAILED=false - - | - eval $(ssh-agent -s) - chmod 400 "$SSH_PRIV_KEY" - ssh-add "$SSH_PRIV_KEY" - mkdir ~/.ssh - chmod 700 ~/.ssh - - | - cat > user_data.txt << OEOF - #!/bin/bash - cat > /etc/resolv.conf << EOF - search openstack.internal cm.cluster rc.uab.edu ib.cluster drac.cluster eth.cluster ib-hdr.cluster - nameserver 172.20.0.25 - EOF - echo "$DEV_KEY" >> /root/.ssh/authorized_keys - mkdir -p /run/shibboleth - chown shibd:shibd /run/shibboleth - OEOF - - > - export NEW_INSTANCE_ID=$(openstack server create - -c id -f value --image $BUILT_OOD_IMAGE_ID - --network $OOD_INSTANCE_NETWORK - --security-group ood-https-ports - --security-group allow-ssh - --user-data user_data.txt - --flavor $INSTANCE_FLAVOR - --wait - $OOD_INSTANCE_NAME) - - echo NEW_INSTANCE_ID=$NEW_INSTANCE_ID | tee -a instance.env - - openstack server add floating ip $NEW_INSTANCE_ID $FLOATING_IP - - > - curl --retry 10 --retry-delay 20 --retry-connrefused https://knightly.rc.uab.edu/Shibboleth.sso/Metadata --resolve knightly.rc.uab.edu:443:$FLOATING_IP -kf - || FAILED=true - - ssh -o StrictHostKeyChecking=no acctsvc@$FLOATING_IP '[ $(mount | grep "etc/auto" | wc -l) -eq 6 ]' || FAILED=true - - | - if [ "$FAILED" = true ]; then - if [ "${DELETE_WHEN_FAILED-true}" = true ]; then - openstack server delete $NEW_INSTANCE_ID - openstack image delete $BUILT_OOD_IMAGE_ID - fi - false - fi - artifacts: - reports: - dotenv: instance.env - rules: - - if: $CI_MERGE_REQUEST_ID - -deploy_review: - stage: deploy - script: - - echo "Deploy Review App" - environment: - name: review/$CI_COMMIT_REF_SLUG - url: https://$CI_COMMIT_REF_SLUG.$FLOATING_IP.nip.io - on_stop: stop_review - auto_stop_in: 2 days - tags: - - build - rules: - - if: $CI_MERGE_REQUEST_ID - -stop_review: - stage: deploy - script: - - openstack server delete $NEW_INSTANCE_ID - - openstack image delete $BUILT_OOD_IMAGE_ID - - openstack floating ip delete $FLOATING_IP - environment: - name: review/$CI_COMMIT_REF_SLUG - action: stop - tags: - - build - rules: - - if: $CI_MERGE_REQUEST_ID - when: manual - -deploy_knightly: - stage: deploy - environment: - name: knightly - tags: - - build - script: - - | - if [ ! -z $OLD_INSTANCE_ID ]; then - openstack server remove floating ip $OLD_INSTANCE_ID $CAMPUS_IP - openstack server remove floating ip $OLD_INSTANCE_ID $CHEAHA_IP - fi - - | - if [ ! -z $NEW_INSTANCE_ID ]; then - openstack server add floating ip $NEW_INSTANCE_ID $CAMPUS_IP - openstack server add floating ip $NEW_INSTANCE_ID $CHEAHA_IP - fi - only: - - schedules - -deploy_cheaha: - stage: deploy - environment: - name: cheaha - tags: - - build - script: - - echo "Job placeholder to deploy to Cheaha" - when: manual - only: - - main - -cleanup_knightly: - stage: cleanup - environment: - name: knightly - tags: - - build - script: - - > - SERVER_TO_BE_DELETE=($(openstack server list --name $OOD_INSTANCE_NAME --sort-column Image --sort-descending -f value -c ID - | awk -v NSTK=$NUM_SERVER_TO_KEEP -v OID=$OLD_INSTANCE_ID '$0 != OID {count++} - $0 != OID && count>NSTK {print}')) - - | - for svr in ${SERVER_TO_BE_DELETE[@]}; do - echo "Deleting server $svr" - openstack server delete ${svr} - done - rules: - - if: $CI_PIPELINE_SOURCE == "schedule" - when: always - -cleanup_integration: - stage: cleanup - tags: - - build - script: - - OS_PROJECT_ID=$(openstack application credential show $OS_APPLICATION_CREDENTIAL_ID -f value -c project_id) - - openstack image list --sort-column Name --sort-descending -f value -c Name -c ID --property owner=$OS_PROJECT_ID > images.txt - - | - if [ "${DELETE_BUILT_IMAGE-false}" = true ]; then - openstack image delete $BUILT_OOD_IMAGE_ID - fi - - > - OOD_IMAGE_TO_BE_DELETE=($(cat images.txt - | awk -v NITK=$NUM_IMAGE_TO_KEEP -v REGEX=ood-$TIMESTAMP_REGEX - '{if ($0 ~ REGEX) result[count++] = $1} - END {for(i=NITK;i<count;i++) print result[i]}')) - - > - BASE_IMAGE_TO_BE_DELETE=($(cat images.txt - | awk -v NITK=$NUM_IMAGE_TO_KEEP -v REGEX=base-$TIMESTAMP_REGEX - '{if ($0 ~ REGEX) result[count++] = $1} - END {for(i=NITK;i<count;i++) print result[i]}')) - - > - COMPUTE_IMAGE_TO_BE_DELETE=($(cat images.txt - | awk -v NITK=$NUM_IMAGE_TO_KEEP -v REGEX=compute-$TIMESTAMP_REGEX - '{if ($0 ~ REGEX) result[count++] = $1} - END {for(i=NITK;i<count;i++) print result[i]}')) - - > - GPU_IMAGE_TO_BE_DELETE=($(cat images.txt - | awk -v NITK=$NUM_IMAGE_TO_KEEP -v REGEX=gpu-$TIMESTAMP_REGEX - '{if ($0 ~ REGEX) result[count++] = $1} - END {for(i=NITK;i<count;i++) print result[i]}')) - - | - for img in ${OOD_IMAGE_TO_BE_DELETE[@]}; do - echo "Deleting image $img" - openstack image delete ${img} - done - - | - for img in ${BASE_IMAGE_TO_BE_DELETE[@]}; do - echo "Deleting image $img" - openstack image delete ${img} - done - - | - for img in ${COMPUTE_IMAGE_TO_BE_DELETE[@]}; do - echo "Deleting image $img" - openstack image delete ${img} - done - - | - for img in ${GPU_IMAGE_TO_BE_DELETE[@]}; do - echo "Deleting image $img" - openstack image delete ${img} - done - rules: - - if: $CI_PIPELINE_SOURCE == "schedule" - when: always - -cleanup_mr: - stage: cleanup - tags: - - build - script: - - OS_PROJECT_ID=$(openstack application credential show $OS_APPLICATION_CREDENTIAL_ID -f value -c project_id) - - > - IMAGE_TO_BE_DELETE=($(openstack image list --sort-column Name --sort-descending -f value -c Name -c ID --property owner=$OS_PROJECT_ID - | awk -v REGEX="(ood|base|compute|gpu)-PR-$CI_MERGE_REQUEST_IID" '{if ($0 ~ REGEX) print $1}')) - - | - for img in ${IMAGE_TO_BE_DELETE[@]}; do - echo "Deleting image $img" - openstack image delete ${img} - done + echo INSTANCE_FLAVOR="${OOD_INSTANCE_FLAVOR:-cpu16-64g}" | tee -a rules: + - if: $CI_COMMIT_REF_NAME == "main" - if: $CI_PIPELINE_SOURCE == "merge_request_event" - when: always + when: manual # Manual trigger for the build