variables: GIT_SUBMODULE_STRATEGY: recursive STACKNAME: CICD_${CI_PROJECT_NAME}_$CI_COMMIT_REF_NAME NECTAR_ALLOCATION: HPCCICD ANSIBLE_HOST_KEY_CHECKING: "False" DEFAULT_PATH: "CICD" stages: - lint - extended #- heat_test - heat - ansible_create_cluster_stage - push_button_spawn_cluster - tests - integration_test #https://docs.gitlab.com/ee/ci/triggers/ - clean - testlustre trigger_pipeline_in_Clusterbuild: stage: integration_test tags: - ansible script: - echo ${CI_JOB_TOKEN} - curl --request POST --form token=${CI_JOB_TOKEN} --form "variables[TRIGGER_CI_COMMIT_SHA]=${CI_COMMIT_SHA}" --form ref=master https://gitlab.erc.monash.edu.au/api/v4/projects/193/trigger/pipeline # ID is from clusterbuild only: - ${CI_PROJECT_NAME} == 'HPCasCode' trigger_pipeline_in_monarch: stage: integration_test tags: - ansible script: - echo ${CI_JOB_TOKEN} - curl --request POST --form token=${CI_JOB_TOKEN} --form "variables[TRIGGER_CI_COMMIT_SHA]=${CI_COMMIT_SHA}" --form ref=master https://gitlab.erc.monash.edu.au/api/v4/projects/385/trigger/pipeline # ID is from monarch only: - ${CI_PROJECT_NAME} == 'HPCasCode' yamllint: stage: lint allow_failure: true tags: - yamllint script: - echo "stage yamllint" - cd CICD - yamllint -c ./.yamllintheat.yaml ./heat # delete_stack_manual: # stage: delete_stack_manual # tags: # - heat # before_script: # - echo "$GC_KEY" > gc_key.pem # - chmod 400 gc_key.pem # - echo "$HPCCICD_openrc" > ./$NECTAR_ALLOCATION-openrc.sh # script: # - echo "heat stage" # - source ./$NECTAR_ALLOCATION-openrc.sh # - openstack stack list # - bash -x ./CICD/heat/heatcicdwrapper.sh delete_if_exists $STACKNAME # when: manual ansiblelint: allow_failure: true stage: lint tags: - ansiblelint script: - echo "stage ansiblelint" - cd CICD - python3 ansiblelint/run_lint.py --targets master_playbook.yml - python3 ansiblelint/run_lint.py --targets ../qa.yml - python3 ansiblelint/run_lint.py --targets ../maintenance.yml testlustre: stage: testlustre allow_failure: true tags: - heat before_script: - cd $DEFAULT_PATH - echo "$GC_KEY" > gc_key.pem - chmod 400 gc_key.pem - echo "$HPCCICD_openrc" > ./$NECTAR_ALLOCATION-openrc.sh - sleep 60 script: - echo "heat stage" - source ./$NECTAR_ALLOCATION-openrc.sh - openstack stack list - openstack stack create --wait --template heat/lustre_HOT.yaml --parameter "project_name=$STACKNAME" $STACKNAME - python3 ../scripts/make_inventory.py static $STACKNAME | tee ./files/inventory.$STACKNAME && chmod 755 ./files/inventory.$STACKNAME - cd plays/testlustre - sleep 100 - ansible-playbook -i files/inventory.$STACKNAME --key-file ../../gc_key.pem testlustre.yml - sleep 60 - cd ../../ - bash -x ./heat/heatcicdwrapper.sh delete_if_exists $STACKNAME after_script: - sleep 60 # the cluster needs to delete first build_cluster_cicd: stage: heat allow_failure: false tags: - heat before_script: - cd $DEFAULT_PATH - echo "$GC_KEY" > gc_key.pem - chmod 400 gc_key.pem - echo "$HPCCICD_openrc" > ./$NECTAR_ALLOCATION-openrc.sh script: - echo "heat stage" - source ./$NECTAR_ALLOCATION-openrc.sh - openstack stack list - bash -x ./heat/heatcicdwrapper.sh create_or_update $STACKNAME - bash -x ./heat/server_rebuild.sh all after_script: - sleep 30 # artifically wait a bit to give the nodes time to boot # only: # changes: #https://docs.gitlab.com/ee/ci/yaml/#onlychangesexceptchanges # - "heat/*HOT*.yaml" # - schedules # - ./.gitlab-ci.yml ansible_create_cluster_stage: stage: ansible_create_cluster_stage tags: - ansible before_script: - cd $DEFAULT_PATH/.. - echo "$GC_KEY" > gc_key.pem - chmod 400 gc_key.pem - echo "$HPCCICD_openrc" > ./$NECTAR_ALLOCATION-openrc.sh script: - echo "ansible_create_cluster_stage" - bash -x ./CICD/ansible_create_cluster_script.sh - cd CICD - ansible-playbook -i files/inventory.$STACKNAME --key-file ../gc_key.pem --skip-tags SiteSpecific master_playbook.yml - sleep 15 - echo uglyuglyfix - ansible -i files/inventory.$STACKNAME --key-file ../gc_key.pem -b -a "systemctl restart slurmdbd" ManagementNodes - ansible -i files/inventory.$STACKNAME --key-file ../gc_key.pem -b -a "systemctl restart slurmctld" ManagementNodes - cd plays - ansible-playbook -i files/inventory.$STACKNAME --key-file ../../gc_key.pem --skip-tags monitoring computenodes.yml | tee nochange.log - echo [ `grep changed= ./nochange.log -c` = `grep changed=0 ./nochange.log -c` ] > bashtest.sh # a crude way to make sure all changed lines are equal to changed=0 - bash ./bashtest.sh - ansible-playbook -i files/inventory.$STACKNAME --key-file ../../gc_key.pem --skip-tags monitoring --check computenodes.yml tests: stage: tests tags: - ansible before_script: - echo "$GC_KEY" > gc_key.pem - chmod 400 gc_key.pem - echo "$HPCCICD_openrc" > ./$NECTAR_ALLOCATION-openrc.sh script: - echo "tests stage" - source ./$NECTAR_ALLOCATION-openrc.sh - openstack stack list - cd CICD - python3 ../scripts/make_inventory.py static $STACKNAME | tee ./files/inventory.$STACKNAME && chmod 755 ./files/inventory.$STACKNAME - grep -qv "I could not find any resouces tagged with project_name:" ./files/inventory.$STACKNAME #fail if inventory file is empty - ansible -m ping -i files/inventory.$STACKNAME --key-file ../gc_key.pem all - ansible -i files/inventory.$STACKNAME --key-file ../gc_key.pem -a "sudo ls" all - echo -e '[defaults]\r\nallow_world_readable_tmpfiles = True' > ansible.cfg - ansible-playbook -i files/inventory.$STACKNAME --key-file ../gc_key.pem ./tests/mockSlurmData.yml - ansible -B 30 -i files/inventory.$STACKNAME --key-file ../gc_key.pem -a "/opt/slurm-latest/bin/sinfo" ManagementNodes - ansible -B 30 -i files/inventory.$STACKNAME --key-file ../gc_key.pem -a "/opt/slurm-latest/bin/squeue" ManagementNodes - ansible -B 30 -i files/inventory.$STACKNAME --key-file ../gc_key.pem -a "/opt/slurm-latest/bin/scontrol ping" LoginNodes - ansible -B 30 -i files/inventory.$STACKNAME --key-file ../gc_key.pem -a "systemctl is-active --quiet ntpd" CentosNodes - ansible -B 30 -i files/inventory.$STACKNAME --key-file ../gc_key.pem -a "systemctl is-active --quiet ntp" UbuntuNodes - ansible -B 30 -i files/inventory.$STACKNAME --key-file ../gc_key.pem -a "systemctl is-active --quiet mariadb" SQLNodes - ansible -B 30 -i files/inventory.$STACKNAME --key-file ../gc_key.pem -a "systemctl is-active --quiet slurmctld" ManagementNodes - ansible -B 30 -i files/inventory.$STACKNAME --key-file ../gc_key.pem -a "systemctl is-active --quiet slurmdbd" ManagementNodes - bash -e ./tests/run_tests.sh all "files/inventory.$STACKNAME" "../gc_key.pem" - bash -e ./tests/run_tests.sh ComputeNodes "files/inventory.$STACKNAME" "../gc_key.pem" - bash -e ./tests/run_tests.sh LoginNodes "files/inventory.$STACKNAME" "../gc_key.pem" - bash -e ./tests/run_tests.sh ManagementNodes "files/inventory.$STACKNAME" "../gc_key.pem" - bash -e ./tests/run_tests.sh NFSNodes "files/inventory.$STACKNAME" "../gc_key.pem" - bash -e ./tests/run_tests.sh SQLNodes "files/inventory.$STACKNAME" "../gc_key.pem" - bash -e ./tests/run_tests.sh slurm "files/inventory.$STACKNAME" "../gc_key.pem" - ansible -i files/inventory.$STACKNAME --key-file ../gc_key.pem -a 'sudo su - user1 -c whoami' LoginNodes,ComputeNodes # to test ldap #- sshpass -p 'redhat' ssh -o StrictHostKeyChecking=no user1@server.example.com extended: stage: extended tags: - heat - ansible before_script: - echo "cleanup stack" - sleep 30 - echo "$HPCCICD_openrc" > ./$NECTAR_ALLOCATION-openrc.sh script: - source ./$NECTAR_ALLOCATION-openrc.sh - bash -x ./CICD/heat/heatcicdwrapper.sh delete_if_exists $STACKNAME ${CI_PROJECT_NAME} only: variables: - $EXTENDED != null manual_cluster_spawn: stage: push_button_spawn_cluster tags: - heat - ansible before_script: - echo "press button spawn cluster." - echo "for this to work you have to provide a variable called manual stackname" - echo I still need to handle os password - echo $MANUAL_STACKNAME - echo "$GC_KEY" > gc_key.pem - chmod 400 gc_key.pem - echo "$HPCCICD_openrc" > ./$NECTAR_ALLOCATION-openrc.sh script: - source ./$NECTAR_ALLOCATION-openrc.sh - bash -x ./CICD/heat/heatcicdwrapper.sh create $MANUAL_STACKNAME ${CI_PROJECT_NAME} - openstack stack list - export STACKNAME=$MANUAL_STACKNAME - sleep 25 - bash -x CICD/ansible_create_cluster_script.sh when: manual only: refs: - "cicd" clean: stage: clean tags: - heat before_script: - echo "cleanup stack" - sleep 30 - echo "$HPCCICD_openrc" > ./$NECTAR_ALLOCATION-openrc.sh script: - source ./$NECTAR_ALLOCATION-openrc.sh - bash -x ./CICD/heat/heatcicdwrapper.sh delete_if_exists $STACKNAME # heat_test: # stage: heat_test # allow_failure: false # tags: # - heat # before_script: # - echo "$GC_KEY" > gc_key.pem # - chmod 400 gc_key.pem # - echo "$HPCCICD_openrc" > ./$NECTAR_ALLOCATION-openrc.sh # - source ./$NECTAR_ALLOCATION-openrc.sh # - export HEAT_TEST_STACKNAME=_TESTING_HEAT # - bash -x ./CICD/heat/heatcicdwrapper.sh delete_if_exists $HEAT_TEST_STACKNAME # - sleep 60 # script: # - echo "heat_test stage" # - source ./$NECTAR_ALLOCATION-openrc.sh # - bash -x ./CICD/heat/heatcicdwrapper.sh create $HEAT_TEST_STACKNAME # - openstack stack list # - bash -x ./CICD/heat/heatcicdwrapper.sh update $HEAT_TEST_STACKNAME # - openstack stack list # - bash -x ./CICD/heat/heatcicdwrapper.sh delete_if_exists $HEAT_TEST_STACKNAME # - openstack stack list # after_script: # - sleep 20 # artifically wait a bit to make sure it is really dead