diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 48993503ea087b304d8d3ef8fc8b86ac0c4240a2..5704eb68e50b7d4b2bf24fe1ac7690ed54c31144 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -106,7 +106,7 @@ ansible_create_cluster_stage: - echo "ansible_create_cluster_stage" - bash -x ./CICD/ansible_create_cluster_script.sh #after_script: - #- rm ./files/cicd_inventory + #- rm ./files/inventory.$STACKNAME #only: # changes: #https://docs.gitlab.com/ee/ci/yaml/#onlychangesexceptchanges # - "master_playbook.yml" @@ -124,24 +124,29 @@ tests: - echo "$HPCCICD_openrc" > ./$NECTAR_ALLOCATION-openrc.sh script: - echo "tests stage" - - export ANSIBLE_HOST_KEY_CHECKING=False - source ./$NECTAR_ALLOCATION-openrc.sh - - python3 dependencies/ansible_cluster_in_a_box/scripts/make_inventory.py static CICD$STACKNAME | tee ./files/cicd_inventory - - grep -qv "I could not find any resouces tagged with project_name:" ./files/cicd_inventory #fail if inventory file is empty - - chmod 755 ./files/cicd_inventory - - chmod 400 ./keys/gc_key.pem - - ansible -B 30 -i files/cicd_inventory --key-file keys/gc_key.pem -a "sinfo" ManagementNodes - - ansible -B 30 -i files/cicd_inventory --key-file keys/gc_key.pem -a "squeue" ManagementNodes - - ansible -B 30 -i files/cicd_inventory --key-file keys/gc_key.pem -a "systemctl is-active --quiet mariadb" SQLNodes - - ansible -B 30 -i files/cicd_inventory --key-file keys/gc_key.pem -a "systemctl is-active --quiet slurmctld" ManagementNodes - - ansible -B 30 -i files/cicd_inventory --key-file keys/gc_key.pem -a "systemctl is-active --quiet slurmdbd" ManagementNodes + - openstack stack list + - cd CICD + - STACKNAME=CICD$STACKNAME + - export ANSIBLE_HOST_KEY_CHECKING=False + - echo "[ansible_create_cluster_script] Prefixing the Stackname with CICD because heatcicdwrapper does he same" + - python3 ../scripts/make_inventory.py static $STACKNAME | tee ./files/inventory.$STACKNAME && chmod 755 ./files/inventory.$STACKNAME + - grep -qv "I could not find any resouces tagged with project_name:" ./files/inventory.$STACKNAME #fail if inventory file is empty + - ansible -m ping -i files/inventory.$STACKNAME --key-file ../gc_key.pem all + - ansible -i files/inventory.$STACKNAME --key-file ../gc_key.pem -a "sudo ls" all + + - ansible -B 30 -i files/inventory.$STACKNAME --key-file ../gc_key.pem -a "sinfo" ManagementNodes + - ansible -B 30 -i files/inventory.$STACKNAME --key-file ../gc_key.pem -a "squeue" ManagementNodes + - ansible -B 30 -i files/inventory.$STACKNAME --key-file ../gc_key.pem -a "systemctl is-active --quiet mariadb" SQLNodes + - ansible -B 30 -i files/inventory.$STACKNAME --key-file ../gc_key.pem -a "systemctl is-active --quiet slurmctld" ManagementNodes + - ansible -B 30 -i files/inventory.$STACKNAME --key-file ../gc_key.pem -a "systemctl is-active --quiet slurmdbd" ManagementNodes - - bash -e ./tests/run_tests.sh all "files/cicd_inventory" "keys/gc_key.pem" - - bash -e ./tests/run_tests.sh ComputeNodes "files/cicd_inventory" "keys/gc_key.pem" - - bash -e ./tests/run_tests.sh LoginNodes "files/cicd_inventory" "keys/gc_key.pem" - - bash -e ./tests/run_tests.sh ManagementNodes "files/cicd_inventory" "keys/gc_key.pem" - - bash -e ./tests/run_tests.sh NFSNodes "files/cicd_inventory" "keys/gc_key.pem" - - bash -e ./tests/run_tests.sh SQLNodes "files/cicd_inventory" "keys/gc_key.pem" + - bash -e ./tests/run_tests.sh all "files/inventory.$STACKNAME" "../gc_key.pem" + - bash -e ./tests/run_tests.sh ComputeNodes "files/inventory.$STACKNAME" "../gc_key.pem" + - bash -e ./tests/run_tests.sh LoginNodes "files/inventory.$STACKNAME" "../gc_key.pem" + - bash -e ./tests/run_tests.sh ManagementNodes "files/inventory.$STACKNAME" "../gc_key.pem" + - bash -e ./tests/run_tests.sh NFSNodes "files/inventory.$STACKNAME" "../gc_key.pem" + - bash -e ./tests/run_tests.sh SQLNodes "files/inventory.$STACKNAME" "../gc_key.pem" # licensing https://gitlab.erc.monash.edu.au/hpc-team/license_server/tree/master/roles/avizo_license_monitor @@ -172,7 +177,7 @@ manual_cluster_spawn: when: manual only: refs: - - "devel" + - "cicd" extended: stage: extended diff --git a/CICD/tests/LoginNodes/run_slurm_testsuite.inactive b/CICD/tests/LoginNodes/run_slurm_testsuite.inactive new file mode 100755 index 0000000000000000000000000000000000000000..c5d2f24f1f3cf99c3f7481f3bc467907444425d6 --- /dev/null +++ b/CICD/tests/LoginNodes/run_slurm_testsuite.inactive @@ -0,0 +1,29 @@ +#!/bin/bash +OUTPUT_LOG=$(realpath ${1-slurmtest.out}) +if ! type "scontrol" > /dev/null; then + echo "cannot find slurm" + exit 1 +fi +SLURM_DIR=${2-$(dirname $(dirname $(which scontrol)))} +#SLURM_DIR=$slurm_dir + +#if [[ -d $2 ]];then +# SLURM_SRC_DIR=$2 +#else +# SLURM_SRC_DIR=./slurm_src +# git clone https://github.com/SchedMD/slurm.git $SLURM_SRC_DIR +# cd $SLURM_SRC_DIR && ./configure +#fi +#cd $SLURM_SRC_DIR/testsuite/expect +#echo -en "set slurm_dir=$SLURM_DIR\nset max_job_delay 300\n" > globals.local +#make +#echo "log is written to $OUTPUT_LOG" +#echo "slurm dir is defined as $SLURM_DIR" +./regression > /dev/null 2> $OUTPUT_LOG +failures="$(sed -n 's/Failures: \(.*\)/\1/p' $OUTPUT_LOG)" +if (( $failures > 0 ));then + echo "$failures failures found, refer to $OUTPUT_LOG for log" + exit 1 +fi +exit 0 + diff --git a/CICD/tests/ManagementNodes/check.yml b/CICD/tests/ManagementNodes/check.yml new file mode 100644 index 0000000000000000000000000000000000000000..95e06a0a034c32c5e8ae30c2a58c40e10a738afc --- /dev/null +++ b/CICD/tests/ManagementNodes/check.yml @@ -0,0 +1,8 @@ +--- +- hosts: ManagementNodes + gather_facts: false + tasks: + - name: have ssh running + service: + name: sshd + state: started \ No newline at end of file diff --git a/CICD/tests/Readme.md b/CICD/tests/Readme.md new file mode 100644 index 0000000000000000000000000000000000000000..feca06268d107c2aeae9a6a8f61a2ed59e8648bc --- /dev/null +++ b/CICD/tests/Readme.md @@ -0,0 +1,7 @@ +this folder should contain tests that will be run automatically by the CICD pipeline + +all files with fileending .sh will be executed by a shell +all files with fileending yml will be executed by ansible-playbook +./tmp can be used as temporary folder and will be cleaned after execution + +because I can I am prefixing tests with 0-9 to give the execution some priority \ No newline at end of file diff --git a/CICD/tests/all/0_EXAMPLE_FALSE.sh b/CICD/tests/all/0_EXAMPLE_FALSE.sh new file mode 100755 index 0000000000000000000000000000000000000000..10c48607688d030fbbf054b1046e18d431b869c3 --- /dev/null +++ b/CICD/tests/all/0_EXAMPLE_FALSE.sh @@ -0,0 +1,5 @@ +#!/bin/bash +/bin/false + +status=$? +[ $status -eq 1 ] \ No newline at end of file diff --git a/CICD/tests/all/0_EXAMPLE_TRUE.sh b/CICD/tests/all/0_EXAMPLE_TRUE.sh new file mode 100755 index 0000000000000000000000000000000000000000..3634c7aa3076c2e8cd2159aca337adb35f1f31cf --- /dev/null +++ b/CICD/tests/all/0_EXAMPLE_TRUE.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +/bin/true + +status=$? +[ $status -eq 0 ] diff --git a/CICD/tests/run_tests.sh b/CICD/tests/run_tests.sh new file mode 100644 index 0000000000000000000000000000000000000000..d063e98d1d7e4617882bb14a5e1c51d9e8cda381 --- /dev/null +++ b/CICD/tests/run_tests.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +function usage { + echo $"Usage: $0 {all, ComputeNodes, LoginNodes, ManagementNodes, NFSNodes, sql}" INVENTORY_FILE KEY + exit 1 +} + +function run_them () +{ + #limit='--limit '"$1" + #if [ "$1" = "all" ] + #then + # limit="all" + #fi + for filename in ./tests/$1/*.sh; do # this is not sorted yet + [ -e "$filename" ] || continue + #/bin/bash -x $filename # local execution. nice for dev + ansible -i $2 --key-file $3 -m script -a "$filename" $1 + done + for filename in ./tests/$1/*.yml; do # this is not sorted yet + [ -e "$filename" ] || continue + ansible-playbook -i $2 --key-file $3 $filename # I am assuming the playbook cares about visibility here. might have to change later + done +} + +# I think I am just checking the if $1 is one of the listes strings (see usage) not proud of this at all but works +case "$1" in + all) + ;; + ComputeNodes) + ;; + ManagementNodes) + ;; + NFSNodes) + ;; + SQLNodes) + ;; + LoginNodes) + ;; + *) + usage +esac + +run_them $1 $2 $3 \ No newline at end of file