pipeline fixes and more tests

.gitlab-ci.yml

@@ -106,7 +106,7 @@ ansible_create_cluster_stage:
     - echo "ansible_create_cluster_stage"
     - bash -x ./CICD/ansible_create_cluster_script.sh
   #after_script:
-    #- rm ./files/cicd_inventory
+    #- rm ./files/inventory.$STACKNAME
   #only:
   #  changes: #https://docs.gitlab.com/ee/ci/yaml/#onlychangesexceptchanges
   #  - "master_playbook.yml"
@@ -124,24 +124,29 @@ tests:
     - echo "$HPCCICD_openrc" > ./$NECTAR_ALLOCATION-openrc.sh
   script:
     - echo "tests stage"
-    - export ANSIBLE_HOST_KEY_CHECKING=False
     - source ./$NECTAR_ALLOCATION-openrc.sh
-    - python3 dependencies/ansible_cluster_in_a_box/scripts/make_inventory.py static CICD$STACKNAME | tee ./files/cicd_inventory
-    - grep -qv "I could not find any resouces tagged with project_name:" ./files/cicd_inventory   #fail if inventory file is empty
-    - chmod 755 ./files/cicd_inventory
-    - chmod 400 ./keys/gc_key.pem
-    - ansible -B 30 -i files/cicd_inventory --key-file keys/gc_key.pem -a "sinfo" ManagementNodes
-    - ansible -B 30 -i files/cicd_inventory --key-file keys/gc_key.pem -a "squeue" ManagementNodes
-    - ansible -B 30 -i files/cicd_inventory --key-file keys/gc_key.pem -a "systemctl is-active --quiet mariadb" SQLNodes
-    - ansible -B 30 -i files/cicd_inventory --key-file keys/gc_key.pem -a "systemctl is-active --quiet slurmctld" ManagementNodes
-    - ansible -B 30 -i files/cicd_inventory --key-file keys/gc_key.pem -a "systemctl is-active --quiet slurmdbd" ManagementNodes
+    - openstack stack list
+    - cd CICD
+    - STACKNAME=CICD$STACKNAME
+    - export ANSIBLE_HOST_KEY_CHECKING=False
+    - echo "[ansible_create_cluster_script] Prefixing the Stackname with CICD because heatcicdwrapper does he same"
+    - python3 ../scripts/make_inventory.py static $STACKNAME | tee ./files/inventory.$STACKNAME && chmod 755 ./files/inventory.$STACKNAME
+    - grep -qv "I could not find any resouces tagged with project_name:" ./files/inventory.$STACKNAME   #fail if inventory file is empty
+    - ansible -m ping -i files/inventory.$STACKNAME --key-file ../gc_key.pem all
+    - ansible -i files/inventory.$STACKNAME --key-file ../gc_key.pem -a "sudo ls" all
+    
+    - ansible -B 30 -i files/inventory.$STACKNAME --key-file ../gc_key.pem -a "sinfo" ManagementNodes
+    - ansible -B 30 -i files/inventory.$STACKNAME --key-file ../gc_key.pem -a "squeue" ManagementNodes
+    - ansible -B 30 -i files/inventory.$STACKNAME --key-file ../gc_key.pem -a "systemctl is-active --quiet mariadb" SQLNodes
+    - ansible -B 30 -i files/inventory.$STACKNAME --key-file ../gc_key.pem -a "systemctl is-active --quiet slurmctld" ManagementNodes
+    - ansible -B 30 -i files/inventory.$STACKNAME --key-file ../gc_key.pem -a "systemctl is-active --quiet slurmdbd" ManagementNodes
     
-    - bash -e ./tests/run_tests.sh all "files/cicd_inventory" "keys/gc_key.pem"
-    - bash -e ./tests/run_tests.sh ComputeNodes "files/cicd_inventory" "keys/gc_key.pem"
-    - bash -e ./tests/run_tests.sh LoginNodes "files/cicd_inventory" "keys/gc_key.pem"
-    - bash -e ./tests/run_tests.sh ManagementNodes "files/cicd_inventory" "keys/gc_key.pem"
-    - bash -e ./tests/run_tests.sh NFSNodes "files/cicd_inventory" "keys/gc_key.pem"
-    - bash -e ./tests/run_tests.sh SQLNodes "files/cicd_inventory" "keys/gc_key.pem"
+    - bash -e ./tests/run_tests.sh all "files/inventory.$STACKNAME" "../gc_key.pem"
+    - bash -e ./tests/run_tests.sh ComputeNodes "files/inventory.$STACKNAME" "../gc_key.pem"
+    - bash -e ./tests/run_tests.sh LoginNodes "files/inventory.$STACKNAME" "../gc_key.pem"
+    - bash -e ./tests/run_tests.sh ManagementNodes "files/inventory.$STACKNAME" "../gc_key.pem"
+    - bash -e ./tests/run_tests.sh NFSNodes "files/inventory.$STACKNAME" "../gc_key.pem"
+    - bash -e ./tests/run_tests.sh SQLNodes "files/inventory.$STACKNAME" "../gc_key.pem"
     
     # licensing https://gitlab.erc.monash.edu.au/hpc-team/license_server/tree/master/roles/avizo_license_monitor
 
@@ -172,7 +177,7 @@ manual_cluster_spawn:
   when: manual 
   only:
     refs:
-      - "devel"
+      - "cicd"
 
 extended:
   stage: extended

CICD/tests/LoginNodes/run_slurm_testsuite.inactive

+#!/bin/bash
+OUTPUT_LOG=$(realpath ${1-slurmtest.out})
+if ! type "scontrol" > /dev/null; then
+	echo "cannot find slurm"
+	exit 1
+fi
+SLURM_DIR=${2-$(dirname $(dirname $(which scontrol)))}
+#SLURM_DIR=$slurm_dir
+
+#if [[ -d $2 ]];then
+#    SLURM_SRC_DIR=$2    
+#else
+#    SLURM_SRC_DIR=./slurm_src
+#    git clone https://github.com/SchedMD/slurm.git $SLURM_SRC_DIR
+#    cd $SLURM_SRC_DIR && ./configure
+#fi
+#cd $SLURM_SRC_DIR/testsuite/expect
+#echo -en "set slurm_dir=$SLURM_DIR\nset max_job_delay 300\n" > globals.local
+#make
+#echo "log is written to $OUTPUT_LOG"
+#echo "slurm dir is defined as $SLURM_DIR"
+./regression > /dev/null 2> $OUTPUT_LOG
+failures="$(sed -n 's/Failures:   \(.*\)/\1/p' $OUTPUT_LOG)"
+if (( $failures > 0 ));then
+	echo "$failures failures found, refer to $OUTPUT_LOG for log"
+	exit 1
+fi
+exit 0
+

CICD/tests/ManagementNodes/check.yml

+---
+- hosts: ManagementNodes
+  gather_facts: false
+  tasks:
+  - name: have ssh running
+    service:
+      name: sshd
+      state: started
\ No newline at end of file

CICD/tests/Readme.md

+this folder should contain tests that will be run automatically by the CICD pipeline
+
+all files with fileending .sh will be executed by a shell
+all files with fileending yml will be executed by ansible-playbook
+./tmp can be used as temporary folder and will be cleaned after execution
+
+because I can I am prefixing tests with 0-9 to give the execution some priority
\ No newline at end of file

CICD/tests/all/0_EXAMPLE_FALSE.sh

+#!/bin/bash
+/bin/false
+
+status=$?
+[ $status -eq 1 ] 
\ No newline at end of file

CICD/tests/all/0_EXAMPLE_TRUE.sh

+#!/bin/bash
+
+/bin/true
+
+status=$?
+[ $status -eq 0 ] 

CICD/tests/run_tests.sh

+#!/bin/bash
+
+function usage {
+    echo $"Usage: $0 {all, ComputeNodes, LoginNodes, ManagementNodes, NFSNodes, sql}" INVENTORY_FILE KEY
+    exit 1
+}
+
+function run_them () 
+{
+    #limit='--limit '"$1"
+    #if [ "$1" = "all" ]
+    #then
+    #  limit="all"
+    #fi
+    for filename in ./tests/$1/*.sh; do   # this is not sorted yet
+        [ -e "$filename" ] || continue
+        #/bin/bash -x $filename # local execution. nice for dev
+        ansible -i $2 --key-file $3 -m script -a "$filename" $1
+    done
+    for filename in ./tests/$1/*.yml; do   # this is not sorted yet
+        [ -e "$filename" ] || continue
+        ansible-playbook -i $2 --key-file $3 $filename # I am assuming the playbook cares about visibility here. might have to change later
+    done
+}
+
+# I think I am just checking the if $1 is one of the listes strings (see usage) not proud of this at all but works
+case "$1" in
+        all)
+        ;;
+        ComputeNodes)
+        ;;
+        ManagementNodes)
+        ;;
+        NFSNodes)
+        ;;
+        SQLNodes)
+        ;;
+        LoginNodes)
+        ;;
+        *)
+        usage
+esac
+
+run_them $1 $2 $3
\ No newline at end of file