From 79e51991e5a17d325af63794427ddc886f301daf Mon Sep 17 00:00:00 2001
From: handreas <andreas.hamacher@monash.edu>
Date: Thu, 30 Jan 2020 10:17:49 +0000
Subject: [PATCH] pipeline fixes and more tests
Former-commit-id: 3c5da4d60f874f22b1806b7a45bb5259ac2aeb72
---
.gitlab-ci.yml | 41 +++++++++--------
.../LoginNodes/run_slurm_testsuite.inactive | 29 ++++++++++++
CICD/tests/ManagementNodes/check.yml | 8 ++++
CICD/tests/Readme.md | 7 +++
CICD/tests/all/0_EXAMPLE_FALSE.sh | 5 +++
CICD/tests/all/0_EXAMPLE_TRUE.sh | 6 +++
CICD/tests/run_tests.sh | 44 +++++++++++++++++++
7 files changed, 122 insertions(+), 18 deletions(-)
create mode 100755 CICD/tests/LoginNodes/run_slurm_testsuite.inactive
create mode 100644 CICD/tests/ManagementNodes/check.yml
create mode 100644 CICD/tests/Readme.md
create mode 100755 CICD/tests/all/0_EXAMPLE_FALSE.sh
create mode 100755 CICD/tests/all/0_EXAMPLE_TRUE.sh
create mode 100644 CICD/tests/run_tests.sh
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 48993503..5704eb68 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -106,7 +106,7 @@ ansible_create_cluster_stage:
- echo "ansible_create_cluster_stage"
- bash -x ./CICD/ansible_create_cluster_script.sh
#after_script:
- #- rm ./files/cicd_inventory
+ #- rm ./files/inventory.$STACKNAME
#only:
# changes: #https://docs.gitlab.com/ee/ci/yaml/#onlychangesexceptchanges
# - "master_playbook.yml"
@@ -124,24 +124,29 @@ tests:
- echo "$HPCCICD_openrc" > ./$NECTAR_ALLOCATION-openrc.sh
script:
- echo "tests stage"
- - export ANSIBLE_HOST_KEY_CHECKING=False
- source ./$NECTAR_ALLOCATION-openrc.sh
- - python3 dependencies/ansible_cluster_in_a_box/scripts/make_inventory.py static CICD$STACKNAME | tee ./files/cicd_inventory
- - grep -qv "I could not find any resouces tagged with project_name:" ./files/cicd_inventory #fail if inventory file is empty
- - chmod 755 ./files/cicd_inventory
- - chmod 400 ./keys/gc_key.pem
- - ansible -B 30 -i files/cicd_inventory --key-file keys/gc_key.pem -a "sinfo" ManagementNodes
- - ansible -B 30 -i files/cicd_inventory --key-file keys/gc_key.pem -a "squeue" ManagementNodes
- - ansible -B 30 -i files/cicd_inventory --key-file keys/gc_key.pem -a "systemctl is-active --quiet mariadb" SQLNodes
- - ansible -B 30 -i files/cicd_inventory --key-file keys/gc_key.pem -a "systemctl is-active --quiet slurmctld" ManagementNodes
- - ansible -B 30 -i files/cicd_inventory --key-file keys/gc_key.pem -a "systemctl is-active --quiet slurmdbd" ManagementNodes
+ - openstack stack list
+ - cd CICD
+ - STACKNAME=CICD$STACKNAME
+ - export ANSIBLE_HOST_KEY_CHECKING=False
+ - echo "[ansible_create_cluster_script] Prefixing the Stackname with CICD because heatcicdwrapper does he same"
+ - python3 ../scripts/make_inventory.py static $STACKNAME | tee ./files/inventory.$STACKNAME && chmod 755 ./files/inventory.$STACKNAME
+ - grep -qv "I could not find any resouces tagged with project_name:" ./files/inventory.$STACKNAME #fail if inventory file is empty
+ - ansible -m ping -i files/inventory.$STACKNAME --key-file ../gc_key.pem all
+ - ansible -i files/inventory.$STACKNAME --key-file ../gc_key.pem -a "sudo ls" all
+
+ - ansible -B 30 -i files/inventory.$STACKNAME --key-file ../gc_key.pem -a "sinfo" ManagementNodes
+ - ansible -B 30 -i files/inventory.$STACKNAME --key-file ../gc_key.pem -a "squeue" ManagementNodes
+ - ansible -B 30 -i files/inventory.$STACKNAME --key-file ../gc_key.pem -a "systemctl is-active --quiet mariadb" SQLNodes
+ - ansible -B 30 -i files/inventory.$STACKNAME --key-file ../gc_key.pem -a "systemctl is-active --quiet slurmctld" ManagementNodes
+ - ansible -B 30 -i files/inventory.$STACKNAME --key-file ../gc_key.pem -a "systemctl is-active --quiet slurmdbd" ManagementNodes
- - bash -e ./tests/run_tests.sh all "files/cicd_inventory" "keys/gc_key.pem"
- - bash -e ./tests/run_tests.sh ComputeNodes "files/cicd_inventory" "keys/gc_key.pem"
- - bash -e ./tests/run_tests.sh LoginNodes "files/cicd_inventory" "keys/gc_key.pem"
- - bash -e ./tests/run_tests.sh ManagementNodes "files/cicd_inventory" "keys/gc_key.pem"
- - bash -e ./tests/run_tests.sh NFSNodes "files/cicd_inventory" "keys/gc_key.pem"
- - bash -e ./tests/run_tests.sh SQLNodes "files/cicd_inventory" "keys/gc_key.pem"
+ - bash -e ./tests/run_tests.sh all "files/inventory.$STACKNAME" "../gc_key.pem"
+ - bash -e ./tests/run_tests.sh ComputeNodes "files/inventory.$STACKNAME" "../gc_key.pem"
+ - bash -e ./tests/run_tests.sh LoginNodes "files/inventory.$STACKNAME" "../gc_key.pem"
+ - bash -e ./tests/run_tests.sh ManagementNodes "files/inventory.$STACKNAME" "../gc_key.pem"
+ - bash -e ./tests/run_tests.sh NFSNodes "files/inventory.$STACKNAME" "../gc_key.pem"
+ - bash -e ./tests/run_tests.sh SQLNodes "files/inventory.$STACKNAME" "../gc_key.pem"
# licensing https://gitlab.erc.monash.edu.au/hpc-team/license_server/tree/master/roles/avizo_license_monitor
@@ -172,7 +177,7 @@ manual_cluster_spawn:
when: manual
only:
refs:
- - "devel"
+ - "cicd"
extended:
stage: extended
diff --git a/CICD/tests/LoginNodes/run_slurm_testsuite.inactive b/CICD/tests/LoginNodes/run_slurm_testsuite.inactive
new file mode 100755
index 00000000..c5d2f24f
--- /dev/null
+++ b/CICD/tests/LoginNodes/run_slurm_testsuite.inactive
@@ -0,0 +1,29 @@
+#!/bin/bash
+OUTPUT_LOG=$(realpath ${1-slurmtest.out})
+if ! type "scontrol" > /dev/null; then
+ echo "cannot find slurm"
+ exit 1
+fi
+SLURM_DIR=${2-$(dirname $(dirname $(which scontrol)))}
+#SLURM_DIR=$slurm_dir
+
+#if [[ -d $2 ]];then
+# SLURM_SRC_DIR=$2
+#else
+# SLURM_SRC_DIR=./slurm_src
+# git clone https://github.com/SchedMD/slurm.git $SLURM_SRC_DIR
+# cd $SLURM_SRC_DIR && ./configure
+#fi
+#cd $SLURM_SRC_DIR/testsuite/expect
+#echo -en "set slurm_dir=$SLURM_DIR\nset max_job_delay 300\n" > globals.local
+#make
+#echo "log is written to $OUTPUT_LOG"
+#echo "slurm dir is defined as $SLURM_DIR"
+./regression > /dev/null 2> $OUTPUT_LOG
+failures="$(sed -n 's/Failures: \(.*\)/\1/p' $OUTPUT_LOG)"
+if (( $failures > 0 ));then
+ echo "$failures failures found, refer to $OUTPUT_LOG for log"
+ exit 1
+fi
+exit 0
+
diff --git a/CICD/tests/ManagementNodes/check.yml b/CICD/tests/ManagementNodes/check.yml
new file mode 100644
index 00000000..95e06a0a
--- /dev/null
+++ b/CICD/tests/ManagementNodes/check.yml
@@ -0,0 +1,8 @@
+---
+- hosts: ManagementNodes
+ gather_facts: false
+ tasks:
+ - name: have ssh running
+ service:
+ name: sshd
+ state: started
\ No newline at end of file
diff --git a/CICD/tests/Readme.md b/CICD/tests/Readme.md
new file mode 100644
index 00000000..feca0626
--- /dev/null
+++ b/CICD/tests/Readme.md
@@ -0,0 +1,7 @@
+this folder should contain tests that will be run automatically by the CICD pipeline
+
+all files with fileending .sh will be executed by a shell
+all files with fileending yml will be executed by ansible-playbook
+./tmp can be used as temporary folder and will be cleaned after execution
+
+because I can I am prefixing tests with 0-9 to give the execution some priority
\ No newline at end of file
diff --git a/CICD/tests/all/0_EXAMPLE_FALSE.sh b/CICD/tests/all/0_EXAMPLE_FALSE.sh
new file mode 100755
index 00000000..10c48607
--- /dev/null
+++ b/CICD/tests/all/0_EXAMPLE_FALSE.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+/bin/false
+
+status=$?
+[ $status -eq 1 ]
\ No newline at end of file
diff --git a/CICD/tests/all/0_EXAMPLE_TRUE.sh b/CICD/tests/all/0_EXAMPLE_TRUE.sh
new file mode 100755
index 00000000..3634c7aa
--- /dev/null
+++ b/CICD/tests/all/0_EXAMPLE_TRUE.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+/bin/true
+
+status=$?
+[ $status -eq 0 ]
diff --git a/CICD/tests/run_tests.sh b/CICD/tests/run_tests.sh
new file mode 100644
index 00000000..d063e98d
--- /dev/null
+++ b/CICD/tests/run_tests.sh
@@ -0,0 +1,44 @@
+#!/bin/bash
+
+function usage {
+ echo $"Usage: $0 {all, ComputeNodes, LoginNodes, ManagementNodes, NFSNodes, sql}" INVENTORY_FILE KEY
+ exit 1
+}
+
+function run_them ()
+{
+ #limit='--limit '"$1"
+ #if [ "$1" = "all" ]
+ #then
+ # limit="all"
+ #fi
+ for filename in ./tests/$1/*.sh; do # this is not sorted yet
+ [ -e "$filename" ] || continue
+ #/bin/bash -x $filename # local execution. nice for dev
+ ansible -i $2 --key-file $3 -m script -a "$filename" $1
+ done
+ for filename in ./tests/$1/*.yml; do # this is not sorted yet
+ [ -e "$filename" ] || continue
+ ansible-playbook -i $2 --key-file $3 $filename # I am assuming the playbook cares about visibility here. might have to change later
+ done
+}
+
+# I think I am just checking the if $1 is one of the listes strings (see usage) not proud of this at all but works
+case "$1" in
+ all)
+ ;;
+ ComputeNodes)
+ ;;
+ ManagementNodes)
+ ;;
+ NFSNodes)
+ ;;
+ SQLNodes)
+ ;;
+ LoginNodes)
+ ;;
+ *)
+ usage
+esac
+
+run_them $1 $2 $3
\ No newline at end of file
--
GitLab