Merge branch 'cicd' into 'master'

Cicd merge back

See merge request hpc-team/ansible_cluster_in_a_box!297

Former-commit-id: 2d3d261e

.gitlab-ci.yml

 variables:
   GIT_SUBMODULE_STRATEGY: recursive
-  STACKNAME: CICD_reporef$CI_COMMIT_REF_NAME
+  STACKNAME: CICD_aciab_$CI_COMMIT_REF_NAME
   NECTAR_ALLOCATION: HPCCICD
   ANSIBLE_HOST_KEY_CHECKING: "False"
+  DEFAULT_PATH: "CICD"
 
 stages:
-#  - integration_test_downstream # working but unwanted here
-#  - trigger_pipeline_in_B   # working but unwanted here
   - lint
-  #- delete_stack_manual
   - extended
   #- heat_test
   - heat
   - ansible_create_cluster_stage
   - push_button_spawn_cluster
-#  - e2e
   - tests
-  - clean # manually delete stack
+  - integration_test #https://docs.gitlab.com/ee/ci/triggers/
+  - clean
+  
+ 
+trigger_pipeline_in_Clusterbuild:
+  stage: integration_test
+  tags: 
+  - ansible
+  script:
+  - echo ${CI_JOB_TOKEN}
+  - curl --request POST --form token=${CI_JOB_TOKEN} --form "variables[TRIGGER_CI_COMMIT_SHA]=${CI_COMMIT_SHA}" --form ref=aciab_upstream https://gitlab.erc.monash.edu.au/api/v4/projects/193/trigger/pipeline  # ID is from clusterbuild
 
 
-  
-#trigger_pipeline_in_B:
-#  stage: integration_test_downstream
-#  tags: 
-#  - ansible
-#  script:
-#  - "curl --request POST --form token=${CI_JOB_TOKEN} --form ref=master https://gitlab.erc.monash.edu.au/api/v4/projects/1085/trigger/pipeline"  # ID is from pysshauthz
+trigger_pipeline_in_monarch:
+  stage: integration_test
+  tags: 
+  - ansible
+  script:
+  - echo ${CI_JOB_TOKEN}
+  - curl --request POST --form token=${CI_JOB_TOKEN} --form "variables[TRIGGER_CI_COMMIT_SHA]=${CI_COMMIT_SHA}" --form ref=cicd https://gitlab.erc.monash.edu.au/api/v4/projects/385/trigger/pipeline  # ID is from monarch
 
-# heat_test:
-#   stage: heat_test
-#   allow_failure: false
-#   tags:
-#   - heat
-#   before_script:
-#     - echo "$GC_KEY" > gc_key.pem
-#     - chmod 400 gc_key.pem
-#     - echo "$HPCCICD_openrc" > ./$NECTAR_ALLOCATION-openrc.sh
-#     - source ./$NECTAR_ALLOCATION-openrc.sh
-#     - export HEAT_TEST_STACKNAME=_TESTING_HEAT
-#     - bash -x ./CICD/heat/heatcicdwrapper.sh delete_if_exists $HEAT_TEST_STACKNAME
-#     - sleep 60
-#   script:
-#     - echo "heat_test stage"
-#     - source ./$NECTAR_ALLOCATION-openrc.sh
-#     - bash -x ./CICD/heat/heatcicdwrapper.sh create $HEAT_TEST_STACKNAME
-#     - openstack stack list
-#     - bash -x ./CICD/heat/heatcicdwrapper.sh update $HEAT_TEST_STACKNAME
-#     - openstack stack list
-#     - bash -x ./CICD/heat/heatcicdwrapper.sh delete_if_exists $HEAT_TEST_STACKNAME
-#     - openstack stack list
-#   after_script:
-#     - sleep 20 # artifically wait a bit to make sure it is really dead
-#   when: manual
 
 yamllint:
   stage: lint
@@ -61,7 +43,6 @@ yamllint:
   script:
     - echo "stage yamllint"
     - cd CICD
-    # - ansible-lint -c .yamllintconf.yaml -x ANSIBLE0002 master_playbook.yml
     - yamllint -c ./.yamllintheat.yaml ./heat
 
 # delete_stack_manual:
@@ -96,6 +77,7 @@ build_cluster_cicd:
   tags:
   - heat
   before_script:
+    - cd $DEFAULT_PATH
     - echo "$GC_KEY" > gc_key.pem
     - chmod 400 gc_key.pem
     - echo "$HPCCICD_openrc" > ./$NECTAR_ALLOCATION-openrc.sh
@@ -103,9 +85,10 @@ build_cluster_cicd:
     - echo "heat stage"
     - source ./$NECTAR_ALLOCATION-openrc.sh
     - openstack stack list
-    - bash -x ./CICD/heat/heatcicdwrapper.sh create_or_update $STACKNAME
+    - bash -x ./heat/heatcicdwrapper.sh create_or_update $STACKNAME
+    - bash -x ./heat/server_rebuild.sh all
   after_script:
-    - sleep 20 # artifically wait a bit to give the nodes time to boot
+    - sleep 30 # artifically wait a bit to give the nodes time to boot
 #  only:
 #    changes: #https://docs.gitlab.com/ee/ci/yaml/#onlychangesexceptchanges
 #    - "heat/*HOT*.yaml"
@@ -117,20 +100,22 @@ ansible_create_cluster_stage:
   tags: 
   - ansible
   before_script:
+    - cd $DEFAULT_PATH/..
     - echo "$GC_KEY" > gc_key.pem
     - chmod 400 gc_key.pem
     - echo "$HPCCICD_openrc" > ./$NECTAR_ALLOCATION-openrc.sh
   script:
     - echo "ansible_create_cluster_stage"
     - bash -x ./CICD/ansible_create_cluster_script.sh
-  #after_script:
-    #- rm ./files/inventory.$STACKNAME
-  #only:
-  #  changes: #https://docs.gitlab.com/ee/ci/yaml/#onlychangesexceptchanges
-  #  - "master_playbook.yml"
-  #  - "vars/*.{yml,yaml}"
-  #  - schedules
-  #  - CICD/.gitlab-ci.yml
+    - cd CICD
+    - ansible-playbook -i files/inventory.$STACKNAME --key-file ../gc_key.pem --skip-tags monitoring master_playbook.yml
+    - sleep 15
+    - echo uglyuglyfix
+    - ansible -i files/inventory.$STACKNAME --key-file ../gc_key.pem -b -a "systemctl restart slurmdbd" ManagementNodes
+    - sleep 60
+    - echo do it again
+    - ansible-playbook -i files/inventory.$STACKNAME --key-file ../gc_key.pem --skip-tags monitoring master_playbook.yml
+    
 
 tests:
   stage: tests
@@ -163,7 +148,22 @@ tests:
     - bash -e ./tests/run_tests.sh NFSNodes "files/inventory.$STACKNAME" "../gc_key.pem"
     - bash -e ./tests/run_tests.sh SQLNodes "files/inventory.$STACKNAME" "../gc_key.pem"
     
-    # licensing https://gitlab.erc.monash.edu.au/hpc-team/license_server/tree/master/roles/avizo_license_monitor
+extended:
+  stage: extended
+  tags:
+  - heat
+  - ansible
+  before_script:
+    - echo "cleanup stack"
+    - sleep 30
+    - echo "$HPCCICD_openrc" > ./$NECTAR_ALLOCATION-openrc.sh
+  script:
+    - source ./$NECTAR_ALLOCATION-openrc.sh
+    - bash -x ./CICD/heat/heatcicdwrapper.sh delete_if_exists $STACKNAME
+  only:
+    variables:
+      - $EXTENDED != null
+  
 
 manual_cluster_spawn:
   stage: push_button_spawn_cluster
@@ -189,22 +189,6 @@ manual_cluster_spawn:
   only:
     refs:
       - "cicd"
-
-extended:
-  stage: extended
-  tags:
-  - heat
-  - ansible
-  before_script:
-    - echo "cleanup stack"
-    - sleep 30
-    - echo "$HPCCICD_openrc" > ./$NECTAR_ALLOCATION-openrc.sh
-  script:
-    - source ./$NECTAR_ALLOCATION-openrc.sh
-    - bash -x ./CICD/heat/heatcicdwrapper.sh delete_if_exists $STACKNAME
-  only:
-    variables:
-      - $EXTENDED != null
   
 clean:
   stage: clean
@@ -217,4 +201,30 @@ clean:
   script:
     - source ./$NECTAR_ALLOCATION-openrc.sh
     - bash -x ./CICD/heat/heatcicdwrapper.sh delete_if_exists $STACKNAME
-  #when: manual
+
\ No newline at end of file
+  
+# heat_test:
+#   stage: heat_test
+#   allow_failure: false
+#   tags:
+#   - heat
+#   before_script:
+#     - echo "$GC_KEY" > gc_key.pem
+#     - chmod 400 gc_key.pem
+#     - echo "$HPCCICD_openrc" > ./$NECTAR_ALLOCATION-openrc.sh
+#     - source ./$NECTAR_ALLOCATION-openrc.sh
+#     - export HEAT_TEST_STACKNAME=_TESTING_HEAT
+#     - bash -x ./CICD/heat/heatcicdwrapper.sh delete_if_exists $HEAT_TEST_STACKNAME
+#     - sleep 60
+#   script:
+#     - echo "heat_test stage"
+#     - source ./$NECTAR_ALLOCATION-openrc.sh
+#     - bash -x ./CICD/heat/heatcicdwrapper.sh create $HEAT_TEST_STACKNAME
+#     - openstack stack list
+#     - bash -x ./CICD/heat/heatcicdwrapper.sh update $HEAT_TEST_STACKNAME
+#     - openstack stack list
+#     - bash -x ./CICD/heat/heatcicdwrapper.sh delete_if_exists $HEAT_TEST_STACKNAME
+#     - openstack stack list
+#   after_script:
+#     - sleep 20 # artifically wait a bit to make sure it is really dead
+

CICD/ChordDiagramm/Chord_Diagramm - Sheet1.csv

+0,0,0,1,1,1,1,1,1,0
+0,0,0,0,0,0,1,1,0,0
+0,0,0,0,1,1,1,1,0,0
+1,0,0,0,0,0,0,1,0,0
+1,0,1,0,0,0,0,1,0,0
+1,0,1,0,0,0,0,1,0,0
+1,1,1,0,0,0,0,1,1,0
+1,1,1,1,1,1,1,0,0,0
+1,0,0,0,0,0,1,0,0,0
+0,0,0,0,0,0,0,0,0,0
\ No newline at end of file

CICD/ChordDiagramm/genChordDiagramm.py

+#!/usr/bin/env python3
+# script copied from https://github.com/fengwangPhysics/matplotlib-chord-diagram/blob/master/README.md
+# source data manually edited via https://docs.google.com/spreadsheets/d/1JN9S_A5ICPQOvgyVbWJSFJiw-5gO2vF-4AeYuWl-lbs/edit#gid=0
+# chord diagram
+import matplotlib.pyplot as plt
+from matplotlib.path import Path
+import matplotlib.patches as patches
+
+import numpy as np
+
+LW = 0.3
+
+def polar2xy(r, theta):
+    return np.array([r*np.cos(theta), r*np.sin(theta)])
+
+def hex2rgb(c):
+    return tuple(int(c[i:i+2], 16)/256.0 for i in (1, 3 ,5))
+
+def IdeogramArc(start=0, end=60, radius=1.0, width=0.2, ax=None, color=(1,0,0)):
+    # start, end should be in [0, 360)
+    if start > end:
+        start, end = end, start
+    start *= np.pi/180.
+    end *= np.pi/180.
+    # optimal distance to the control points
+    # https://stackoverflow.com/questions/1734745/how-to-create-circle-with-b%C3%A9zier-curves
+    opt = 4./3. * np.tan((end-start)/ 4.) * radius
+    inner = radius*(1-width)
+    verts = [
+        polar2xy(radius, start),
+        polar2xy(radius, start) + polar2xy(opt, start+0.5*np.pi),
+        polar2xy(radius, end) + polar2xy(opt, end-0.5*np.pi),
+        polar2xy(radius, end),
+        polar2xy(inner, end),
+        polar2xy(inner, end) + polar2xy(opt*(1-width), end-0.5*np.pi),
+        polar2xy(inner, start) + polar2xy(opt*(1-width), start+0.5*np.pi),
+        polar2xy(inner, start),
+        polar2xy(radius, start),
+        ]
+
+    codes = [Path.MOVETO,
+             Path.CURVE4,
+             Path.CURVE4,
+             Path.CURVE4,
+             Path.LINETO,
+             Path.CURVE4,
+             Path.CURVE4,
+             Path.CURVE4,
+             Path.CLOSEPOLY,
+             ]
+
+    if ax == None:
+        return verts, codes
+    else:
+        path = Path(verts, codes)
+        patch = patches.PathPatch(path, facecolor=color+(0.5,), edgecolor=color+(0.4,), lw=LW)
+        ax.add_patch(patch)
+
+
+def ChordArc(start1=0, end1=60, start2=180, end2=240, radius=1.0, chordwidth=0.7, ax=None, color=(1,0,0)):
+    # start, end should be in [0, 360)
+    if start1 > end1:
+        start1, end1 = end1, start1
+    if start2 > end2:
+        start2, end2 = end2, start2
+    start1 *= np.pi/180.
+    end1 *= np.pi/180.
+    start2 *= np.pi/180.
+    end2 *= np.pi/180.
+    opt1 = 4./3. * np.tan((end1-start1)/ 4.) * radius
+    opt2 = 4./3. * np.tan((end2-start2)/ 4.) * radius
+    rchord = radius * (1-chordwidth)
+    verts = [
+        polar2xy(radius, start1),
+        polar2xy(radius, start1) + polar2xy(opt1, start1+0.5*np.pi),
+        polar2xy(radius, end1) + polar2xy(opt1, end1-0.5*np.pi),
+        polar2xy(radius, end1),
+        polar2xy(rchord, end1),
+        polar2xy(rchord, start2),
+        polar2xy(radius, start2),
+        polar2xy(radius, start2) + polar2xy(opt2, start2+0.5*np.pi),
+        polar2xy(radius, end2) + polar2xy(opt2, end2-0.5*np.pi),
+        polar2xy(radius, end2),
+        polar2xy(rchord, end2),
+        polar2xy(rchord, start1),
+        polar2xy(radius, start1),
+        ]
+
+    codes = [Path.MOVETO,
+             Path.CURVE4,
+             Path.CURVE4,
+             Path.CURVE4,
+             Path.CURVE4,
+             Path.CURVE4,
+             Path.CURVE4,
+             Path.CURVE4,
+             Path.CURVE4,
+             Path.CURVE4,
+             Path.CURVE4,
+             Path.CURVE4,
+             Path.CURVE4,
+             ]
+
+    if ax == None:
+        return verts, codes
+    else:
+        path = Path(verts, codes)
+        patch = patches.PathPatch(path, facecolor=color+(0.5,), edgecolor=color+(0.4,), lw=LW)
+        ax.add_patch(patch)
+
+def selfChordArc(start=0, end=60, radius=1.0, chordwidth=0.7, ax=None, color=(1,0,0)):
+    # start, end should be in [0, 360)
+    if start > end:
+        start, end = end, start
+    start *= np.pi/180.
+    end *= np.pi/180.
+    opt = 4./3. * np.tan((end-start)/ 4.) * radius
+    rchord = radius * (1-chordwidth)
+    verts = [
+        polar2xy(radius, start),
+        polar2xy(radius, start) + polar2xy(opt, start+0.5*np.pi),
+        polar2xy(radius, end) + polar2xy(opt, end-0.5*np.pi),
+        polar2xy(radius, end),
+        polar2xy(rchord, end),
+        polar2xy(rchord, start),
+        polar2xy(radius, start),
+        ]
+
+    codes = [Path.MOVETO,
+             Path.CURVE4,
+             Path.CURVE4,
+             Path.CURVE4,
+             Path.CURVE4,
+             Path.CURVE4,
+             Path.CURVE4,
+             ]
+
+    if ax == None:
+        return verts, codes
+    else:
+        path = Path(verts, codes)
+        patch = patches.PathPatch(path, facecolor=color+(0.5,), edgecolor=color+(0.4,), lw=LW)
+        ax.add_patch(patch)
+
+def chordDiagram(X, ax, colors=None, width=0.1, pad=2, chordwidth=0.7):
+    """Plot a chord diagram
+    Parameters
+    ----------
+    X :
+        flux data, X[i, j] is the flux from i to j
+    ax :
+        matplotlib `axes` to show the plot
+    colors : optional
+        user defined colors in rgb format. Use function hex2rgb() to convert hex color to rgb color. Default: d3.js category10
+    width : optional
+        width/thickness of the ideogram arc
+    pad : optional
+        gap pad between two neighboring ideogram arcs, unit: degree, default: 2 degree
+    chordwidth : optional
+        position of the control points for the chords, controlling the shape of the chords
+    """
+    # X[i, j]:  i -> j
+    x = X.sum(axis = 1) # sum over rows
+    ax.set_xlim(-1.1, 1.1)
+    ax.set_ylim(-1.1, 1.1)
+
+    if colors is None:
+    # use d3.js category10 https://github.com/d3/d3-3.x-api-reference/blob/master/Ordinal-Scales.md#category10
+        colors = ['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd',
+                  '#8c564b', '#e377c2', '#7f7f7f', '#bcbd22', '#17becf', '#c49c94']
+        if len(x) > len(colors):
+            print('x is too large! Use x smaller than 11')
+        colors = [hex2rgb(colors[i]) for i in range(len(x))]
+
+    # find position for each start and end
+    y = x/np.sum(x).astype(float) * (360 - pad*len(x))
+
+    pos = {}
+    arc = []
+    nodePos = []
+    start = 0
+    for i in range(len(x)):
+        end = start + y[i]
+        arc.append((start, end))
+        angle = 0.5*(start+end)
+        #print(start, end, angle)
+        if -30 <= angle <= 210:
+            angle -= 90
+        else:
+            angle -= 270
+        nodePos.append(tuple(polar2xy(1.1, 0.5*(start+end)*np.pi/180.)) + (angle,))
+        z = (X[i, :]/x[i].astype(float)) * (end - start)
+        ids = np.argsort(z)
+        z0 = start
+        for j in ids:
+            pos[(i, j)] = (z0, z0+z[j])
+            z0 += z[j]
+        start = end + pad
+
+    for i in range(len(x)):
+        start, end = arc[i]
+        IdeogramArc(start=start, end=end, radius=1.0, ax=ax, color=colors[i], width=width)
+        start, end = pos[(i,i)]
+        selfChordArc(start, end, radius=1.-width, color=colors[i], chordwidth=chordwidth*0.7, ax=ax)
+        for j in range(i):
+            color = colors[i]
+            if X[i, j] > X[j, i]:
+                color = colors[j]
+            start1, end1 = pos[(i,j)]
+            start2, end2 = pos[(j,i)]
+            ChordArc(start1, end1, start2, end2,
+                     radius=1.-width, color=colors[i], chordwidth=chordwidth, ax=ax)
+
+    #print(nodePos)
+    return nodePos
+
+##################################
+if __name__ == "__main__":
+    fig = plt.figure(figsize=(6,6))
+    flux = np.array([
+      [ 0, 1, 0, 0], #OS Sum:2 ; Centos, Ubuntu
+      [ 0, 0, 0, 0], #Plays
+      [ 0, 0, 0, 1], # Cluster: Sum5; Generic, M3, Monarch, SHPC, ACCS
+      [ 0, 0, 1, 2]  #Cloud Sum3: AWS,Nimbus,Nectar
+    ])
+    from numpy import genfromtxt
+
+    flux = genfromtxt('Chord_Diagramm - Sheet1.csv', delimiter=',')
+    ax = plt.axes([0,0,1,1])
+
+    #nodePos = chordDiagram(flux, ax, colors=[hex2rgb(x) for x in ['#666666', '#66ff66', '#ff6666', '#6666ff']])
+    nodePos = chordDiagram(flux, ax)
+    ax.axis('off')
+    prop = dict(fontsize=16*0.8, ha='center', va='center')
+    nodes = ['OS_Centos76','OS_Centos8','OS_Ubuntu1804','PLY_NFSSQL','PLY_MGMT','PLY_Login','PLY_Compute','C_Generic','C_M3','C_Monarch']
+    #nodes = ['M3_MONARCH','SHPC','Ubuntu','Centos7','Centos8','Tested','Security','Nectar','?AWS?','DGX@Baremetal','ML@M3','CVL@UWA','CVL_SW','CVL_Desktop','Strudel','/usr/local']
+    for i in range(len(nodes)):
+        ax.text(nodePos[i][0], nodePos[i][1], nodes[i], rotation=nodePos[i][2], **prop)
+
+    plt.savefig("Chord_Diagramm.png", dpi=600,transparent=False,bbox_inches='tight', pad_inches=0.02)
+    plt.show(pad_inches=0.02)

CICD/ansible_create_cluster_script.sh

@@ -15,7 +15,3 @@ ansible -i files/inventory.$STACKNAME --key-file ../gc_key.pem -a "sudo ls" all
 #cd roles 
     #- "egrep -lRZ 'sudo: true' . | xargs -0 -l sed -i -e 's/sudo: true/become: true/g' "
 #cd ..
-ansible-playbook -i files/inventory.$STACKNAME --key-file ../gc_key.pem master_playbook.yml
-sleep 15
-echo uglyuglyfix
-ansible -i files/inventory.$STACKNAME --key-file ../gc_key.pem -b -a "systemctl restart slurmdbd" ManagementNodes
\ No newline at end of file

CICD/heat/gc_HOT.yaml

@@ -27,7 +27,7 @@ parameters:
     default: Classic Provider
   Flavour:
     type: string
-    default: t3.xsmall
+    default: m3.xsmall
 
 
 resources:
@@ -67,7 +67,8 @@ resources:
      name: "heatmysqlsecgroup"
      rules: [ { protocol: tcp,
                port_range_min: 3306,
-               port_range_max: 3306} ]
+               port_range_max: 3306,
+               remote_mode: "remote_group_id"} ]
   SSHMonashSecGroup:
    type: "OS::Neutron::SecurityGroup"
    properties:
@@ -104,7 +105,7 @@ resources:
     name:
      list_join: [ '-', [ { get_param: "OS::stack_name" }, 'sql0' ]]
     availability_zone: { get_param: avz }
-    flavor: m3.small
+    flavor: m3.xsmall
     image: { get_param: centos_7_image_id }
     key_name: { get_param: ssh_key }
     security_groups: [ { get_resource: SSHMonashSecGroup }, { get_resource: SlurmSecGroup }, { get_resource: MySQLSecGroup }, { get_resource: NFSSecGroup } ]
@@ -257,7 +258,7 @@ resources:
 #    name:
 #     list_join: [ '-', [ { get_param: "OS::stack_name" }, 'pysshautz' ]]
 #    availability_zone: { get_param: avz }
-#    flavor: t3.xsmall
+#    flavor: m3.xsmall
 #    image: { get_param: ubuntu_1804_image_id }
 #    key_name: { get_param: ssh_key }
 #    security_groups: [ { get_resource: SSHMonashSecGroup }, { get_resource: webaccess } ]

CICD/heat/heatcicdwrapper.sh

@@ -60,7 +60,7 @@ function create_stack {
     echo "I will NOT create existing stack maybe use update"
     exit -44
  fi
- openstack stack create --wait --template ./CICD/heat/gc_HOT.yaml --parameter "project_name=$STACKNAME" -e ./CICD/heat/resource_registry.yaml $STACKNAME
+ openstack stack create --wait --template ./heat/gc_HOT.yaml --parameter "project_name=$STACKNAME" -e ./heat/resource_registry.yaml $STACKNAME
  createreturn=$?
  if [ $createreturn -ne "0" ]
  then
@@ -83,14 +83,14 @@ case "$1" in
                    echo "I cannot update a stack which does not exist"
                    exit -45
             fi
-            openstack stack update --wait --template ./CICD/heat/gc_HOT.yaml --parameter "project_name=$STACKNAME" -e ./CICD/heat/resource_registry.yaml $STACKNAME
+            openstack stack update --wait --template ./heat/gc_HOT.yaml --parameter "project_name=$STACKNAME" -e ./heat/resource_registry.yaml $STACKNAME
             ret=$?
             exit $ret
             ;;
         create_or_update)
             if  check_stack_exists
                then
-               openstack stack update --wait --template ./CICD/heat/gc_HOT.yaml --parameter "project_name=$STACKNAME" -e ./CICD/heat/resource_registry.yaml $STACKNAME
+               openstack stack update --wait --template ./heat/gc_HOT.yaml --parameter "project_name=$STACKNAME" -e ./heat/resource_registry.yaml $STACKNAME
                ret=$?
                exit $ret
             fi

CICD/heat/server_rebuild.sh

+#!/bin/bash
+
+REBOOT_TIME=90
+
+NODETYPES=${@}
+
+# make sure you define variable STACKNAME in current environment
+if [[ -z $STACKNAME ]]; then
+    echo "please define STACKNAME variable"  
+    exit 1
+fi
+
+# prepend CICD to stack name
+if [[ "$STACKNAME" == "CICD"* ]]; then
+  echo "CICD found in stackname. doing nothing"
+else
+  STACKNAME="CICD"$STACKNAME
+fi
+
+function check_stack_exists {
+    if openstack stack list | grep -w $STACKNAME;
+        then 
+            echo "stack found";
+        else 
+            echo "stack not found";
+            return 1
+    fi
+}
+
+if ! check_stack_exists
+then
+    exit 0
+fi
+
+# return and dictionary in json format, which map server name to ansible_host_groups. There will be IndexError if the servers, which names are prefixed by STACKNAME, don't have ansible_host_groups property
+host_group_mapping=$(openstack server list --long -f json | python3 -c "import json,sys,re;ivt=json.load(sys.stdin);json.dump({i['Name']: re.search('ansible_host_groups\=\'\[(.+)\]\'', i['Properties']).groups()[0].replace('\"', '').split() for i in ivt if re.match('$STACKNAME',i['Name'])}, fp=sys.stdout)")
+
+# all available ansible_host_groups
+available_groups=$(echo $host_group_mapping |  python3 -c "import json,sys;mapping=json.load(sys.stdin);output=[];[output.extend(v) for v in mapping.values()];print(output)" | tr -d "[',]")
+
+echo "available groups are $available_groups"
+# if the first input parameter is all then rebuild all groups
+if [[ "$1" == "all" ]]; then
+    NODETYPES=$available_groups
+fi
+echo "going to update group $NODETYPES"
+server_list=$(echo $host_group_mapping | python3 -c "import json,sys;mapping=json.load(sys.stdin);node_types='$NODETYPES'.split();output=[k for k in mapping.keys() if len(set(mapping[k]).intersection(set(node_types))) > 0];print(output)" | tr -d "[,]'" | xargs -n1 | sort -u)
+
+echo "server list is $server_list"
+rebuild_func(){
+    echo "rebuilding server $1"
+    openstack server rebuild --wait $1
+}
+
+# for eaech line in data
+for server in $server_list
+do
+rebuild_func $server & # run parallel in background
+done
+wait # wait for all servers to be rebuild
+# add an extra time for reboot
+echo "waiting for reboot"
+sleep $REBOOT_TIME
+echo "All done"
+

CICD/master_playbook.yml

@@ -5,3 +5,4 @@
 - import_playbook: plays/nfssqlnodes.yml
 - import_playbook: plays/mgmtnodes.yml
 - import_playbook: plays/computenodes.yml
+- import_playbook: plays/loginnodes.yml
\ No newline at end of file

CICD/plays/allnodes.yml

@@ -46,3 +46,4 @@
   - { role: calculateKnownHosts, tags: [ calculateKnownHosts ] }
   - { role: SSHKnownHosts, tags: [ known_hosts ] }
   - { role: jasons_ssh_ca, tags: [ ssh_ca ] }
+  - { role: set_timezone }

CICD/plays/loginnodes.yml

+../tests/ManagementNodes/check.yml
\ No newline at end of file

CICD/plays/mgmtnodes.yml

@@ -38,6 +38,7 @@
   - { role: slurm-common, tags: [ slurm, slurm-common ]  }
   - { role: slurm_config, tags: [ slurm, slurm-config ] }
   - { role: slurm-start, start_slurmdbd: True, start_slurmctld: True, tags: [ slurm-start ]  }
+  - { role: telegraf, tags: [ monitoring ] }
 #  - { role: provision_slurm, use_active_directory: False, lockpath: "/mnt/home", tags: [ slurm ]  }
 #  - { role: provision_homedir, use_active_directory: False, mntpt: "/mnt/home", tags: [ provisioning ] }
 

CICD/tests/Readme.md

-this folder should contain tests that will be run automatically by the CICD pipeline
+This folder should contain tests that will be run automatically by the CICD pipeline.
 
-all files with fileending .sh will be executed by a shell
-all files with fileending yml will be executed by ansible-playbook
-./tmp can be used as temporary folder and will be cleaned after execution
+The trigger mechanism to execute these tests is `.gitlab-ci.yml` in the toplevel folder of this repository.
 
-because I can I am prefixing tests with 0-9 to give the execution some priority
+ - *.sh will be executed by a shell, *.yml will be executed by ansible-playbook
\ No newline at end of file
+ - A test should return 0 as success or otherwise as fail. See 0_Example*
+ - Tests should aim not to alter the system except /tmp
+ - Prefixing tests with 0-9 to give the execution some priority
+ - Tests should not need elevated priviliges. A special user TODO is considered to be present
+ - TODO a way to deal with Test Payload. This will be determined on test submission
\ No newline at end of file

README.md

@@ -8,6 +8,10 @@ as our architecture document.
 
 [![pipeline status](https://gitlab.erc.monash.edu.au/hpc-team/ansible_cluster_in_a_box/badges/cicd/pipeline.svg)](https://gitlab.erc.monash.edu.au/hpc-team/ansible_cluster_in_a_box/commits/cicd)
 
+![Coverage Diagramm](CICD/ChordDiagramm/Chord_Diagramm.png)
+A connecting line represents a test coverage of its two endpoints. E.G. Centos8 is only tested on the Cluster_Generic for ComputeNodes
+
+
 We aim to make these roles as generic as possible. You should be able to start from an inventory file, an ssh key and a git clone of this and end up with a working cluster. In the longer term we might branch to include utilities to make an inventory file using NeCTAR credentials.
 
 If you need a password use get_or_make_password.py (delegated to the passwword server/localhost) to generate a random one that can be shared between nodes

plays/allnodes.yml

-- hosts: 'all'
-  vars_files: 
-  - vars/passwords.yml 
-  - vars/names.yml
-  - vars/ldapConfig.yml
-  - vars/filesystems.yml 
-  - vars/slurm.yml 
-  - vars/vars.yml 
-  tasks:
-  - { name: set use shared state, set_fact: usesharedstatedir=False }
-  - { name: set hostgroup, set_fact: hostgroup='ComputeNodes' }
-  tags: [ always ]
-
-- hosts: 'all'
-  vars_files: 
-  - vars/passwords.yml 
-  - vars/names.yml
-  - vars/ldapConfig.yml
-  - vars/filesystems.yml 
-  - vars/slurm.yml 
-  - vars/vars.yml 
-  strategy: free
-  roles:
-#  - { role: disable_selinux, tags: [ disableselinux ] }
-  - { role: upgrade }
-  - { role: set_password }
-  - { role: etcHosts, tags: [ networking ] }
-#  - { role: config_repos, tags: [ repos ] }
-
-- hosts: 'DesktopNodes,ComputeNodes,LoginNodes,ManagementNodes'
-  vars_files: 
-  - vars/passwords.yml 
-  - vars/names.yml
-  - vars/ldapConfig.yml
-  - vars/filesystems.yml 
-  - vars/slurm.yml 
-  - vars/vars.yml 
-  strategy: free
-  roles:
-  - { role: disable_selinux, tags: [ disableselinux ] }
-  - { role: ldapclient, tags: [ authentication ] }
-  - { role: ssh-password-login, tags: [ authentication ] }
-  - { role: enable_sudo_group, tags: [ authentication, sudo ] }
-  - { role: move_homedir }
-  - { role: calculateKnownHosts, tags: [ calculateKnownHosts ] }
-  - { role: SSHKnownHosts, tags: [ known_hosts ] }
-  - { role: jasons_ssh_ca, tags: [ ssh_ca ] }

plays/computenodes.yml

-
-- hosts: 'DesktopNodes,ComputeNodes,LoginNodes,VisNodes'
-  vars_files: 
-  - vars/passwords.yml 
-  - vars/names.yml
-  - vars/ldapConfig.yml
-  - vars/filesystems.yml 
-  - vars/slurm.yml 
-  - vars/vars.yml 
-  tasks:
-  - { name: set use shared state, set_fact: usesharedstatedir=False }
-  tags: [ always ]
-
-- hosts: 'DesktopNodes,ComputeNodes,LoginNodes'
-  vars_files: 
-  - vars/passwords.yml 
-  - vars/names.yml
-  - vars/ldapConfig.yml
-  - vars/filesystems.yml 
-  - vars/slurm.yml 
-  - vars/vars.yml 
-  strategy: free
-  roles:
-  - { role: move_homedir, tags: [ authentication, filesystems ] }
-  - { role: nfs-client, nfsMounts: "{{ computeNfsMounts }}", tags: [ filesystems ] }
-  - { role: slurm-common, tags: [ slurm, slurm-common ] }
-  - { role: lmod, tags: [ other ] }
-  - { role: enable_modules, default_modules: "lmod", tags: [ other ] }
-  - { role: postfix, tags: [ mail, other ] }
-
-- hosts: 'VisNodes'
-  vars_files:
-  - vars/passwords.yml
-  - vars/names.yml
-  - vars/ldapConfig.yml
-  - vars/filesystems.yml
-  - vars/slurm.yml
-  - vars/vars.yml
-  roles:
-  - { role: gpu, tags: [ gpu ] }
-
-- hosts: 'DesktopNodes,ComputeNodes,LoginNodes'
-  vars_files:
-  - vars/passwords.yml
-  - vars/names.yml
-  - vars/ldapConfig.yml
-  - vars/filesystems.yml
-  - vars/slurm.yml
-  - vars/vars.yml
-  roles:
-  - { role: slurm_config, tags: [slurm, slurm_config] }
-
-- hosts: 'DesktopNodes,ComputeNodes'
-  vars_files:
-  - vars/passwords.yml
-  - vars/names.yml
-  - vars/ldapConfig.yml
-  - vars/filesystems.yml
-  - vars/slurm.yml
-  - vars/vars.yml
-  strategy: free
-  roles:
-  - { role: slurm-start, start_slurmd: True, tags: [ slurm, slurmstart ] }
-  - { role: mate-de-install, tags: [ mate-de-install ] }   # TODO this crashes for everything except cmca

plays/files

-../files
\ No newline at end of file

plays/init_slurmconf.yml

----
-- hosts: 'all'
-  tasks:
-  - include_vars: vars/passwords.yml 
-  - include_vars: vars/names.yml
-  - include_vars: vars/ldapConfig.yml
-  - include_vars: vars/filesystems.yml 
-  - include_vars: vars/slurm.yml 
-  - include_vars: vars/vars.yml 
-- hosts: 'all'
-  tasks:
-  - { name: setup, setup: }
-- hosts: 'ManagementNodes'
-  roles:
-  - { role: calculateSlurmConf }

plays/make_files.yml

----
-# just calculates an etc hosts
-- hosts: 'all'
-  tasks:
-  - include_vars: vars/passwords.yml
-  - include_vars: vars/names.yml
-  - include_vars: vars/ldapConfig.yml
-  - include_vars: vars/filesystems.yml
-  - include_vars: vars/slurm.yml
-  - include_vars: vars/vars.yml
-- hosts: 'all'
-  tasks:
-  - { name: setup, setup: }
-- hosts: 'ManagementNodes'
-  roles:
-  - { role: calculateEtcHosts }
-  
-#- hosts: 'NFSNodes'
-#  roles:
-#  - { role: calculateExports }
-
-

plays/mgmtnodes.yml

-# Basic stuff to make the nodes functionl
-# i.e. upgrade operating systems, etc
-#
-
-- hosts: 'ManagementNodes'
-  vars_files: 
-  - vars/passwords.yml 
-  - vars/names.yml
-  - vars/ldapConfig.yml
-  - vars/filesystems.yml 
-  - vars/slurm.yml 
-  - vars/vars.yml 
-  tasks:
-      #  - { name: set hostgroup, set_fact: hostgroup='ManagementNodes' }
-  - { name: set use shared state, set_fact: usesharedstatedir=True }
-  tags: [ always ]
-
-- hosts: 'ManagementNodes'
-  strategy: free
-  gather_facts: False
-  vars_files: 
-  - vars/passwords.yml 
-  - vars/names.yml
-  - vars/ldapConfig.yml
-  - vars/filesystems.yml 
-  - vars/slurm.yml 
-  - vars/vars.yml 
-  roles:
-#  - { role: ldapclient, tags: [ authentication ] }
-#  - { role: ssh-password-login }
-#  - { role: enable_sudo_group }
-#  - { role: make_filesystems, volumes: "{{ glustervolumes }}" }
-#  - { role: gluster_server, volname: "gv", brickmnt: '/gbrick', gluster_servers: "{{ groups['ManagementNodes'] }}", replicas: 2, tags: [ gluster_server ]  }
-#  - { role: gluster_volcreate, volname: "gv", gluster_servers: "{{ groups['ManagementNodes'] }}", brickmnt: '/gbrick', replicas: 2 }
-#  - { role: gluster_client, volname: "gv", gluster_servers: ['mgmt0','mgmt1','sql0'], volmnt: '/glusterVolume' }
-  - { role: nfs-client, nfsMounts: "{{ mgmtNfsMounts }}", tags: [ nfs ] }
-  - { role: slurmdb-config, tags: [ slurm, slurmdb-config ] }
-  - { role: slurm-common, tags: [ slurm, slurm-common ]  }
-  - { role: slurm_config, tags: [ slurm, slurm-config ] }
-  - { role: slurm-start, start_slurmdbd: True, start_slurmctld: True, tags: [ slurm-start ]  }
-  - { role: telegraf, tags: [ monitoring ] }
-#  - { role: provision_slurm, use_active_directory: False, lockpath: "/mnt/home", tags: [ slurm ]  }
-#  - { role: provision_homedir, use_active_directory: False, mntpt: "/mnt/home", tags: [ provisioning ] }
-