Skip to content
Snippets Groups Projects
Commit 355800ee authored by Chris Hines's avatar Chris Hines
Browse files

Merge branch 'cicd' into 'master' 

Cicd merge back

See merge request hpc-team/ansible_cluster_in_a_box!297

Former-commit-id: 2d3d261e
parents 4d564f5b e9323d92
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 417 additions and 276 deletions
.gitlab-ci.yml
+ 77
− 67
View file @ 355800ee
variables:
GIT_SUBMODULE_STRATEGY: recursive
STACKNAME: CICD_reporef$CI_COMMIT_REF_NAME
STACKNAME: CICD_aciab_$CI_COMMIT_REF_NAME
NECTAR_ALLOCATION: HPCCICD
ANSIBLE_HOST_KEY_CHECKING: "False"
DEFAULT_PATH: "CICD"
stages:
# - integration_test_downstream # working but unwanted here
# - trigger_pipeline_in_B # working but unwanted here
- lint
#- delete_stack_manual
- extended
#- heat_test
- heat
- ansible_create_cluster_stage
- push_button_spawn_cluster
# - e2e
- tests
- clean # manually delete stack
- integration_test #https://docs.gitlab.com/ee/ci/triggers/
- clean
trigger_pipeline_in_Clusterbuild:
stage: integration_test
tags:
- ansible
script:
- echo ${CI_JOB_TOKEN}
- curl --request POST --form token=${CI_JOB_TOKEN} --form "variables[TRIGGER_CI_COMMIT_SHA]=${CI_COMMIT_SHA}" --form ref=aciab_upstream https://gitlab.erc.monash.edu.au/api/v4/projects/193/trigger/pipeline # ID is from clusterbuild
#trigger_pipeline_in_B:
# stage: integration_test_downstream
# tags:
# - ansible
# script:
# - "curl --request POST --form token=${CI_JOB_TOKEN} --form ref=master https://gitlab.erc.monash.edu.au/api/v4/projects/1085/trigger/pipeline" # ID is from pysshauthz
trigger_pipeline_in_monarch:
stage: integration_test
tags:
- ansible
script:
- echo ${CI_JOB_TOKEN}
- curl --request POST --form token=${CI_JOB_TOKEN} --form "variables[TRIGGER_CI_COMMIT_SHA]=${CI_COMMIT_SHA}" --form ref=cicd https://gitlab.erc.monash.edu.au/api/v4/projects/385/trigger/pipeline # ID is from monarch
# heat_test:
# stage: heat_test
# allow_failure: false
# tags:
# - heat
# before_script:
# - echo "$GC_KEY" > gc_key.pem
# - chmod 400 gc_key.pem
# - echo "$HPCCICD_openrc" > ./$NECTAR_ALLOCATION-openrc.sh
# - source ./$NECTAR_ALLOCATION-openrc.sh
# - export HEAT_TEST_STACKNAME=_TESTING_HEAT
# - bash -x ./CICD/heat/heatcicdwrapper.sh delete_if_exists $HEAT_TEST_STACKNAME
# - sleep 60
# script:
# - echo "heat_test stage"
# - source ./$NECTAR_ALLOCATION-openrc.sh
# - bash -x ./CICD/heat/heatcicdwrapper.sh create $HEAT_TEST_STACKNAME
# - openstack stack list
# - bash -x ./CICD/heat/heatcicdwrapper.sh update $HEAT_TEST_STACKNAME
# - openstack stack list
# - bash -x ./CICD/heat/heatcicdwrapper.sh delete_if_exists $HEAT_TEST_STACKNAME
# - openstack stack list
# after_script:
# - sleep 20 # artifically wait a bit to make sure it is really dead
# when: manual
yamllint:
stage: lint
......@@ -61,7 +43,6 @@ yamllint:
script:
- echo "stage yamllint"
- cd CICD
# - ansible-lint -c .yamllintconf.yaml -x ANSIBLE0002 master_playbook.yml
- yamllint -c ./.yamllintheat.yaml ./heat
# delete_stack_manual:
......@@ -96,6 +77,7 @@ build_cluster_cicd:
tags:
- heat
before_script:
- cd $DEFAULT_PATH
- echo "$GC_KEY" > gc_key.pem
- chmod 400 gc_key.pem
- echo "$HPCCICD_openrc" > ./$NECTAR_ALLOCATION-openrc.sh
......@@ -103,9 +85,10 @@ build_cluster_cicd:
- echo "heat stage"
- source ./$NECTAR_ALLOCATION-openrc.sh
- openstack stack list
- bash -x ./CICD/heat/heatcicdwrapper.sh create_or_update $STACKNAME
- bash -x ./heat/heatcicdwrapper.sh create_or_update $STACKNAME
- bash -x ./heat/server_rebuild.sh all
after_script:
- sleep 20 # artifically wait a bit to give the nodes time to boot
- sleep 30 # artifically wait a bit to give the nodes time to boot
# only:
# changes: #https://docs.gitlab.com/ee/ci/yaml/#onlychangesexceptchanges
# - "heat/*HOT*.yaml"
......@@ -117,20 +100,22 @@ ansible_create_cluster_stage:
tags:
- ansible
before_script:
- cd $DEFAULT_PATH/..
- echo "$GC_KEY" > gc_key.pem
- chmod 400 gc_key.pem
- echo "$HPCCICD_openrc" > ./$NECTAR_ALLOCATION-openrc.sh
script:
- echo "ansible_create_cluster_stage"
- bash -x ./CICD/ansible_create_cluster_script.sh
#after_script:
#- rm ./files/inventory.$STACKNAME
#only:
# changes: #https://docs.gitlab.com/ee/ci/yaml/#onlychangesexceptchanges
# - "master_playbook.yml"
# - "vars/*.{yml,yaml}"
# - schedules
# - CICD/.gitlab-ci.yml
- cd CICD
- ansible-playbook -i files/inventory.$STACKNAME --key-file ../gc_key.pem --skip-tags monitoring master_playbook.yml
- sleep 15
- echo uglyuglyfix
- ansible -i files/inventory.$STACKNAME --key-file ../gc_key.pem -b -a "systemctl restart slurmdbd" ManagementNodes
- sleep 60
- echo do it again
- ansible-playbook -i files/inventory.$STACKNAME --key-file ../gc_key.pem --skip-tags monitoring master_playbook.yml
tests:
stage: tests
......@@ -163,7 +148,22 @@ tests:
- bash -e ./tests/run_tests.sh NFSNodes "files/inventory.$STACKNAME" "../gc_key.pem"
- bash -e ./tests/run_tests.sh SQLNodes "files/inventory.$STACKNAME" "../gc_key.pem"
# licensing https://gitlab.erc.monash.edu.au/hpc-team/license_server/tree/master/roles/avizo_license_monitor
extended:
stage: extended
tags:
- heat
- ansible
before_script:
- echo "cleanup stack"
- sleep 30
- echo "$HPCCICD_openrc" > ./$NECTAR_ALLOCATION-openrc.sh
script:
- source ./$NECTAR_ALLOCATION-openrc.sh
- bash -x ./CICD/heat/heatcicdwrapper.sh delete_if_exists $STACKNAME
only:
variables:
- $EXTENDED != null
manual_cluster_spawn:
stage: push_button_spawn_cluster
......@@ -189,22 +189,6 @@ manual_cluster_spawn:
only:
refs:
- "cicd"
extended:
stage: extended
tags:
- heat
- ansible
before_script:
- echo "cleanup stack"
- sleep 30
- echo "$HPCCICD_openrc" > ./$NECTAR_ALLOCATION-openrc.sh
script:
- source ./$NECTAR_ALLOCATION-openrc.sh
- bash -x ./CICD/heat/heatcicdwrapper.sh delete_if_exists $STACKNAME
only:
variables:
- $EXTENDED != null
clean:
stage: clean
......@@ -217,4 +201,30 @@ clean:
script:
- source ./$NECTAR_ALLOCATION-openrc.sh
- bash -x ./CICD/heat/heatcicdwrapper.sh delete_if_exists $STACKNAME
#when: manual
\ No newline at end of file
# heat_test:
# stage: heat_test
# allow_failure: false
# tags:
# - heat
# before_script:
# - echo "$GC_KEY" > gc_key.pem
# - chmod 400 gc_key.pem
# - echo "$HPCCICD_openrc" > ./$NECTAR_ALLOCATION-openrc.sh
# - source ./$NECTAR_ALLOCATION-openrc.sh
# - export HEAT_TEST_STACKNAME=_TESTING_HEAT
# - bash -x ./CICD/heat/heatcicdwrapper.sh delete_if_exists $HEAT_TEST_STACKNAME
# - sleep 60
# script:
# - echo "heat_test stage"
# - source ./$NECTAR_ALLOCATION-openrc.sh
# - bash -x ./CICD/heat/heatcicdwrapper.sh create $HEAT_TEST_STACKNAME
# - openstack stack list
# - bash -x ./CICD/heat/heatcicdwrapper.sh update $HEAT_TEST_STACKNAME
# - openstack stack list
# - bash -x ./CICD/heat/heatcicdwrapper.sh delete_if_exists $HEAT_TEST_STACKNAME
# - openstack stack list
# after_script:
# - sleep 20 # artifically wait a bit to make sure it is really dead
CICD/ChordDiagramm/Chord_Diagramm - Sheet1.csv 0 → 100644
+ 10
− 0
View file @ 355800ee
0,0,0,1,1,1,1,1,1,0
0,0,0,0,0,0,1,1,0,0
0,0,0,0,1,1,1,1,0,0
1,0,0,0,0,0,0,1,0,0
1,0,1,0,0,0,0,1,0,0
1,0,1,0,0,0,0,1,0,0
1,1,1,0,0,0,0,1,1,0
1,1,1,1,1,1,1,0,0,0
1,0,0,0,0,0,1,0,0,0
0,0,0,0,0,0,0,0,0,0
\ No newline at end of file
CICD/ChordDiagramm/Chord_Diagramm.png 0 → 100644
+ 0
− 0
View file @ 355800ee
CICD/ChordDiagramm/Chord_Diagramm.png

1.87 MiB

CICD/ChordDiagramm/genChordDiagramm.py 0 → 100644
+ 241
− 0
View file @ 355800ee
#!/usr/bin/env python3
# script copied from https://github.com/fengwangPhysics/matplotlib-chord-diagram/blob/master/README.md
# source data manually edited via https://docs.google.com/spreadsheets/d/1JN9S_A5ICPQOvgyVbWJSFJiw-5gO2vF-4AeYuWl-lbs/edit#gid=0
# chord diagram
import matplotlib.pyplot as plt
from matplotlib.path import Path
import matplotlib.patches as patches
import numpy as np
LW = 0.3
def polar2xy(r, theta):
return np.array([r*np.cos(theta), r*np.sin(theta)])
def hex2rgb(c):
return tuple(int(c[i:i+2], 16)/256.0 for i in (1, 3 ,5))
def IdeogramArc(start=0, end=60, radius=1.0, width=0.2, ax=None, color=(1,0,0)):
# start, end should be in [0, 360)
if start > end:
start, end = end, start
start *= np.pi/180.
end *= np.pi/180.
# optimal distance to the control points
# https://stackoverflow.com/questions/1734745/how-to-create-circle-with-b%C3%A9zier-curves
opt = 4./3. * np.tan((end-start)/ 4.) * radius
inner = radius*(1-width)
verts = [
polar2xy(radius, start),
polar2xy(radius, start) + polar2xy(opt, start+0.5*np.pi),
polar2xy(radius, end) + polar2xy(opt, end-0.5*np.pi),
polar2xy(radius, end),
polar2xy(inner, end),
polar2xy(inner, end) + polar2xy(opt*(1-width), end-0.5*np.pi),
polar2xy(inner, start) + polar2xy(opt*(1-width), start+0.5*np.pi),
polar2xy(inner, start),
polar2xy(radius, start),
]
codes = [Path.MOVETO,
Path.CURVE4,
Path.CURVE4,
Path.CURVE4,
Path.LINETO,
Path.CURVE4,
Path.CURVE4,
Path.CURVE4,
Path.CLOSEPOLY,
]
if ax == None:
return verts, codes
else:
path = Path(verts, codes)
patch = patches.PathPatch(path, facecolor=color+(0.5,), edgecolor=color+(0.4,), lw=LW)
ax.add_patch(patch)
def ChordArc(start1=0, end1=60, start2=180, end2=240, radius=1.0, chordwidth=0.7, ax=None, color=(1,0,0)):
# start, end should be in [0, 360)
if start1 > end1:
start1, end1 = end1, start1
if start2 > end2:
start2, end2 = end2, start2
start1 *= np.pi/180.
end1 *= np.pi/180.
start2 *= np.pi/180.
end2 *= np.pi/180.
opt1 = 4./3. * np.tan((end1-start1)/ 4.) * radius
opt2 = 4./3. * np.tan((end2-start2)/ 4.) * radius
rchord = radius * (1-chordwidth)
verts = [
polar2xy(radius, start1),
polar2xy(radius, start1) + polar2xy(opt1, start1+0.5*np.pi),
polar2xy(radius, end1) + polar2xy(opt1, end1-0.5*np.pi),
polar2xy(radius, end1),
polar2xy(rchord, end1),
polar2xy(rchord, start2),
polar2xy(radius, start2),
polar2xy(radius, start2) + polar2xy(opt2, start2+0.5*np.pi),
polar2xy(radius, end2) + polar2xy(opt2, end2-0.5*np.pi),
polar2xy(radius, end2),
polar2xy(rchord, end2),
polar2xy(rchord, start1),
polar2xy(radius, start1),
]
codes = [Path.MOVETO,
Path.CURVE4,
Path.CURVE4,
Path.CURVE4,
Path.CURVE4,
Path.CURVE4,
Path.CURVE4,
Path.CURVE4,
Path.CURVE4,
Path.CURVE4,
Path.CURVE4,
Path.CURVE4,
Path.CURVE4,
]
if ax == None:
return verts, codes
else:
path = Path(verts, codes)
patch = patches.PathPatch(path, facecolor=color+(0.5,), edgecolor=color+(0.4,), lw=LW)
ax.add_patch(patch)
def selfChordArc(start=0, end=60, radius=1.0, chordwidth=0.7, ax=None, color=(1,0,0)):
# start, end should be in [0, 360)
if start > end:
start, end = end, start
start *= np.pi/180.
end *= np.pi/180.
opt = 4./3. * np.tan((end-start)/ 4.) * radius
rchord = radius * (1-chordwidth)
verts = [
polar2xy(radius, start),
polar2xy(radius, start) + polar2xy(opt, start+0.5*np.pi),
polar2xy(radius, end) + polar2xy(opt, end-0.5*np.pi),
polar2xy(radius, end),
polar2xy(rchord, end),
polar2xy(rchord, start),
polar2xy(radius, start),
]
codes = [Path.MOVETO,
Path.CURVE4,
Path.CURVE4,
Path.CURVE4,
Path.CURVE4,
Path.CURVE4,
Path.CURVE4,
]
if ax == None:
return verts, codes
else:
path = Path(verts, codes)
patch = patches.PathPatch(path, facecolor=color+(0.5,), edgecolor=color+(0.4,), lw=LW)
ax.add_patch(patch)
def chordDiagram(X, ax, colors=None, width=0.1, pad=2, chordwidth=0.7):
"""Plot a chord diagram
Parameters
----------
X :
flux data, X[i, j] is the flux from i to j
ax :
matplotlib `axes` to show the plot
colors : optional
user defined colors in rgb format. Use function hex2rgb() to convert hex color to rgb color. Default: d3.js category10
width : optional
width/thickness of the ideogram arc
pad : optional
gap pad between two neighboring ideogram arcs, unit: degree, default: 2 degree
chordwidth : optional
position of the control points for the chords, controlling the shape of the chords
"""
# X[i, j]: i -> j
x = X.sum(axis = 1) # sum over rows
ax.set_xlim(-1.1, 1.1)
ax.set_ylim(-1.1, 1.1)
if colors is None:
# use d3.js category10 https://github.com/d3/d3-3.x-api-reference/blob/master/Ordinal-Scales.md#category10
colors = ['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd',
'#8c564b', '#e377c2', '#7f7f7f', '#bcbd22', '#17becf', '#c49c94']
if len(x) > len(colors):
print('x is too large! Use x smaller than 11')
colors = [hex2rgb(colors[i]) for i in range(len(x))]
# find position for each start and end
y = x/np.sum(x).astype(float) * (360 - pad*len(x))
pos = {}
arc = []
nodePos = []
start = 0
for i in range(len(x)):
end = start + y[i]
arc.append((start, end))
angle = 0.5*(start+end)
#print(start, end, angle)
if -30 <= angle <= 210:
angle -= 90
else:
angle -= 270
nodePos.append(tuple(polar2xy(1.1, 0.5*(start+end)*np.pi/180.)) + (angle,))
z = (X[i, :]/x[i].astype(float)) * (end - start)
ids = np.argsort(z)
z0 = start
for j in ids:
pos[(i, j)] = (z0, z0+z[j])
z0 += z[j]
start = end + pad
for i in range(len(x)):
start, end = arc[i]
IdeogramArc(start=start, end=end, radius=1.0, ax=ax, color=colors[i], width=width)
start, end = pos[(i,i)]
selfChordArc(start, end, radius=1.-width, color=colors[i], chordwidth=chordwidth*0.7, ax=ax)
for j in range(i):
color = colors[i]
if X[i, j] > X[j, i]:
color = colors[j]
start1, end1 = pos[(i,j)]
start2, end2 = pos[(j,i)]
ChordArc(start1, end1, start2, end2,
radius=1.-width, color=colors[i], chordwidth=chordwidth, ax=ax)
#print(nodePos)
return nodePos
##################################
if __name__ == "__main__":
fig = plt.figure(figsize=(6,6))
flux = np.array([
[ 0, 1, 0, 0], #OS Sum:2 ; Centos, Ubuntu
[ 0, 0, 0, 0], #Plays
[ 0, 0, 0, 1], # Cluster: Sum5; Generic, M3, Monarch, SHPC, ACCS
[ 0, 0, 1, 2] #Cloud Sum3: AWS,Nimbus,Nectar
])
from numpy import genfromtxt
flux = genfromtxt('Chord_Diagramm - Sheet1.csv', delimiter=',')
ax = plt.axes([0,0,1,1])
#nodePos = chordDiagram(flux, ax, colors=[hex2rgb(x) for x in ['#666666', '#66ff66', '#ff6666', '#6666ff']])
nodePos = chordDiagram(flux, ax)
ax.axis('off')
prop = dict(fontsize=16*0.8, ha='center', va='center')
nodes = ['OS_Centos76','OS_Centos8','OS_Ubuntu1804','PLY_NFSSQL','PLY_MGMT','PLY_Login','PLY_Compute','C_Generic','C_M3','C_Monarch']
#nodes = ['M3_MONARCH','SHPC','Ubuntu','Centos7','Centos8','Tested','Security','Nectar','?AWS?','DGX@Baremetal','ML@M3','CVL@UWA','CVL_SW','CVL_Desktop','Strudel','/usr/local']
for i in range(len(nodes)):
ax.text(nodePos[i][0], nodePos[i][1], nodes[i], rotation=nodePos[i][2], **prop)
plt.savefig("Chord_Diagramm.png", dpi=600,transparent=False,bbox_inches='tight', pad_inches=0.02)
plt.show(pad_inches=0.02)
CICD/ansible_create_cluster_script.sh
+ 0
− 4
View file @ 355800ee
......@@ -15,7 +15,3 @@ ansible -i files/inventory.$STACKNAME --key-file ../gc_key.pem -a "sudo ls" all
#cd roles
#- "egrep -lRZ 'sudo: true' . | xargs -0 -l sed -i -e 's/sudo: true/become: true/g' "
#cd ..
ansible-playbook -i files/inventory.$STACKNAME --key-file ../gc_key.pem master_playbook.yml
sleep 15
echo uglyuglyfix
ansible -i files/inventory.$STACKNAME --key-file ../gc_key.pem -b -a "systemctl restart slurmdbd" ManagementNodes
\ No newline at end of file
CICD/heat/gc_HOT.yaml
+ 5
− 4
View file @ 355800ee
......@@ -27,7 +27,7 @@ parameters:
default: Classic Provider
Flavour:
type: string
default: t3.xsmall
default: m3.xsmall
resources:
......@@ -67,7 +67,8 @@ resources:
name: "heatmysqlsecgroup"
rules: [ { protocol: tcp,
port_range_min: 3306,
port_range_max: 3306} ]
port_range_max: 3306,
remote_mode: "remote_group_id"} ]
SSHMonashSecGroup:
type: "OS::Neutron::SecurityGroup"
properties:
......@@ -104,7 +105,7 @@ resources:
name:
list_join: [ '-', [ { get_param: "OS::stack_name" }, 'sql0' ]]
availability_zone: { get_param: avz }
flavor: m3.small
flavor: m3.xsmall
image: { get_param: centos_7_image_id }
key_name: { get_param: ssh_key }
security_groups: [ { get_resource: SSHMonashSecGroup }, { get_resource: SlurmSecGroup }, { get_resource: MySQLSecGroup }, { get_resource: NFSSecGroup } ]
......@@ -257,7 +258,7 @@ resources:
# name:
# list_join: [ '-', [ { get_param: "OS::stack_name" }, 'pysshautz' ]]
# availability_zone: { get_param: avz }
# flavor: t3.xsmall
# flavor: m3.xsmall
# image: { get_param: ubuntu_1804_image_id }
# key_name: { get_param: ssh_key }
# security_groups: [ { get_resource: SSHMonashSecGroup }, { get_resource: webaccess } ]
......
CICD/heat/heatcicdwrapper.sh
+ 3
− 3
View file @ 355800ee
......@@ -60,7 +60,7 @@ function create_stack {
echo "I will NOT create existing stack maybe use update"
exit -44
fi
openstack stack create --wait --template ./CICD/heat/gc_HOT.yaml --parameter "project_name=$STACKNAME" -e ./CICD/heat/resource_registry.yaml $STACKNAME
openstack stack create --wait --template ./heat/gc_HOT.yaml --parameter "project_name=$STACKNAME" -e ./heat/resource_registry.yaml $STACKNAME
createreturn=$?
if [ $createreturn -ne "0" ]
then
......@@ -83,14 +83,14 @@ case "$1" in
echo "I cannot update a stack which does not exist"
exit -45
fi
openstack stack update --wait --template ./CICD/heat/gc_HOT.yaml --parameter "project_name=$STACKNAME" -e ./CICD/heat/resource_registry.yaml $STACKNAME
openstack stack update --wait --template ./heat/gc_HOT.yaml --parameter "project_name=$STACKNAME" -e ./heat/resource_registry.yaml $STACKNAME
ret=$?
exit $ret
;;
create_or_update)
if check_stack_exists
then
openstack stack update --wait --template ./CICD/heat/gc_HOT.yaml --parameter "project_name=$STACKNAME" -e ./CICD/heat/resource_registry.yaml $STACKNAME
openstack stack update --wait --template ./heat/gc_HOT.yaml --parameter "project_name=$STACKNAME" -e ./heat/resource_registry.yaml $STACKNAME
ret=$?
exit $ret
fi
......
CICD/heat/server_rebuild.sh 0 → 100755
+ 65
− 0
View file @ 355800ee
#!/bin/bash
REBOOT_TIME=90
NODETYPES=${@}
# make sure you define variable STACKNAME in current environment
if [[ -z $STACKNAME ]]; then
echo "please define STACKNAME variable"
exit 1
fi
# prepend CICD to stack name
if [[ "$STACKNAME" == "CICD"* ]]; then
echo "CICD found in stackname. doing nothing"
else
STACKNAME="CICD"$STACKNAME
fi
function check_stack_exists {
if openstack stack list | grep -w $STACKNAME;
then
echo "stack found";
else
echo "stack not found";
return 1
fi
}
if ! check_stack_exists
then
exit 0
fi
# return and dictionary in json format, which map server name to ansible_host_groups. There will be IndexError if the servers, which names are prefixed by STACKNAME, don't have ansible_host_groups property
host_group_mapping=$(openstack server list --long -f json | python3 -c "import json,sys,re;ivt=json.load(sys.stdin);json.dump({i['Name']: re.search('ansible_host_groups\=\'\[(.+)\]\'', i['Properties']).groups()[0].replace('\"', '').split() for i in ivt if re.match('$STACKNAME',i['Name'])}, fp=sys.stdout)")
# all available ansible_host_groups
available_groups=$(echo $host_group_mapping | python3 -c "import json,sys;mapping=json.load(sys.stdin);output=[];[output.extend(v) for v in mapping.values()];print(output)" | tr -d "[',]")
echo "available groups are $available_groups"
# if the first input parameter is all then rebuild all groups
if [[ "$1" == "all" ]]; then
NODETYPES=$available_groups
fi
echo "going to update group $NODETYPES"
server_list=$(echo $host_group_mapping | python3 -c "import json,sys;mapping=json.load(sys.stdin);node_types='$NODETYPES'.split();output=[k for k in mapping.keys() if len(set(mapping[k]).intersection(set(node_types))) > 0];print(output)" | tr -d "[,]'" | xargs -n1 | sort -u)
echo "server list is $server_list"
rebuild_func(){
echo "rebuilding server $1"
openstack server rebuild --wait $1
}
# for eaech line in data
for server in $server_list
do
rebuild_func $server & # run parallel in background
done
wait # wait for all servers to be rebuild
# add an extra time for reboot
echo "waiting for reboot"
sleep $REBOOT_TIME
echo "All done"
CICD/master_playbook.yml
+ 1
− 0
View file @ 355800ee
......@@ -5,3 +5,4 @@
- import_playbook: plays/nfssqlnodes.yml
- import_playbook: plays/mgmtnodes.yml
- import_playbook: plays/computenodes.yml
- import_playbook: plays/loginnodes.yml
\ No newline at end of file
CICD/plays/allnodes.yml
+ 1
− 0
View file @ 355800ee
......@@ -46,3 +46,4 @@
- { role: calculateKnownHosts, tags: [ calculateKnownHosts ] }
- { role: SSHKnownHosts, tags: [ known_hosts ] }
- { role: jasons_ssh_ca, tags: [ ssh_ca ] }
- { role: set_timezone }
CICD/plays/loginnodes.yml 0 → 120000
+ 1
− 0
View file @ 355800ee
../tests/ManagementNodes/check.yml
\ No newline at end of file
CICD/plays/mgmtnodes.yml
+ 1
− 0
View file @ 355800ee
......@@ -38,6 +38,7 @@
- { role: slurm-common, tags: [ slurm, slurm-common ] }
- { role: slurm_config, tags: [ slurm, slurm-config ] }
- { role: slurm-start, start_slurmdbd: True, start_slurmctld: True, tags: [ slurm-start ] }
- { role: telegraf, tags: [ monitoring ] }
# - { role: provision_slurm, use_active_directory: False, lockpath: "/mnt/home", tags: [ slurm ] }
# - { role: provision_homedir, use_active_directory: False, mntpt: "/mnt/home", tags: [ provisioning ] }
CICD/tests/Readme.md
+ 8
− 5
View file @ 355800ee
this folder should contain tests that will be run automatically by the CICD pipeline
This folder should contain tests that will be run automatically by the CICD pipeline.
all files with fileending .sh will be executed by a shell
all files with fileending yml will be executed by ansible-playbook
./tmp can be used as temporary folder and will be cleaned after execution
The trigger mechanism to execute these tests is `.gitlab-ci.yml` in the toplevel folder of this repository.
because I can I am prefixing tests with 0-9 to give the execution some priority
\ No newline at end of file
- *.sh will be executed by a shell, *.yml will be executed by ansible-playbook
- A test should return 0 as success or otherwise as fail. See 0_Example*
- Tests should aim not to alter the system except /tmp
- Prefixing tests with 0-9 to give the execution some priority
- Tests should not need elevated priviliges. A special user TODO is considered to be present
- TODO a way to deal with Test Payload. This will be determined on test submission
\ No newline at end of file
README.md
+ 4
− 0
View file @ 355800ee
......@@ -8,6 +8,10 @@ as our architecture document.
[![pipeline status](https://gitlab.erc.monash.edu.au/hpc-team/ansible_cluster_in_a_box/badges/cicd/pipeline.svg)](https://gitlab.erc.monash.edu.au/hpc-team/ansible_cluster_in_a_box/commits/cicd)
![Coverage Diagramm](CICD/ChordDiagramm/Chord_Diagramm.png)
A connecting line represents a test coverage of its two endpoints. E.G. Centos8 is only tested on the Cluster_Generic for ComputeNodes
We aim to make these roles as generic as possible. You should be able to start from an inventory file, an ssh key and a git clone of this and end up with a working cluster. In the longer term we might branch to include utilities to make an inventory file using NeCTAR credentials.
If you need a password use get_or_make_password.py (delegated to the passwword server/localhost) to generate a random one that can be shared between nodes
......
plays/allnodes.yml deleted 100644 → 0
+ 0
− 47
View file @ 4d564f5b
- hosts: 'all'
vars_files:
- vars/passwords.yml
- vars/names.yml
- vars/ldapConfig.yml
- vars/filesystems.yml
- vars/slurm.yml
- vars/vars.yml
tasks:
- { name: set use shared state, set_fact: usesharedstatedir=False }
- { name: set hostgroup, set_fact: hostgroup='ComputeNodes' }
tags: [ always ]
- hosts: 'all'
vars_files:
- vars/passwords.yml
- vars/names.yml
- vars/ldapConfig.yml
- vars/filesystems.yml
- vars/slurm.yml
- vars/vars.yml
strategy: free
roles:
# - { role: disable_selinux, tags: [ disableselinux ] }
- { role: upgrade }
- { role: set_password }
- { role: etcHosts, tags: [ networking ] }
# - { role: config_repos, tags: [ repos ] }
- hosts: 'DesktopNodes,ComputeNodes,LoginNodes,ManagementNodes'
vars_files:
- vars/passwords.yml
- vars/names.yml
- vars/ldapConfig.yml
- vars/filesystems.yml
- vars/slurm.yml
- vars/vars.yml
strategy: free
roles:
- { role: disable_selinux, tags: [ disableselinux ] }
- { role: ldapclient, tags: [ authentication ] }
- { role: ssh-password-login, tags: [ authentication ] }
- { role: enable_sudo_group, tags: [ authentication, sudo ] }
- { role: move_homedir }
- { role: calculateKnownHosts, tags: [ calculateKnownHosts ] }
- { role: SSHKnownHosts, tags: [ known_hosts ] }
- { role: jasons_ssh_ca, tags: [ ssh_ca ] }
plays/computenodes.yml deleted 100644 → 0
+ 0
− 64
View file @ 4d564f5b
- hosts: 'DesktopNodes,ComputeNodes,LoginNodes,VisNodes'
vars_files:
- vars/passwords.yml
- vars/names.yml
- vars/ldapConfig.yml
- vars/filesystems.yml
- vars/slurm.yml
- vars/vars.yml
tasks:
- { name: set use shared state, set_fact: usesharedstatedir=False }
tags: [ always ]
- hosts: 'DesktopNodes,ComputeNodes,LoginNodes'
vars_files:
- vars/passwords.yml
- vars/names.yml
- vars/ldapConfig.yml
- vars/filesystems.yml
- vars/slurm.yml
- vars/vars.yml
strategy: free
roles:
- { role: move_homedir, tags: [ authentication, filesystems ] }
- { role: nfs-client, nfsMounts: "{{ computeNfsMounts }}", tags: [ filesystems ] }
- { role: slurm-common, tags: [ slurm, slurm-common ] }
- { role: lmod, tags: [ other ] }
- { role: enable_modules, default_modules: "lmod", tags: [ other ] }
- { role: postfix, tags: [ mail, other ] }
- hosts: 'VisNodes'
vars_files:
- vars/passwords.yml
- vars/names.yml
- vars/ldapConfig.yml
- vars/filesystems.yml
- vars/slurm.yml
- vars/vars.yml
roles:
- { role: gpu, tags: [ gpu ] }
- hosts: 'DesktopNodes,ComputeNodes,LoginNodes'
vars_files:
- vars/passwords.yml
- vars/names.yml
- vars/ldapConfig.yml
- vars/filesystems.yml
- vars/slurm.yml
- vars/vars.yml
roles:
- { role: slurm_config, tags: [slurm, slurm_config] }
- hosts: 'DesktopNodes,ComputeNodes'
vars_files:
- vars/passwords.yml
- vars/names.yml
- vars/ldapConfig.yml
- vars/filesystems.yml
- vars/slurm.yml
- vars/vars.yml
strategy: free
roles:
- { role: slurm-start, start_slurmd: True, tags: [ slurm, slurmstart ] }
- { role: mate-de-install, tags: [ mate-de-install ] } # TODO this crashes for everything except cmca
plays/files deleted 120000 → 0
+ 0
− 1
View file @ 4d564f5b
../files
\ No newline at end of file
plays/init_slurmconf.yml deleted 100644 → 0
+ 0
− 15
View file @ 4d564f5b
---
- hosts: 'all'
tasks:
- include_vars: vars/passwords.yml
- include_vars: vars/names.yml
- include_vars: vars/ldapConfig.yml
- include_vars: vars/filesystems.yml
- include_vars: vars/slurm.yml
- include_vars: vars/vars.yml
- hosts: 'all'
tasks:
- { name: setup, setup: }
- hosts: 'ManagementNodes'
roles:
- { role: calculateSlurmConf }
plays/make_files.yml deleted 100644 → 0
+ 0
− 22
View file @ 4d564f5b
---
# just calculates an etc hosts
- hosts: 'all'
tasks:
- include_vars: vars/passwords.yml
- include_vars: vars/names.yml
- include_vars: vars/ldapConfig.yml
- include_vars: vars/filesystems.yml
- include_vars: vars/slurm.yml
- include_vars: vars/vars.yml
- hosts: 'all'
tasks:
- { name: setup, setup: }
- hosts: 'ManagementNodes'
roles:
- { role: calculateEtcHosts }
#- hosts: 'NFSNodes'
# roles:
# - { role: calculateExports }
plays/mgmtnodes.yml deleted 100644 → 0
+ 0
− 44
View file @ 4d564f5b
# Basic stuff to make the nodes functionl
# i.e. upgrade operating systems, etc
#
- hosts: 'ManagementNodes'
vars_files:
- vars/passwords.yml
- vars/names.yml
- vars/ldapConfig.yml
- vars/filesystems.yml
- vars/slurm.yml
- vars/vars.yml
tasks:
# - { name: set hostgroup, set_fact: hostgroup='ManagementNodes' }
- { name: set use shared state, set_fact: usesharedstatedir=True }
tags: [ always ]
- hosts: 'ManagementNodes'
strategy: free
gather_facts: False
vars_files:
- vars/passwords.yml
- vars/names.yml
- vars/ldapConfig.yml
- vars/filesystems.yml
- vars/slurm.yml
- vars/vars.yml
roles:
# - { role: ldapclient, tags: [ authentication ] }
# - { role: ssh-password-login }
# - { role: enable_sudo_group }
# - { role: make_filesystems, volumes: "{{ glustervolumes }}" }
# - { role: gluster_server, volname: "gv", brickmnt: '/gbrick', gluster_servers: "{{ groups['ManagementNodes'] }}", replicas: 2, tags: [ gluster_server ] }
# - { role: gluster_volcreate, volname: "gv", gluster_servers: "{{ groups['ManagementNodes'] }}", brickmnt: '/gbrick', replicas: 2 }
# - { role: gluster_client, volname: "gv", gluster_servers: ['mgmt0','mgmt1','sql0'], volmnt: '/glusterVolume' }
- { role: nfs-client, nfsMounts: "{{ mgmtNfsMounts }}", tags: [ nfs ] }
- { role: slurmdb-config, tags: [ slurm, slurmdb-config ] }
- { role: slurm-common, tags: [ slurm, slurm-common ] }
- { role: slurm_config, tags: [ slurm, slurm-config ] }
- { role: slurm-start, start_slurmdbd: True, start_slurmctld: True, tags: [ slurm-start ] }
- { role: telegraf, tags: [ monitoring ] }
# - { role: provision_slurm, use_active_directory: False, lockpath: "/mnt/home", tags: [ slurm ] }
# - { role: provision_homedir, use_active_directory: False, mntpt: "/mnt/home", tags: [ provisioning ] }
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment