bugfixes

Former-commit-id: dc7d0662

.gitlab-ci.yml

@@ -13,17 +13,9 @@ stages:
   - ansible_create_cluster_stage
   - push_button_spawn_cluster
   - tests
-  - integration_test #https://docs.gitlab.com/ee/ci/triggers/
-#  - clean
+
   
  
-trigger_pipeline_in_Clusterbuild:
-  stage: integration_test
-  tags: 
-  - ansible
-  script:
-  - echo ${CI_JOB_TOKEN}
-  - curl --request POST --form token=${CI_JOB_TOKEN} --form "variables[TRIGGER_CI_COMMIT_SHA]=${CI_COMMIT_SHA}" --form ref=master https://gitlab.erc.monash.edu.au/api/v4/projects/193/trigger/pipeline  # ID is from clusterbuild
 
 yamllint:
   stage: lint
@@ -44,3 +36,98 @@ ansiblelint:
     - echo "stage ansiblelint"
     - cd CICD
     - python3 ansiblelint/run_lint.py --targets master_playbook.yml
+    
+
+build_cluster_cicd:
+  stage: heat
+  allow_failure: false
+  tags:
+  - heat
+  before_script:
+    - cd $DEFAULT_PATH
+    - echo "$GC_KEY" > gc_key.pem
+    - chmod 400 gc_key.pem
+    - echo "$HPCCICD_openrc" > ./$NECTAR_ALLOCATION-openrc.sh
+  script:
+    - echo "heat stage"
+    - source ./$NECTAR_ALLOCATION-openrc.sh
+    - openstack stack list
+    - bash -x ./heat/heatcicdwrapper.sh create_or_update $STACKNAME
+    - bash -x ./heat/server_rebuild.sh all
+  after_script:
+    - sleep 30 # artifically wait a bit to give the nodes time to boot
+#  only:
+#    changes: #https://docs.gitlab.com/ee/ci/yaml/#onlychangesexceptchanges
+#    - "heat/*HOT*.yaml"
+#    - schedules
+#    - ./.gitlab-ci.yml
+
+ansible_create_cluster_stage:
+  stage: ansible_create_cluster_stage
+  tags: 
+  - ansible
+  before_script:
+    - cd $DEFAULT_PATH/..
+    - echo "$GC_KEY" > gc_key.pem
+    - chmod 400 gc_key.pem
+    - echo "$HPCCICD_openrc" > ./$NECTAR_ALLOCATION-openrc.sh
+  script:
+    - echo "ansible_create_cluster_stage"
+    - bash -x ./CICD/ansible_create_cluster_script.sh
+    - cd CICD
+    - ansible-playbook -i files/inventory.$STACKNAME --key-file ../gc_key.pem --skip-tags monitoring master_playbook.yml
+    - sleep 15
+    - echo uglyuglyfix
+    - ansible -i files/inventory.$STACKNAME --key-file ../gc_key.pem -b -a "systemctl restart slurmdbd" ManagementNodes
+    - sleep 60
+    - echo do it again
+    - ansible-playbook -i files/inventory.$STACKNAME --key-file ../gc_key.pem --skip-tags monitoring master_playbook.yml
+    
+
+tests:
+  stage: tests
+  tags:
+  - ansible
+  before_script:
+    - echo "$GC_KEY" > gc_key.pem
+    - chmod 400 gc_key.pem
+    - echo "$HPCCICD_openrc" > ./$NECTAR_ALLOCATION-openrc.sh
+  script:
+    - echo "tests stage"
+    - source ./$NECTAR_ALLOCATION-openrc.sh
+    - openstack stack list
+    - cd CICD
+    - python3 ../scripts/make_inventory.py static $STACKNAME | tee ./files/inventory.$STACKNAME && chmod 755 ./files/inventory.$STACKNAME
+    - grep -qv "I could not find any resouces tagged with project_name:" ./files/inventory.$STACKNAME   #fail if inventory file is empty
+    - ansible -m ping -i files/inventory.$STACKNAME --key-file ../gc_key.pem all
+    - ansible -i files/inventory.$STACKNAME --key-file ../gc_key.pem -a "sudo ls" all
+    
+    - ansible -B 30 -i files/inventory.$STACKNAME --key-file ../gc_key.pem -a "sinfo" ManagementNodes
+    - ansible -B 30 -i files/inventory.$STACKNAME --key-file ../gc_key.pem -a "squeue" ManagementNodes
+    - ansible -B 30 -i files/inventory.$STACKNAME --key-file ../gc_key.pem -a "systemctl is-active --quiet mariadb" SQLNodes
+    - ansible -B 30 -i files/inventory.$STACKNAME --key-file ../gc_key.pem -a "systemctl is-active --quiet slurmctld" ManagementNodes
+    - ansible -B 30 -i files/inventory.$STACKNAME --key-file ../gc_key.pem -a "systemctl is-active --quiet slurmdbd" ManagementNodes
+    
+    - bash -e ./tests/run_tests.sh all "files/inventory.$STACKNAME" "../gc_key.pem"
+    - bash -e ./tests/run_tests.sh ComputeNodes "files/inventory.$STACKNAME" "../gc_key.pem"
+    - bash -e ./tests/run_tests.sh LoginNodes "files/inventory.$STACKNAME" "../gc_key.pem"
+    - bash -e ./tests/run_tests.sh ManagementNodes "files/inventory.$STACKNAME" "../gc_key.pem"
+    - bash -e ./tests/run_tests.sh NFSNodes "files/inventory.$STACKNAME" "../gc_key.pem"
+    - bash -e ./tests/run_tests.sh SQLNodes "files/inventory.$STACKNAME" "../gc_key.pem"
+    
+extended:
+  stage: extended
+  tags:
+  - heat
+  - ansible
+  before_script:
+    - echo "cleanup stack"
+    - sleep 30
+    - echo "$HPCCICD_openrc" > ./$NECTAR_ALLOCATION-openrc.sh
+  script:
+    - source ./$NECTAR_ALLOCATION-openrc.sh
+    - bash -x ./CICD/heat/heatcicdwrapper.sh delete_if_exists $STACKNAME
+  only:
+    variables:
+      - $EXTENDED != null
+  

CICD/heat/gc_HOT.yaml

@@ -159,7 +159,7 @@ resources:
       key_name: { get_param: ssh_key }
       name:
        list_join: [ '-', [ { get_param: "OS::stack_name" }, 'login%index%' ]]
-      security_groups: [ default, { get_param: PublicSSHSecGroupID }, { get_param: SlurmSecGroupID }, { get_param: NFSSecGroupID }, { get_param: MySQLSecGroupID }  ]
+      security_groups: [ default, { get_param: PublicSSHSecGroupID }, { get_param: SlurmSecGroupID }, { get_param: NFSSecGroupID }, { get_param: MySQLSecGroupID } ]
       metadata:
        ansible_host_groups: [ LoginNodes, ManagementNodes ]
        ansible_ssh_user: ec2-user