diff --git a/smoke_test/smoke_test b/smoke_test/smoke_test
index 0ccc0d01bb00e690e392f828a55d6b164cce77ed..1ab6451ef0d5a4023c00a663447a1fec2f0d2d64 100755
--- a/smoke_test/smoke_test
+++ b/smoke_test/smoke_test
@@ -8,7 +8,9 @@ num_module=${2:-50}
 
 
 testcase_dir='./tests'
-
+declare -i num_success=0
+declare -i num_failed=0
+declare -i num_notest=0
 
 
 mkdir -p $output_dir
@@ -28,17 +30,32 @@ for i in $(head -n $num_module $software_usage); do
     module load $name/$ver
     mkdir -p $output_dir/$name
     echo '' > $output_dir/$name/$ver
-    if [ -d $testcase_dir/$name ]; then
+    if [ ! -d $testcase_dir/$name ]; then
+        echo "No testcase for $m found"
+	num_notest+=1
+    else
         for i in $(ls -X $testcase_dir/$name); do
             t_case=$testcase_dir/$name/$i 
             $t_case >> $output_dir/$name/$ver 2>&1
             exitcode=$?
             if [ $exitcode -ne 0 ]; then
                 echo "$m return non-zero exitcode $exitcode for testcase $t_case"
+		num_failed+=1
+	    else
+		num_success+=1
             fi
         done
     fi
     module purge
 done
 IFS=$IFS_orig
+echo "Summary"
+echo "Failed: $num_failed"
+echo "Success $num_success"
+echo "No tests found: $num_notest"
+if [ "$num_success" -lt "50" ]; then
+    echo "Failing because not enough tests succeeded"
+    exit 2
+fi
 
+[ "$num_failed" -eq "0" ]
diff --git a/smoke_test/tests/java/bashcmd b/smoke_test/tests/java/bashcmd
index 562fef064cf727cdd6d24bb6ab1932c38d8cb75a..7e37013d67edc61125a2dc5b8372735c39a9c781 100755
--- a/smoke_test/tests/java/bashcmd
+++ b/smoke_test/tests/java/bashcmd
@@ -1,2 +1,2 @@
 #!/bin/bash
-java --version
+java -version