Make sure that all unittests are executed (triton-inference-server#1823)

* Verify execution of unittests * Add comments to InferUnit * Improve test.sh to work better in case of failures * improve the error messages * rename the unittests base class and fix minor issues * [WIP] Add counting successful runs to other testcases * Update all "test.sh" files to make it compatible with new test counting * fix a minor bug in test counting * fix whitespace * fix minor bug in check_test_results * fix a minor bug in check_test_results * fix minor bugs in test counting * fix a minor bug in L0_infer * fix minor bug in L0_infer_shm * rename TestResCollector -> TestResultCollector * L0_infer TestResult * Make test counting more robust
hvico · Jul 29, 2020 · 7fb608c · 7fb608c
1 parent b345306
commit 7fb608c
Show file tree

Hide file tree

Showing 66 changed files with 729 additions and 220 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,2 +1,3 @@
 /builddir
 /.vscode
+*.so
diff --git a/qa/L0_batcher/batcher_test.py b/qa/L0_batcher/batcher_test.py
@@ -59,7 +59,7 @@
 _deferred_exceptions = []
 
 
-class BatcherTest(unittest.TestCase):
+class BatcherTest(tu.TestResultCollector):
 
     def setUp(self):
         # The helper client for setup will be GRPC for simplicity.
@@ -77,6 +77,7 @@ def tearDown(self):
             destroy_fun = cudashm.destroy_shared_memory_region
         for precreated_shm_region in self.precreated_shm_regions_:
             destroy_fun(precreated_shm_region)
+        super().tearDown()
 
     # FIXME why only used for outputs
     def create_advance(self, shm_regions=None):

diff --git a/qa/L0_batcher/test.sh b/qa/L0_batcher/test.sh
@@ -138,6 +138,13 @@ for model_type in FIXED VARIABLE; do
         if [ $? -ne 0 ]; then
             echo -e "\n***\n*** Test Failed\n***"
             RET=1
+        else
+            check_test_results $CLIENT_LOG 1
+            if [ $? -ne 0 ]; then
+                cat $CLIENT_LOG
+                echo -e "\n***\n*** Test Result Verification Failed\n***"
+                RET=1
+            fi
         fi
         set -e
 
@@ -170,6 +177,13 @@ for model_type in FIXED VARIABLE; do
         if [ $? -ne 0 ]; then
             echo -e "\n***\n*** Test Failed\n***"
             RET=1
+        else
+            check_test_results $CLIENT_LOG 1
+            if [ $? -ne 0 ]; then
+                cat $CLIENT_LOG
+                echo -e "\n***\n*** Test Result Verification Failed\n***"
+                RET=1
+            fi
         fi
         set -e
 
@@ -201,6 +215,13 @@ for i in \
     if [ $? -ne 0 ]; then
         echo -e "\n***\n*** Test Failed\n***"
         RET=1
+    else
+        check_test_results $CLIENT_LOG 1
+        if [ $? -ne 0 ]; then
+            cat $CLIENT_LOG
+            echo -e "\n***\n*** Test Result Verification Failed\n***"
+            RET=1
+        fi
     fi
     set -e
 
@@ -231,6 +252,13 @@ for i in \
     if [ $? -ne 0 ]; then
         echo -e "\n***\n*** Test Failed\n***"
         RET=1
+    else
+        check_test_results $CLIENT_LOG 1
+        if [ $? -ne 0 ]; then
+            cat $CLIENT_LOG
+            echo -e "\n***\n*** Test Result Verification Failed\n***"
+            RET=1
+        fi
     fi
     set -e
 
@@ -288,6 +316,13 @@ if [[ $BACKENDS == *"custom"* ]]; then
     if [ $? -ne 0 ]; then
         echo -e "\n***\n*** Test Failed\n***"
         RET=1
+    else
+        check_test_results $CLIENT_LOG 1
+        if [ $? -ne 0 ]; then
+            cat $CLIENT_LOG
+            echo -e "\n***\n*** Test Result Verification Failed\n***"
+            RET=1
+        fi
     fi
     set -e
 
@@ -322,6 +357,13 @@ if [[ $BACKENDS == *"custom"* ]]; then
     if [ $? -ne 0 ]; then
         echo -e "\n***\n*** Test Failed\n***"
         RET=1
+    else
+        check_test_results $CLIENT_LOG 1
+        if [ $? -ne 0 ]; then
+            cat $CLIENT_LOG
+            echo -e "\n***\n*** Test Result Verification Failed\n***"
+            RET=1
+        fi
     fi
     set -e
 
@@ -338,15 +380,6 @@ if [[ $BACKENDS == *"custom"* ]]; then
     unset TRITONSERVER_DELAY_SCHEDULER
 fi
 
-# python unittest seems to swallow ImportError and still return 0 exit
-# code. So need to explicitly check CLIENT_LOG to make sure we see
-# some running tests
-grep -c "HTTPSocketPoolResponse status=200" $CLIENT_LOG
-if [ $? -ne 0 ]; then
-    echo -e "\n***\n*** Test Failed To Run\n***"
-    RET=1
-fi
-
 if [ $RET -eq 0 ]; then
     echo -e "\n***\n*** Test Passed\n***"
 else

diff --git a/qa/L0_client_nobatch/client_test.py b/qa/L0_client_nobatch/client_test.py
@@ -37,7 +37,7 @@
 import test_util as tu
 
 
-class ClientNoBatchTest(unittest.TestCase):
+class ClientNoBatchTest(tu.TestResultCollector):
 
     def test_nobatch_request_for_batching_model(self):
         input_size = 16

diff --git a/qa/L0_client_nobatch/test.sh b/qa/L0_client_nobatch/test.sh
@@ -39,6 +39,7 @@ export CUDA_VISIBLE_DEVICES=0
 
 CLIENT_LOG="./client.log"
 CLIENT_TEST=client_test.py
+EXPECTED_NUM_TESTS="4"
 
 DATADIR=/data/inferenceserver/${REPO_VERSION}
 
@@ -67,13 +68,13 @@ if [ $? -ne 0 ]; then
     cat $CLIENT_LOG
     echo -e "\n***\n*** Test Failed\n***"
     RET=1
-fi
-
-grep -c "HTTPSocketPoolResponse status=200" $CLIENT_LOG
-if [ $? -ne 0 ]; then
-    cat $CLIENT_LOG
-    echo -e "\n***\n*** Test Failed To Run\n***"
-    RET=1
+else
+    check_test_results $CLIENT_LOG $EXPECTED_NUM_TESTS
+    if [ $? -ne 0 ]; then
+        cat $CLIENT_LOG
+        echo -e "\n***\n*** Test Result Verification Failed\n***"
+        RET=1
+    fi
 fi
 
 set -e

diff --git a/qa/L0_client_timeout/client_timeout_test.py b/qa/L0_client_timeout/client_timeout_test.py
@@ -35,6 +35,7 @@
 import os
 import time
 import socket
+import test_util as tu
 
 import tritongrpcclient as grpcclient
 import tritonhttpclient as httpclient
@@ -54,7 +55,7 @@ def callback(user_data, result, error):
         user_data._completed_requests.put(result)
 
 
-class ClientTimeoutTest(unittest.TestCase):
+class ClientTimeoutTest(tu.TestResultCollector):
 
     def setUp(self):
         self.model_name_ = "custom_identity_int32"

diff --git a/qa/L0_client_timeout/test.sh b/qa/L0_client_timeout/test.sh
@@ -165,6 +165,13 @@ for i in test_grpc_infer \
         echo -e "\n***\n*** Test $i Failed\n***" >>$CLIENT_LOG
             echo -e "\n***\n*** Test $i Failed\n***"
             RET=1
+    else
+        check_test_results $CLIENT_LOG 1
+        if [ $? -ne 0 ]; then
+            cat $CLIENT_LOG
+            echo -e "\n***\n*** Test Result Verification Failed\n***"
+            RET=1
+        fi
     fi
 done
 

diff --git a/qa/L0_cuda_graph/test.sh b/qa/L0_cuda_graph/test.sh
@@ -77,6 +77,13 @@ if [ $? -ne 0 ]; then
     echo -e "\n***\n*** Test Failed\n***"
     cat $CLIENT_LOG
     RET=1
+else
+    check_test_results $CLIENT_LOG 1
+    if [ $? -ne 0 ]; then
+        cat $CLIENT_LOG
+        echo -e "\n***\n*** Test Result Verification Failed\n***"
+        RET=1
+    fi
 fi
 set -e
 
@@ -126,6 +133,13 @@ if [ $? -ne 0 ]; then
     echo -e "\n***\n*** Test Failed\n***"
     cat $CLIENT_LOG
     RET=1
+else
+    check_test_results $CLIENT_LOG 1
+    if [ $? -ne 0 ]; then
+        cat $CLIENT_LOG
+        echo -e "\n***\n*** Test Result Verification Failed\n***"
+        RET=1
+    fi
 fi
 set -e
 

diff --git a/qa/L0_cuda_graph/trt_cuda_graph_test.py b/qa/L0_cuda_graph/trt_cuda_graph_test.py
@@ -30,10 +30,11 @@
 import unittest
 import numpy as np
 import infer_util as iu
+import test_util as tu
 from tritonclientutils import *
 
 
-class TrtCudaGraphTest(unittest.TestCase):
+class TrtCudaGraphTest(tu.TestResultCollector):
 
     def setUp(self):
         self.dtype_ = np.float32

diff --git a/qa/L0_cuda_shared_memory/cuda_shared_memory_test.py b/qa/L0_cuda_shared_memory/cuda_shared_memory_test.py
@@ -24,17 +24,21 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+import sys
+sys.path.append("../common")
+
 import numpy as np
 import unittest
 import os
+import test_util as tu
 
 import tritongrpcclient as grpcclient
 import tritonhttpclient as httpclient
 import tritonshmutils.cuda_shared_memory as cshm
 from tritonclientutils import *
 
 
-class CudaSharedMemoryTest(unittest.TestCase):
+class CudaSharedMemoryTest(tu.TestResultCollector):
 
     def test_invalid_create_shm(self):
         # Raises error since tried to create invalid cuda shared memory region

diff --git a/qa/L0_cuda_shared_memory/test.sh b/qa/L0_cuda_shared_memory/test.sh
@@ -65,6 +65,13 @@ for i in \
         if [ $? -ne 0 ]; then
             echo -e "\n***\n*** Test Failed\n***"
             RET=1
+        else
+            check_test_results $CLIENT_LOG 1
+            if [ $? -ne 0 ]; then
+                cat $CLIENT_LOG
+                echo -e "\n***\n*** Test Result Verification Failed\n***"
+                RET=1
+            fi
         fi
         set -e
 
@@ -73,11 +80,6 @@ for i in \
     done
 done
 
-if [ `grep -c "HTTPSocketPoolResponse status=200" $CLIENT_LOG` != "40" ]; then
-    echo -e "\n***\n*** Failed. Expected 40 HTTPSocketPoolResponse status=200 headers for client\n***"
-    RET=1
-fi
-
 if [ $RET -eq 0 ]; then
     echo -e "\n***\n*** Test Passed\n***"
 else

diff --git a/qa/L0_custom_ops/cuda_op_test.py b/qa/L0_custom_ops/cuda_op_test.py
@@ -97,9 +97,8 @@
         sys.exit(1)
 
     for i in range(elements):
-        print(
-            str(i) + ": input " + str(input_data[i]) + ", output " +
-            str(output_data[i]))
+        print(str(i) + ": input " + str(input_data[i]) + ", output " +
+              str(output_data[i]))
         if output_data[i] != (input_data[i] + 1):
             print("error: incorrect value")
             sys.exit(1)
diff --git a/qa/L0_custom_ops/mod_op_test.py b/qa/L0_custom_ops/mod_op_test.py
@@ -101,9 +101,8 @@
         sys.exit(1)
 
     for i in range(elements):
-        print(
-            str(i) + ": " + str(input_data[0][i]) + " % " +
-            str(input_data[1][i]) + " = " + str(output_data[i]))
+        print(str(i) + ": " + str(input_data[0][i]) + " % " +
+              str(input_data[1][i]) + " = " + str(output_data[i]))
         if ((input_data[0][i] % input_data[1][i]) != output_data[i]):
             print("error: incorrect value")
             sys.exit(1)
diff --git a/qa/L0_custom_ops/onnx_op_test.py b/qa/L0_custom_ops/onnx_op_test.py
@@ -103,9 +103,8 @@
 
     for i in range(3):
         for j in range(5):
-            print(
-                str(input_data[0][i][j]) + " + " + str(input_data[1][i][j]) +
-                " = " + str(output_data[i][j]))
+            print(str(input_data[0][i][j]) + " + " + str(input_data[1][i][j]) +
+                  " = " + str(output_data[i][j]))
             if ((input_data[0][i][j] + input_data[1][i][j]) !=
                     output_data[i][j]):
                 print("error: incorrect value")

diff --git a/qa/L0_custom_ops/zero_out_test.py b/qa/L0_custom_ops/zero_out_test.py
@@ -97,9 +97,8 @@
         sys.exit(1)
 
     for i in range(elements):
-        print(
-            str(i) + ": input " + str(input_data[i]) + ", output " +
-            str(output_data[i]))
+        print(str(i) + ": input " + str(input_data[i]) + ", output " +
+              str(output_data[i]))
         if (i == 0) and (input_data[i] != output_data[i]):
             print("error: incorrect value")
             sys.exit(1)

diff --git a/qa/L0_decoupled/decoupled_test.py b/qa/L0_decoupled/decoupled_test.py
@@ -34,6 +34,7 @@
 import unittest
 import os
 import time
+import test_util as tu
 
 import tritongrpcclient as grpcclient
 import tritonhttpclient as httpclient
@@ -53,7 +54,7 @@ def callback(user_data, result, error):
         user_data._completed_requests.put(result)
 
 
-class DecoupledTest(unittest.TestCase):
+class DecoupledTest(tu.TestResultCollector):
 
     def setUp(self):
         self.trials_ = [("repeat_int32", None), ("simple_repeat", None),

diff --git a/qa/L0_decoupled/test.sh b/qa/L0_decoupled/test.sh
@@ -72,6 +72,13 @@ for i in \
             echo -e "\n***\n*** Test $i Failed\n***" >>$CLIENT_LOG
             echo -e "\n***\n*** Test $i Failed\n***"
             RET=1
+    else
+        check_test_results $CLIENT_LOG 1
+        if [ $? -ne 0 ]; then
+            cat $CLIENT_LOG
+            echo -e "\n***\n*** Test Result Verification Failed\n***"
+            RET=1
+        fi
     fi
     set -e
 done
@@ -87,7 +94,15 @@ if [ $? -ne 0 ]; then
     echo -e "\n***\n*** Test test_one_to_multi_many Failed\n***" >>$CLIENT_LOG
         echo -e "\n***\n*** Test test_one_to_multi_many Failed\n***"
         RET=1
+else
+    check_test_results $CLIENT_LOG 1
+    if [ $? -ne 0 ]; then
+        cat $CLIENT_LOG
+        echo -e "\n***\n*** Test Result Verification Failed\n***"
+        RET=1
+    fi
 fi
+
 set -e
 
 unset TRITONSERVER_DELAY_GRPC_RESPONSE

diff --git a/qa/L0_http_fuzz/fuzztest.py b/qa/L0_http_fuzz/fuzztest.py
@@ -24,14 +24,18 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+import sys
+sys.path.append("../common")
+
 import unittest
+import test_util as tu
 import sqlite3
 from boofuzz import *
 import glob
 import os
 
 
-class FuzzTest(unittest.TestCase):
+class FuzzTest(tu.TestResultCollector):
 
     def _run_fuzz(self, url, logger):
         session = Session(