Update C++ client libraries to use TritonJson (triton-inference-serve…

…r#1592) * Update C++ client libraries to use TritonJson The C++ API is changes to return JSON as a string, not as a rapidjson object model. The examples themselves continue to use radpijson to parse that json when necessary (but any json library could be used). * Fix build to directly use rapidjson only for examples * Fix custom backend build
hvico · Jun 5, 2020 · 17f3fe8 · 17f3fe8
1 parent 2336403
commit 17f3fe8
Show file tree

Hide file tree

Showing 29 changed files with 853 additions and 685 deletions.
diff --git a/Dockerfile.centos_client b/Dockerfile.centos_client
@@ -41,7 +41,8 @@ RUN yum install -y \
         libtool \
         make \
         python3 \
-        python3-pip && \
+        python3-pip \
+        rapidjson-devel && \
     yum clean all && \
     pip3 install --upgrade wheel setuptools && \
     pip3 install --upgrade grpcio-tools

diff --git a/Dockerfile.custombackend b/Dockerfile.custombackend
@@ -42,7 +42,8 @@ RUN apt-get update && \
             libopencv-core-dev \
             libssl-dev \
             libtool \
-            pkg-config
+            pkg-config \
+            rapidjson-dev
 
 # Build the custom backend library and example custom backends
 WORKDIR /workspace

diff --git a/src/clients/c++/examples/CMakeLists.txt b/src/clients/c++/examples/CMakeLists.txt
@@ -37,12 +37,32 @@ else()
     DESTINATION .
   )
 
+  #
+  # json_utils
+  #
+  find_package(RapidJSON CONFIG REQUIRED)
+  add_library(
+      json-utils-library EXCLUDE_FROM_ALL OBJECT
+      json_utils.h json_utils.cc
+  )
+  target_include_directories(
+    json-utils-library
+    PUBLIC ${RapidJSON_INCLUDE_DIRS}
+  )
+
   #
   # image_client
   #
   find_package(OpenCV REQUIRED)
-  add_executable(image_client image_client.cc)
-  target_include_directories(image_client PRIVATE ${OpenCV_INCLUDE_DIRS})
+  add_executable(
+    image_client
+    image_client.cc
+    $<TARGET_OBJECTS:json-utils-library>
+  )
+  target_include_directories(
+    image_client
+    PRIVATE ${OpenCV_INCLUDE_DIRS}
+  )
   target_link_libraries(
     image_client
     PRIVATE TRITON::grpcclient_static
@@ -57,7 +77,11 @@ else()
   #
   # ensemble_image_client
   #
-  add_executable(ensemble_image_client ensemble_image_client.cc)
+  add_executable(
+    ensemble_image_client
+    ensemble_image_client.cc
+    $<TARGET_OBJECTS:json-utils-library>
+  )
   target_link_libraries(
     ensemble_image_client
     PRIVATE TRITON::grpcclient_static
@@ -208,7 +232,11 @@ else()
   #
   # simple_http_health_metadata
   #
-  add_executable(simple_http_health_metadata simple_http_health_metadata.cc)
+  add_executable(
+    simple_http_health_metadata
+    simple_http_health_metadata.cc
+    $<TARGET_OBJECTS:json-utils-library>
+  )
   target_link_libraries(
     simple_http_health_metadata
     PRIVATE TRITON::httpclient_static
@@ -221,7 +249,11 @@ else()
   #
   # simple_http_model_control
   #
-  add_executable(simple_http_model_control simple_http_model_control.cc)
+  add_executable(
+    simple_http_model_control
+    simple_http_model_control.cc
+    $<TARGET_OBJECTS:json-utils-library>
+  )
   target_link_libraries(
     simple_http_model_control
     PRIVATE TRITON::httpclient_static

diff --git a/src/clients/c++/examples/ensemble_image_client.cc b/src/clients/c++/examples/ensemble_image_client.cc
@@ -32,6 +32,7 @@
 #include <iterator>
 #include <sstream>
 #include <string>
+#include "src/clients/c++/examples/json_utils.h"
 #include "src/clients/c++/library/grpc_client.h"
 #include "src/clients/c++/library/http_client.h"
 
@@ -280,14 +281,20 @@ main(int argc, char** argv)
   // of the images to be processed is limited by the maximum batch size
   size_t batch_size = 0;
   if (protocol == "http") {
-    rapidjson::Document model_config;
+    std::string model_config;
     err = triton_client.http_client_->ModelConfig(&model_config, model_name);
     if (!err.IsOk()) {
       std::cerr << "error: failed to get model config: " << err << std::endl;
     }
 
-    const auto bs_itr = model_config.FindMember("max_batch_size");
-    if (bs_itr != model_config.MemberEnd()) {
+    rapidjson::Document model_config_json;
+    err = nic::ParseJson(&model_config_json, model_config);
+    if (!err.IsOk()) {
+      std::cerr << "error: failed to parse model config: " << err << std::endl;
+    }
+
+    const auto bs_itr = model_config_json.FindMember("max_batch_size");
+    if (bs_itr != model_config_json.MemberEnd()) {
       batch_size = bs_itr->value.GetInt();
     }
   } else {

diff --git a/src/clients/c++/examples/image_client.cc b/src/clients/c++/examples/image_client.cc
@@ -37,6 +37,7 @@
 #include <mutex>
 #include <queue>
 #include <string>
+#include "src/clients/c++/examples/json_utils.h"
 #include "src/clients/c++/library/grpc_client.h"
 #include "src/clients/c++/library/http_client.h"
 
@@ -862,19 +863,31 @@ main(int argc, char** argv)
 
   ModelInfo model_info;
   if (protocol == ProtocolType::HTTP) {
-    rapidjson::Document model_metadata;
+    std::string model_metadata;
     err = triton_client.http_client_->ModelMetadata(
         &model_metadata, model_name, model_version, http_headers);
     if (!err.IsOk()) {
       std::cerr << "error: failed to get model metadata: " << err << std::endl;
     }
-    rapidjson::Document model_config;
+    rapidjson::Document model_metadata_json;
+    err = nic::ParseJson(&model_metadata_json, model_metadata);
+    if (!err.IsOk()) {
+      std::cerr << "error: failed to parse model metadata: " << err
+                << std::endl;
+    }
+    std::string model_config;
     err = triton_client.http_client_->ModelConfig(
         &model_config, model_name, model_version, http_headers);
     if (!err.IsOk()) {
       std::cerr << "error: failed to get model config: " << err << std::endl;
     }
-    ParseModelHttp(model_metadata, model_config, batch_size, &model_info);
+    rapidjson::Document model_config_json;
+    err = nic::ParseJson(&model_config_json, model_config);
+    if (!err.IsOk()) {
+      std::cerr << "error: failed to parse model config: " << err << std::endl;
+    }
+    ParseModelHttp(
+        model_metadata_json, model_config_json, batch_size, &model_info);
   } else {
     ni::ModelMetadataResponse model_metadata;
     err = triton_client.grpc_client_->ModelMetadata(

diff --git a/src/clients/c++/examples/json_utils.cc b/src/clients/c++/examples/json_utils.cc
@@ -0,0 +1,46 @@
+// Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+//  * Redistributions of source code must retain the above copyright
+//    notice, this list of conditions and the following disclaimer.
+//  * Redistributions in binary form must reproduce the above copyright
+//    notice, this list of conditions and the following disclaimer in the
+//    documentation and/or other materials provided with the distribution.
+//  * Neither the name of NVIDIA CORPORATION nor the names of its
+//    contributors may be used to endorse or promote products derived
+//    from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "src/clients/c++/examples/json_utils.h"
+
+#include <rapidjson/error/en.h>
+
+namespace nvidia { namespace inferenceserver { namespace client {
+
+Error
+ParseJson(rapidjson::Document* document, const std::string& json_str)
+{
+  document->Parse(json_str.c_str(), json_str.size());
+  if (document->HasParseError()) {
+    return Error(
+        "failed to parse JSON at" + std::to_string(document->GetErrorOffset()) +
+        ": " + std::string(GetParseError_En(document->GetParseError())));
+  }
+
+  return Error::Success;
+}
+
+}}}  // namespace nvidia::inferenceserver::client
diff --git a/src/clients/c++/examples/json_utils.h b/src/clients/c++/examples/json_utils.h
@@ -0,0 +1,37 @@
+// Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+//  * Redistributions of source code must retain the above copyright
+//    notice, this list of conditions and the following disclaimer.
+//  * Redistributions in binary form must reproduce the above copyright
+//    notice, this list of conditions and the following disclaimer in the
+//    documentation and/or other materials provided with the distribution.
+//  * Neither the name of NVIDIA CORPORATION nor the names of its
+//    contributors may be used to endorse or promote products derived
+//    from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#pragma once
+
+#include <rapidjson/document.h>
+#include <rapidjson/rapidjson.h>
+#include <string>
+#include "src/clients/c++/library/common.h"
+
+namespace nvidia { namespace inferenceserver { namespace client {
+
+Error ParseJson(rapidjson::Document* document, const std::string& json_str);
+
+}}}  // namespace nvidia::inferenceserver::client
diff --git a/src/clients/c++/examples/simple_http_cudashm_client.cc b/src/clients/c++/examples/simple_http_cudashm_client.cc
@@ -299,12 +299,12 @@ main(int argc, char** argv)
     }
   }
 
-  // Get shared memory regions  active/registered within triton
-  rapidjson::Document status;
+  // Get shared memory regions active/registered within triton
+  std::string shm_status;
   FAIL_IF_ERR(
-      client->CudaSharedMemoryStatus(&status),
+      client->CudaSharedMemoryStatus(&shm_status),
       "failed to get shared memory status");
-  std::cout << "Shared Memory Status:\n" << nic::GetJsonText(status) << "\n";
+  std::cout << "Shared Memory Status:\n" << shm_status << "\n";
 
   // Unregister shared memory
   FAIL_IF_ERR(

diff --git a/src/clients/c++/examples/simple_http_health_metadata.cc b/src/clients/c++/examples/simple_http_health_metadata.cc
@@ -27,6 +27,7 @@
 #include <unistd.h>
 #include <iostream>
 #include <string>
+#include "src/clients/c++/examples/json_utils.h"
 #include "src/clients/c++/library/http_client.h"
 
 namespace ni = nvidia::inferenceserver;
@@ -130,51 +131,73 @@ main(int argc, char** argv)
     exit(1);
   }
 
-
-  rapidjson::Document server_metadata;
-  FAIL_IF_ERR(
-      client->ServerMetadata(&server_metadata, http_headers),
-      "unable to get server metadata");
-  if ((std::string(server_metadata["name"].GetString())).compare("triton") !=
-      0) {
-    std::cerr << "error: unexpected server metadata: "
-              << nic::GetJsonText(server_metadata) << std::endl;
-    exit(1);
+  {
+    std::string server_metadata;
+    FAIL_IF_ERR(
+        client->ServerMetadata(&server_metadata, http_headers),
+        "unable to get server metadata");
+
+    rapidjson::Document server_metadata_json;
+    FAIL_IF_ERR(
+        nic::ParseJson(&server_metadata_json, server_metadata),
+        "failed to parse server metadata");
+    if ((std::string(server_metadata_json["name"].GetString()))
+            .compare("triton") != 0) {
+      std::cerr << "error: unexpected server metadata: " << server_metadata
+                << std::endl;
+      exit(1);
+    }
   }
 
-
-  rapidjson::Document model_metadata;
-  FAIL_IF_ERR(
-      client->ModelMetadata(
-          &model_metadata, model_name, model_version, http_headers),
-      "unable to get model metadata");
-  if ((std::string(model_metadata["name"].GetString())).compare(model_name) !=
-      0) {
-    std::cerr << "error: unexpected model metadata: "
-              << nic::GetJsonText(model_metadata) << std::endl;
-    exit(1);
+  {
+    std::string model_metadata;
+    FAIL_IF_ERR(
+        client->ModelMetadata(
+            &model_metadata, model_name, model_version, http_headers),
+        "unable to get model metadata");
+
+    rapidjson::Document model_metadata_json;
+    FAIL_IF_ERR(
+        nic::ParseJson(&model_metadata_json, model_metadata),
+        "failed to parse model metadata");
+    if ((std::string(model_metadata_json["name"].GetString()))
+            .compare(model_name) != 0) {
+      std::cerr << "error: unexpected model metadata: " << model_metadata
+                << std::endl;
+      exit(1);
+    }
   }
 
-  rapidjson::Document model_config;
-  FAIL_IF_ERR(
-      client->ModelConfig(
-          &model_config, model_name, model_version, http_headers),
-      "unable to get model config");
-  if ((std::string(model_config["name"].GetString())).compare(model_name) !=
-      0) {
-    std::cerr << "error: unexpected model config: "
-              << nic::GetJsonText(model_config) << std::endl;
-    exit(1);
+  {
+    std::string model_config;
+    FAIL_IF_ERR(
+        client->ModelConfig(
+            &model_config, model_name, model_version, http_headers),
+        "unable to get model config");
+
+    rapidjson::Document model_config_json;
+    FAIL_IF_ERR(
+        nic::ParseJson(&model_config_json, model_config),
+        "failed to parse model config");
+    if ((std::string(model_config_json["name"].GetString()))
+            .compare(model_name) != 0) {
+      std::cerr << "error: unexpected model config: " << model_config
+                << std::endl;
+      exit(1);
+    }
   }
 
-  nic::Error err = client->ModelMetadata(
-      &model_metadata, "wrong_model_name", model_version, http_headers);
-  if (err.IsOk()) {
-    std::cerr << "error: expected an error but got: " << err << std::endl;
-    exit(1);
+  {
+    std::string model_metadata;
+    nic::Error err = client->ModelMetadata(
+        &model_metadata, "wrong_model_name", model_version, http_headers);
+    if (err.IsOk()) {
+      std::cerr << "error: expected an error but got: " << err << std::endl;
+      exit(1);
+    }
   }
 
-  std::cout << err << std::endl;
+  std::cout << "SUCCESS" << std::endl;
 
   return 0;
 }