feat!(deployment): auto create index for deploy (#3667)

index suggestion for deployment SQL now performed in the same code block as index optimizer.

CMakeLists.txt

@@ -86,6 +86,11 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Darwin")
     add_definitions(-Wno-\#pragma-messages)
 endif ()
 add_definitions(-DHAVE_STDBOOL_H=1)
+
+# TODO(#1528): more strict flags
+# Specifically enable checking null pointers for nonnull functions
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wnonnull -Werror=nonnull")
+
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)

cases/query/union_query.yml

@@ -1,5 +1,6 @@
 cases:
   - id: 0
+    deployable: true
     inputs:
       - name: t1
         columns: ["id string","c2 int","c4 timestamp"]
@@ -62,6 +63,7 @@ cases:
         cc, cc, 6
         dd, dd, 7
   - id: 1
+    deployable: true
     desc: select project over union
     inputs:
       - name: t1
@@ -176,6 +178,7 @@ cases:
         cc, cc, 6
         dd, dd, 7
   - id: 3
+    deployable: true
     desc: lastjoin(filter<un-optimized>(union)
     inputs:
       - name: t1
@@ -239,6 +242,7 @@ cases:
         cc, cc, 3
         dd, NULL, NULL
   - id: 4
+    deployable: true
     desc: lastjoin(filter<optimized>(union)
     inputs:
       - name: t1
@@ -300,6 +304,7 @@ cases:
         cc, NULL, NULL
         dd, NULL, NULL
   - id: 5
+    deployable: true
     desc: union(filter<un-optimized>(t2), filter<un-optimized>(t3))
     inputs:
       - name: t0
@@ -361,6 +366,7 @@ cases:
         20, NULL, NULL, NULL, NULL
         30, bb, 30, bb, 5
   - id: 6
+    deployable: true
     desc: nested union
     inputs:
       - name: t1

hybridse/include/vm/engine.h

@@ -17,21 +17,14 @@
 #ifndef HYBRIDSE_INCLUDE_VM_ENGINE_H_
 #define HYBRIDSE_INCLUDE_VM_ENGINE_H_
 
-#include <map>
 #include <memory>
-#include <mutex>  //NOLINT
 #include <set>
 #include <string>
 #include <utility>
 #include <vector>
 #include <unordered_map>
-#include "base/raw_buffer.h"
 #include "base/spin_lock.h"
 #include "codec/fe_row_codec.h"
-#include "codec/list_iterator_codec.h"
-#include "gflags/gflags.h"
-#include "llvm-c/Target.h"
-#include "proto/fe_common.pb.h"
 #include "vm/catalog.h"
 #include "vm/engine_context.h"
 #include "vm/router.h"
@@ -183,12 +176,17 @@ class RunSession {
         options_ = options;
     }
 
+    void SetIndexHintsHandler(std::shared_ptr<IndexHintHandler> handler) { index_hints_ = handler; }
+
  protected:
     std::shared_ptr<hybridse::vm::CompileInfo> compile_info_;
     hybridse::vm::EngineMode engine_mode_;
     bool is_debug_;
     std::string sp_name_;
     std::shared_ptr<const std::unordered_map<std::string, std::string>> options_ = nullptr;
+
+    // [ALPHA] output possible diagnostic infos from compiler
+    std::shared_ptr<IndexHintHandler> index_hints_;
     friend Engine;
 };
 

hybridse/include/vm/engine_context.h

@@ -38,6 +38,26 @@ struct BatchRequestInfo {
     std::set<size_t> output_common_column_indices;
 };
 
+class IndexHintHandler {
+ public:
+    virtual ~IndexHintHandler() {}
+    // report a index hint.
+    //
+    // a index hint is determined by a few things:
+    // 1. source database & source table: where possible index will create upon
+    // 2. keys and ts: suggested index info
+    // 3. epxr node: referring to the physical node contains possible optimizable expression,
+    //    e.g. a ReqeustJoin node contains the join condition 't1.key = t2.key' that may do optimize
+    //
+    // TODO(ace): multiple index suggestion ? choose one.
+    // Say a join condition: 't1.key1 = t2.key1 and t1.key2 = t2.keys', those indexes all sufficient for t2:
+    // 1. key = key1
+    // 2. key = key2
+    // 3. key = key1 + key2
+    virtual void Report(absl::string_view db, absl::string_view table, absl::Span<std::string const> keys,
+                        absl::string_view ts, const PhysicalOpNode* expr_node) = 0;
+};
+
 enum ComileType {
     kCompileSql,
 };
@@ -56,13 +76,10 @@ class CompileInfo {
     virtual const Schema& GetParameterSchema() const = 0;
     virtual const std::string& GetRequestName() const = 0;
     virtual const std::string& GetRequestDbName() const = 0;
-    virtual const hybridse::vm::BatchRequestInfo& GetBatchRequestInfo()
-        const = 0;
+    virtual const hybridse::vm::BatchRequestInfo& GetBatchRequestInfo() const = 0;
     virtual const hybridse::vm::PhysicalOpNode* GetPhysicalPlan() const = 0;
-    virtual void DumpPhysicalPlan(std::ostream& output,
-                                  const std::string& tab) = 0;
-    virtual void DumpClusterJob(std::ostream& output,
-                                const std::string& tab) = 0;
+    virtual void DumpPhysicalPlan(std::ostream& output, const std::string& tab) = 0;
+    virtual void DumpClusterJob(std::ostream& output, const std::string& tab) = 0;
 };
 
 /// @typedef EngineLRUCache

hybridse/include/vm/physical_op.h

@@ -478,6 +478,22 @@ class PhysicalOpNode : public node::NodeBase<PhysicalOpNode> {
      */
     size_t GetNodeId() const { return node_id(); }
 
+    // Return this node cast as a NodeType.
+    // Use only when this node is known to be that type, otherwise, behavior is undefined.
+    template <typename NodeType>
+    const NodeType *GetAsOrNull() const {
+        static_assert(std::is_base_of<PhysicalOpNode, NodeType>::value,
+                      "NodeType must be a member of the PhysicalOpNode class hierarchy");
+        return dynamic_cast<const NodeType *>(this);
+    }
+
+    template <typename NodeType>
+    NodeType *GetAsOrNull() {
+        static_assert(std::is_base_of<PhysicalOpNode, NodeType>::value,
+                      "NodeType must be a member of the PhysicalOpNode class hierarchy");
+        return dynamic_cast<NodeType *>(this);
+    }
+
  protected:
     const PhysicalOpType type_;
     const bool is_block_;