diff --git a/src/k2/connector/yb/entities/index.cc b/src/k2/connector/yb/entities/index.cc index 3aedbfb2..3770ff25 100644 --- a/src/k2/connector/yb/entities/index.cc +++ b/src/k2/connector/yb/entities/index.cc @@ -84,15 +84,15 @@ namespace sql { // the dependency list does not need to be cached in a member id list for fast access. bool IndexInfo::CheckColumnDependency(ColumnId column_id) const { for (const IndexColumn &index_col : columns_) { - // The protobuf data contains IDs of all columns that this index is referencing. + // The index data contains IDs of all columns that this index is referencing. // Examples: // 1. Index by column // - INDEX ON tab (a_column) - // - The ID of "a_column" is included in protobuf data. + // - The ID of "a_column" is included in index data. // // 2. Index by expression of column: // - INDEX ON tab (j_column->>'field') - // - The ID of "j_column" is included in protobuf data. + // - The ID of "j_column" is included in index data. if (index_col.indexed_column_id == column_id) { return true; } diff --git a/src/k2/connector/yb/entities/index.h b/src/k2/connector/yb/entities/index.h index 5bb4c67f..1da4ea68 100644 --- a/src/k2/connector/yb/entities/index.h +++ b/src/k2/connector/yb/entities/index.h @@ -81,23 +81,32 @@ namespace k2pg { ColumnId column_id; // Column id in the index table. std::string column_name; // Column name in the index table - colexpr.MangledName(). ColumnId indexed_column_id; // Corresponding column id in indexed table. - PgExpr colexpr; // Index expression. + std::shared_ptr colexpr = nullptr; // Index expression. explicit IndexColumn(ColumnId in_column_id, std::string in_column_name, - ColumnId in_indexed_column_id, PgExpr in_colexpr) + ColumnId in_indexed_column_id, std::shared_ptr in_colexpr) : column_id(in_column_id), column_name(std::move(in_column_name)), - indexed_column_id(in_indexed_column_id), colexpr(std::move(colexpr)) { + indexed_column_id(in_indexed_column_id), colexpr(in_colexpr) { + } + + explicit IndexColumn(ColumnId in_column_id, std::string in_column_name, + ColumnId in_indexed_column_id) + : column_id(in_column_id), column_name(std::move(in_column_name)), + indexed_column_id(in_indexed_column_id) { } }; class IndexInfo { public: - explicit IndexInfo(TableId table_id, TableId indexed_table_id, uint32_t schema_version, + explicit IndexInfo(TableId table_id, std::string table_name, uint32_t pg_oid, + TableId indexed_table_id, uint32_t schema_version, bool is_unique, std::vector columns, size_t hash_column_count, size_t range_column_count, std::vector indexed_hash_column_ids, std::vector indexed_range_column_ids, IndexPermissions index_permissions, bool use_mangled_column_name) : table_id_(table_id), + table_name_(table_name), + pg_oid_(pg_oid), indexed_table_id_(indexed_table_id), schema_version_(schema_version), is_unique_(is_unique), @@ -109,22 +118,52 @@ namespace k2pg { index_permissions_(index_permissions) { } + explicit IndexInfo(TableId table_id, + std::string table_name, + uint32_t pg_oid, + TableId indexed_table_id, + uint32_t schema_version, + bool is_unique, + std::vector columns, + IndexPermissions index_permissions) + : table_id_(std::move(table_id)), + table_name_(std::move(table_name)), + pg_oid_(pg_oid), + indexed_table_id_(indexed_table_id), + schema_version_(schema_version), + is_unique_(is_unique), + columns_(std::move(columns)), + // All the index columns are primary keys and we don't manage range keys in PG gate, as a result, + // we treat all primary keys as the (hash) partition keys. + hash_column_count_(columns_.size()), + range_column_count_(0), + index_permissions_(index_permissions) { + } + const TableId& table_id() const { return table_id_; } - const TableId& indexed_table_id() const { - return indexed_table_id_; + const std::string& table_name() const { + return table_name_; } - uint32_t schema_version() const { - return schema_version_; + const uint32_t pg_oid() const { + return pg_oid_; + } + + const TableId& indexed_table_id() const { + return indexed_table_id_; } bool is_unique() const { return is_unique_; } + const uint32_t version() const { + return schema_version_; + } + const std::vector& columns() const { return columns_; } @@ -193,6 +232,8 @@ namespace k2pg { private: const TableId table_id_; // Index table id. + const std::string table_name_; // Index table name. + const uint32_t pg_oid_; const TableId indexed_table_id_; // Indexed table id. const uint32_t schema_version_ = 0; // Index table's schema version. const bool is_unique_ = false; // Whether this is a unique index. diff --git a/src/k2/connector/yb/entities/schema.cc b/src/k2/connector/yb/entities/schema.cc index 244d1acb..0b3b9867 100644 --- a/src/k2/connector/yb/entities/schema.cc +++ b/src/k2/connector/yb/entities/schema.cc @@ -264,6 +264,14 @@ namespace sql { return ColumnId(column_id(column_index)); } + std::pair Schema::FindColumnIdByName(const std::string& column_name) const { + size_t column_index = find_column(column_name); + if (column_index == Schema::kColumnNotFound) { + return std::make_pair(false, -1); + } + return std::make_pair(true, ColumnId(column_id(column_index))); + } + ColumnId Schema::first_column_id() { return kFirstColumnId; } diff --git a/src/k2/connector/yb/entities/schema.h b/src/k2/connector/yb/entities/schema.h index 032f8c82..43a02caa 100644 --- a/src/k2/connector/yb/entities/schema.h +++ b/src/k2/connector/yb/entities/schema.h @@ -243,8 +243,11 @@ namespace sql { SortingType sorting_type_; }; - class TableProperties{ + class TableProperties { public: + TableProperties() = default; + ~TableProperties() = default; + bool operator==(const TableProperties& other) const { return default_time_to_live_ == other.default_time_to_live_; } @@ -457,6 +460,8 @@ namespace sql { Result ColumnIndexByName(GStringPiece col_name) const; + std::pair FindColumnIdByName(const std::string& col_name) const; + // Returns true if the schema contains nullable columns bool has_nullables() const { return has_nullables_; @@ -549,11 +554,11 @@ namespace sql { static ColumnId first_column_id(); - uint64_t version() const { + uint32_t version() const { return version_; } - void set_version(uint64_t version) { + void set_version(uint32_t version) { version_ = version; } @@ -590,7 +595,7 @@ namespace sql { TableProperties table_properties_; - uint64_t version_; + uint32_t version_; }; // Helper used for schema creation/editing. diff --git a/src/k2/connector/yb/entities/table.cc b/src/k2/connector/yb/entities/table.cc index 0b4267a7..f23dcd4a 100644 --- a/src/k2/connector/yb/entities/table.cc +++ b/src/k2/connector/yb/entities/table.cc @@ -20,7 +20,7 @@ namespace k2pg { namespace sql { - Result TableInfo::FindIndex(const TableId& index_id) const { + Result TableInfo::FindIndex(const std::string& index_id) const { return index_map_.FindIndex(index_id); } diff --git a/src/k2/connector/yb/entities/table.h b/src/k2/connector/yb/entities/table.h index eddb221e..62ab7ed1 100644 --- a/src/k2/connector/yb/entities/table.h +++ b/src/k2/connector/yb/entities/table.h @@ -29,9 +29,12 @@ namespace k2pg { namespace sql { struct TableIdentifier { - NamespaceName namespace_name; // Can be empty, that means the namespace has not been set yet. - TableName table_name; - TableIdentifier(NamespaceName ns, TableName tn) : namespace_name(ns), table_name(tn) { + std::string namespace_id; + std::string namespace_name; // Can be empty, that means the namespace has not been set yet. + std::string table_id; + std::string table_name; + TableIdentifier(std::string ns_id, std::string ns_name, std::string tb_id, std::string tb_name) : + namespace_id(ns_id), namespace_name(ns_name), table_id(tb_id), table_name(tb_name) { } }; @@ -40,15 +43,23 @@ namespace sql { typedef std::shared_ptr SharedPtr; - TableInfo(NamespaceName namespace_name, TableName table_name, Schema schema) : - table_id_(namespace_name, table_name), schema_(std::move(schema)) { + TableInfo(std::string namespace_id, std::string namespace_name, std::string table_id, std::string table_name, Schema schema) : + table_id_(namespace_id, namespace_name, table_id, table_name), schema_(std::move(schema)) { } - const NamespaceName& namespace_name() const { + const std::string& namespace_id() const { + return table_id_.namespace_id; + } + + const std::string& namespace_name() const { return table_id_.namespace_name; } - const TableName& table_name() const { + const std::string& table_id() const { + return table_id_.table_id; + } + + const std::string& table_name() const { return table_id_.table_name; } @@ -56,6 +67,22 @@ namespace sql { return table_id_; } + void set_pg_oid(uint32_t pg_oid) { + pg_oid_ = pg_oid; + } + + uint32_t pg_oid() { + return pg_oid_; + } + + void set_next_column_id(int32_t next_column_id) { + next_column_id_ = next_column_id; + } + + int32_t next_column_id() { + return next_column_id_; + } + const Schema& schema() const { return schema_; } @@ -84,17 +111,36 @@ namespace sql { return schema_.num_range_key_columns(); } - void add_secondary_index(const TableId& index_id, const IndexInfo& index_info) { + void add_secondary_index(const std::string& index_id, const IndexInfo& index_info) { index_map_.emplace(index_id, index_info); } - Result FindIndex(const TableId& index_id) const; + const IndexMap& secondary_indexes() { + return index_map_; + } - private: - + void drop_index(const std::string& index_id) { + index_map_.erase(index_id); + } + + Result FindIndex(const std::string& index_id) const; + + void set_is_sys_table(bool is_sys_table) { + is_sys_table_ = is_sys_table; + } + + bool is_sys_table() { + return is_sys_table_; + } + + private: TableIdentifier table_id_; + // PG internal object id + uint32_t pg_oid_; Schema schema_; IndexMap index_map_; + int32_t next_column_id_ = 0; + bool is_sys_table_ = false; }; } // namespace sql diff --git a/src/k2/connector/yb/pggate/CMakeLists.txt b/src/k2/connector/yb/pggate/CMakeLists.txt index 2adb901f..95b55397 100644 --- a/src/k2/connector/yb/pggate/CMakeLists.txt +++ b/src/k2/connector/yb/pggate/CMakeLists.txt @@ -1,5 +1,5 @@ -file(GLOB HEADERS "*.h") -file(GLOB SOURCES "*.cc") +file(GLOB HEADERS "*.h" "*/*.h") +file(GLOB SOURCES "*.cc" "*/*.cc") add_library(ybpggate STATIC ${HEADERS} ${SOURCES}) diff --git a/src/k2/connector/yb/pggate/catalog/base_handler.cc b/src/k2/connector/yb/pggate/catalog/base_handler.cc new file mode 100644 index 00000000..22a70ab5 --- /dev/null +++ b/src/k2/connector/yb/pggate/catalog/base_handler.cc @@ -0,0 +1,90 @@ +/* +MIT License + +Copyright(c) 2020 Futurewei Cloud + + Permission is hereby granted, + free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : + + The above copyright notice and this permission notice shall be included in all copies + or + substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", + WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER + LIABILITY, + WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ + +#include "yb/pggate/catalog/base_handler.h" + +#include + +namespace k2pg { +namespace sql { +namespace catalog { +BaseHandler::BaseHandler(std::shared_ptr k2_adapter) : k2_adapter_(k2_adapter) { +} + +BaseHandler::~BaseHandler() { +} + +RStatus BaseHandler::CreateSKVSchema(std::string collection_name, std::shared_ptr schema) { + RStatus response; + auto result = k2_adapter_->CreateSchema(collection_name, schema).get(); + if (!result.status.is2xxOK()) { + LOG(FATAL) << "Failed to create SKV schema for " << schema->name << "in" << collection_name + << " due to error code " << result.status.code + << " and message: " << result.status.message; + response.code = StatusCode::INTERNAL_ERROR; + response.errorMessage = std::move(result.status.message); + } else { + response.Succeed(); + } + return response; +} + +RStatus BaseHandler::PersistSKVRecord(std::shared_ptr context, k2::dto::SKVRecord& record) { + return SaveOrUpdateSKVRecord(context, record, false); +} + +RStatus BaseHandler::DeleteSKVRecord(std::shared_ptr context, k2::dto::SKVRecord& record) { + return SaveOrUpdateSKVRecord(context, record, true); +} + +RStatus BaseHandler::BatchDeleteSKVRecords(std::shared_ptr context, std::vector& records) { + RStatus response; + for (auto& record : records) { + RStatus result = DeleteSKVRecord(context, record); + if (!result.IsSucceeded()) { + return result; + } + } + response.Succeed(); + return response; +} + +RStatus BaseHandler::SaveOrUpdateSKVRecord(std::shared_ptr context, k2::dto::SKVRecord& record, bool isDelete) { + RStatus response; + auto result = context->GetTxn()->write(std::move(record), isDelete).get(); + if (!result.status.is2xxOK()) { + LOG(FATAL) << "Failed to " << (isDelete ? "Delete" : "Save") + <<" SKV record " + << " due to error code " << result.status.code + << " and message: " << result.status.message; + response.code = StatusCode::INTERNAL_ERROR; + response.errorMessage = std::move(result.status.message); + } else { + response.Succeed(); + } + return response; +} + +} // namespace sql +} // namespace sql +} // namespace k2pg diff --git a/src/k2/connector/yb/pggate/catalog/base_handler.h b/src/k2/connector/yb/pggate/catalog/base_handler.h new file mode 100644 index 00000000..9152eaf4 --- /dev/null +++ b/src/k2/connector/yb/pggate/catalog/base_handler.h @@ -0,0 +1,63 @@ +/* +MIT License + +Copyright(c) 2020 Futurewei Cloud + + Permission is hereby granted, + free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : + + The above copyright notice and this permission notice shall be included in all copies + or + substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", + WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER + LIABILITY, + WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ + +#ifndef CHOGORI_SQL_BASE_HANDLER_H +#define CHOGORI_SQL_BASE_HANDLER_H + +#include + +#include "yb/pggate/catalog/sql_catalog_defaults.h" +#include "yb/pggate/catalog/sql_catalog_entity.h" +#include "yb/pggate/k2_adapter.h" + +namespace k2pg { +namespace sql { +namespace catalog { + +using k2pg::gate::K2Adapter; +using k2pg::gate::K23SITxn; + +class BaseHandler : public std::enable_shared_from_this { + public: + BaseHandler(std::shared_ptr k2_adapter); + ~BaseHandler(); + + RStatus CreateSKVSchema(std::string collection_name, std::shared_ptr schema); + + RStatus PersistSKVRecord(std::shared_ptr context, k2::dto::SKVRecord& record); + + RStatus DeleteSKVRecord(std::shared_ptr context, k2::dto::SKVRecord& record); + + RStatus BatchDeleteSKVRecords(std::shared_ptr context, std::vector& records); + + RStatus SaveOrUpdateSKVRecord(std::shared_ptr context, k2::dto::SKVRecord& record, bool isDelete); + + protected: + std::shared_ptr k2_adapter_; +}; + +} // namespace sql +} // namespace sql +} // namespace k2pg + +#endif //CHOGORI_SQL_BASE_HANDLER_H \ No newline at end of file diff --git a/src/k2/connector/yb/pggate/catalog/cluster_info_handler.cc b/src/k2/connector/yb/pggate/catalog/cluster_info_handler.cc new file mode 100644 index 00000000..1a015334 --- /dev/null +++ b/src/k2/connector/yb/pggate/catalog/cluster_info_handler.cc @@ -0,0 +1,97 @@ +/* +MIT License + +Copyright(c) 2020 Futurewei Cloud + + Permission is hereby granted, + free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : + + The above copyright notice and this permission notice shall be included in all copies + or + substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", + WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER + LIABILITY, + WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ + +#include "yb/pggate/catalog/cluster_info_handler.h" + +#include + +namespace k2pg { +namespace sql { +namespace catalog { + +ClusterInfoHandler::ClusterInfoHandler(std::shared_ptr k2_adapter) + : BaseHandler(k2_adapter), + collection_name_(CatalogConsts::skv_collection_name_sql_primary), + schema_name_(CatalogConsts::skv_schema_name_cluster_info) { + schema_ptr = std::make_shared(schema); +} + +ClusterInfoHandler::~ClusterInfoHandler() { +} + +CreateClusterInfoResult ClusterInfoHandler::CreateClusterInfo(std::shared_ptr context, ClusterInfo& cluster_info) { + CreateClusterInfoResult response; + RStatus schema_result = CreateSKVSchema(collection_name_, schema_ptr); + if (!schema_result.IsSucceeded()) { + response.status = std::move(schema_result); + return response; + } + UpdateClusterInfoResult result = UpdateClusterInfo(context, cluster_info); + response.status = std::move(result.status); + return response; +} + +UpdateClusterInfoResult ClusterInfoHandler::UpdateClusterInfo(std::shared_ptr context, ClusterInfo& cluster_info) { + UpdateClusterInfoResult response; + k2::dto::SKVRecord record(collection_name_, schema_ptr); + record.serializeNext(cluster_info.GetClusterId()); + // use signed integers for unsigned integers since SKV does not support them + record.serializeNext(cluster_info.GetCatalogVersion()); + record.serializeNext(cluster_info.IsInitdbDone()); + response.status = PersistSKVRecord(context, record); + return response; +} + +GetClusterInfoResult ClusterInfoHandler::ReadClusterInfo(std::shared_ptr context, const std::string& cluster_id) { + GetClusterInfoResult response; + k2::dto::SKVRecord record(collection_name_, schema_ptr); + record.serializeNext(cluster_id); + std::future> read_result_future = context->GetTxn()->read(std::move(record)); + k2::ReadResult read_result = read_result_future.get(); + if (read_result.status == k2::dto::K23SIStatus::KeyNotFound) { + LOG(INFO) << "Cluster info record does not exist"; + response.clusterInfo = nullptr; + response.status.Succeed(); + return response; + } + + if (!read_result.status.is2xxOK()) { + LOG(FATAL) << "Failed to read SKV record due to error code " << read_result.status.code + << " and message: " << read_result.status.message; + response.status.code = StatusCode::INTERNAL_ERROR; + response.status.errorMessage = read_result.status.message; + return response; + } + std::shared_ptr cluster_info = std::make_shared(); + cluster_info->SetClusterId(read_result.value.deserializeNext().value()); + // use signed integers for unsigned integers since SKV does not support them + cluster_info->SetCatalogVersion(read_result.value.deserializeNext().value()); + cluster_info->SetInitdbDone(read_result.value.deserializeNext().value()); + response.clusterInfo = cluster_info; + response.status.Succeed(); + return response; +} + +} // namespace sql +} // namespace sql +} // namespace k2pg diff --git a/src/k2/connector/yb/pggate/catalog/cluster_info_handler.h b/src/k2/connector/yb/pggate/catalog/cluster_info_handler.h new file mode 100644 index 00000000..74b33748 --- /dev/null +++ b/src/k2/connector/yb/pggate/catalog/cluster_info_handler.h @@ -0,0 +1,82 @@ +/* +MIT License + +Copyright(c) 2020 Futurewei Cloud + + Permission is hereby granted, + free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : + + The above copyright notice and this permission notice shall be included in all copies + or + substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", + WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER + LIABILITY, + WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ + +#ifndef CHOGORI_SQL_CLUSTER_INFO_HANDLER_H +#define CHOGORI_SQL_CLUSTER_INFO_HANDLER_H + +#include + +#include "yb/pggate/catalog/base_handler.h" + +namespace k2pg { +namespace sql { +namespace catalog { + +struct CreateClusterInfoResult { + RStatus status; +}; + +struct UpdateClusterInfoResult { + RStatus status; +}; + +struct GetClusterInfoResult { + RStatus status; + std::shared_ptr clusterInfo; +}; + +class ClusterInfoHandler : public BaseHandler { + public: + typedef std::shared_ptr SharedPtr; + + static inline k2::dto::Schema schema { + .name = CatalogConsts::skv_schema_name_cluster_info, + .version = 1, + .fields = std::vector { + {k2::dto::FieldType::STRING, "ClusterId", false, false}, + {k2::dto::FieldType::INT64T, "CatalogVersion", false, false}, + {k2::dto::FieldType::BOOL, "InitDbDone", false, false}}, + .partitionKeyFields = std::vector { 0 }, + .rangeKeyFields = std::vector {} + }; + + ClusterInfoHandler(std::shared_ptr k2_adapter); + ~ClusterInfoHandler(); + + CreateClusterInfoResult CreateClusterInfo(std::shared_ptr context, ClusterInfo& cluster_info); + + UpdateClusterInfoResult UpdateClusterInfo(std::shared_ptr context, ClusterInfo& cluster_info); + + GetClusterInfoResult ReadClusterInfo(std::shared_ptr context, const std::string& cluster_id); + + private: + std::string collection_name_; + std::string schema_name_; + std::shared_ptr schema_ptr; +}; + +} // namespace sql +} // namespace sql +} // namespace k2pg + +#endif //CHOGORI_SQL_CLUSTER_INFO_HANDLER_H \ No newline at end of file diff --git a/src/k2/connector/yb/pggate/catalog/namespace_info_handler.cc b/src/k2/connector/yb/pggate/catalog/namespace_info_handler.cc new file mode 100644 index 00000000..250139a3 --- /dev/null +++ b/src/k2/connector/yb/pggate/catalog/namespace_info_handler.cc @@ -0,0 +1,145 @@ +/* +MIT License + +Copyright(c) 2020 Futurewei Cloud + + Permission is hereby granted, + free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : + + The above copyright notice and this permission notice shall be included in all copies + or + substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", + WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER + LIABILITY, + WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ + +#include "yb/pggate/catalog/namespace_info_handler.h" + +#include + +namespace k2pg { +namespace sql { +namespace catalog { + +NamespaceInfoHandler::NamespaceInfoHandler(std::shared_ptr k2_adapter) + : BaseHandler(k2_adapter), + collection_name_(CatalogConsts::skv_collection_name_sql_primary), + schema_name_(CatalogConsts::skv_schema_name_namespace_info) { + schema_ptr = std::make_shared(schema); +} + +NamespaceInfoHandler::~NamespaceInfoHandler() { +} + +CreateNamespaceTableResult NamespaceInfoHandler::CreateNamespaceTableIfNecessary() { + // check if the schema already exists or not, which is an indication of whether if we have created the table or not + std::future schema_result_future = k2_adapter_->GetSchema(collection_name_, schema_name_, 1); + k2::GetSchemaResult schema_result = schema_result_future.get(); + CreateNamespaceTableResult response; + // TODO: double check if this check is valid for schema + if (schema_result.status == k2::dto::K23SIStatus::KeyNotFound) { + LOG(INFO) << "Namespace info table does not exist"; + // create the table schema since it does not exist + RStatus schema_result = CreateSKVSchema(collection_name_, schema_ptr); + response.status = std::move(schema_result); + } else { + response.status.Succeed(); + } + return response; +} + +AddOrUpdateNamespaceResult NamespaceInfoHandler::AddOrUpdateNamespace(std::shared_ptr context, std::shared_ptr namespace_info) { + AddOrUpdateNamespaceResult response; + k2::dto::SKVRecord record(collection_name_, schema_ptr); + record.serializeNext(namespace_info->GetNamespaceId()); + record.serializeNext(namespace_info->GetNamespaceName()); + // use int64_t to represent uint32_t since since SKV does not support them + record.serializeNext(namespace_info->GetNamespaceOid()); + record.serializeNext(namespace_info->GetNextPgOid()); + response.status = PersistSKVRecord(context, record); + return response; +} + +GetNamespaceResult NamespaceInfoHandler::GetNamespace(std::shared_ptr context, const std::string& namespace_id) { + GetNamespaceResult response; + k2::dto::SKVRecord record(collection_name_, schema_ptr); + record.serializeNext(namespace_id); + std::future> read_result_future = context->GetTxn()->read(std::move(record)); + k2::ReadResult read_result = read_result_future.get(); + if (read_result.status == k2::dto::K23SIStatus::KeyNotFound) { + LOG(INFO) << "SKV record does not exist for namespace " << namespace_id; + response.namespaceInfo = nullptr; + response.status.Succeed(); + return response; + } + + if (!read_result.status.is2xxOK()) { + LOG(FATAL) << "Failed to read SKV record due to error code " << read_result.status.code + << " and message: " << read_result.status.message; + response.status.code = StatusCode::INTERNAL_ERROR; + response.status.errorMessage = std::move(read_result.status.message); + return response; + } + std::shared_ptr namespace_ptr = std::make_shared(); + namespace_ptr->SetNamespaceId(read_result.value.deserializeNext().value()); + namespace_ptr->SetNamespaceName(read_result.value.deserializeNext().value()); + // use int64_t to represent uint32_t since since SKV does not support them + namespace_ptr->SetNamespaceOid(read_result.value.deserializeNext().value()); + namespace_ptr->SetNextPgOid(read_result.value.deserializeNext().value()); + response.namespaceInfo = namespace_ptr; + response.status.Succeed(); + return response; +} + +ListNamespacesResult NamespaceInfoHandler::ListNamespaces(std::shared_ptr context) { + ListNamespacesResult response; + std::future create_result_future = k2_adapter_->CreateScanRead(collection_name_, schema_name_); + CreateScanReadResult create_result = create_result_future.get(); + if (!create_result.status.is2xxOK()) { + LOG(FATAL) << "Failed to create scan read due to error code " << create_result.status.code + << " and message: " << create_result.status.message; + response.status.code = StatusCode::INTERNAL_ERROR; + response.status.errorMessage = std::move(create_result.status.message); + return response; + } + + std::shared_ptr query = create_result.query; + do { + std::future query_result_future = context->GetTxn()->scanRead(query); + k2::QueryResult query_result = query_result_future.get(); + if (!query_result.status.is2xxOK()) { + LOG(FATAL) << "Failed to run scan read due to error code " << query_result.status.code + << " and message: " << query_result.status.message; + response.status.code = StatusCode::INTERNAL_ERROR; + response.status.errorMessage = std::move(query_result.status.message); + return response; + } + + if (!query_result.records.empty()) { + for (k2::dto::SKVRecord& record : query_result.records) { + std::shared_ptr namespace_ptr = std::make_shared(); + namespace_ptr->SetNamespaceId(record.deserializeNext().value()); + namespace_ptr->SetNamespaceName(record.deserializeNext().value()); + // use int64_t to represent uint32_t since since SKV does not support them + namespace_ptr->SetNamespaceOid(record.deserializeNext().value()); + namespace_ptr->SetNextPgOid(record.deserializeNext().value()); + response.namespaceInfos.push_back(namespace_ptr); + } + } + // if the query is not done, the query itself is updated with the pagination token for the next call + } while (!query->isDone()); + response.status.Succeed(); + return response; +} + +} // namespace catalog +} // namespace sql +} // namespace k2pg diff --git a/src/k2/connector/yb/pggate/catalog/namespace_info_handler.h b/src/k2/connector/yb/pggate/catalog/namespace_info_handler.h new file mode 100644 index 00000000..72f4a14d --- /dev/null +++ b/src/k2/connector/yb/pggate/catalog/namespace_info_handler.h @@ -0,0 +1,94 @@ +/* +MIT License + +Copyright(c) 2020 Futurewei Cloud + + Permission is hereby granted, + free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : + + The above copyright notice and this permission notice shall be included in all copies + or + substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", + WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER + LIABILITY, + WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ +#ifndef CHOGORI_SQL_NAMESPACE_INFO_HANDLER_H +#define CHOGORI_SQL_NAMESPACE_INFO_HANDLER_H + +#include + +#include "yb/pggate/catalog/base_handler.h" + +namespace k2pg { +namespace sql { +namespace catalog { + +using k2pg::gate::CreateScanReadResult; + +struct CreateNamespaceTableResult { + RStatus status; +}; + +struct AddOrUpdateNamespaceResult { + RStatus status; +}; + +struct GetNamespaceResult { + RStatus status; + std::shared_ptr namespaceInfo; +}; + +struct ListNamespacesResult { + RStatus status; + std::vector> namespaceInfos; +}; + +class NamespaceInfoHandler : public BaseHandler { + public: + typedef std::shared_ptr SharedPtr; + + static inline k2::dto::Schema schema { + .name = CatalogConsts::skv_schema_name_namespace_info, + .version = 1, + .fields = std::vector { + {k2::dto::FieldType::STRING, "NamespaceId", false, false}, + {k2::dto::FieldType::STRING, "NamespaceName", false, false}, + {k2::dto::FieldType::INT64T, "NamespaceOid", false, false}, + {k2::dto::FieldType::INT64T, "NextPgOid", false, false}}, + .partitionKeyFields = std::vector { 0 }, + .rangeKeyFields = std::vector {} + }; + + NamespaceInfoHandler(std::shared_ptr k2_adapter); + + ~NamespaceInfoHandler(); + + CreateNamespaceTableResult CreateNamespaceTableIfNecessary(); + + AddOrUpdateNamespaceResult AddOrUpdateNamespace(std::shared_ptr context, std::shared_ptr namespace_info); + + GetNamespaceResult GetNamespace(std::shared_ptr context, const std::string& namespace_id); + + ListNamespacesResult ListNamespaces(std::shared_ptr context); + + // TODO: add partial update for next_pg_oid once SKV supports partial update + + private: + std::string collection_name_; + std::string schema_name_; + std::shared_ptr schema_ptr; +}; + +} // namespace catalog +} // namespace sql +} // namespace k2pg + +#endif //CHOGORI_SQL_NAMESPACE_INFO_HANDLER_H \ No newline at end of file diff --git a/src/k2/connector/yb/pggate/catalog/sql_catalog_client.cc b/src/k2/connector/yb/pggate/catalog/sql_catalog_client.cc new file mode 100644 index 00000000..acd5fc75 --- /dev/null +++ b/src/k2/connector/yb/pggate/catalog/sql_catalog_client.cc @@ -0,0 +1,210 @@ +/* +MIT License + +Copyright(c) 2020 Futurewei Cloud + + Permission is hereby granted, + free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : + + The above copyright notice and this permission notice shall be included in all copies + or + substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", + WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER + LIABILITY, + WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ + +#include "yb/pggate/catalog/sql_catalog_client.h" + +namespace k2pg { +namespace sql { +namespace catalog { + +Status SqlCatalogClient::IsInitDbDone(bool* isDone) { + GetInitDbRequest request; + GetInitDbResponse response = catalog_manager_->IsInitDbDone(request); + if(!response.status.IsSucceeded()) { + return STATUS_SUBSTITUTE(RuntimeError, + "Failed to check init_db state due to code $0 and message $1", response.status.code, response.status.errorMessage); + } + *isDone = response.isInitDbDone; + return Status::OK(); +} + +Status SqlCatalogClient::CreateNamespace(const std::string& namespace_name, + const std::string& creator_role_name, + const std::string& namespace_id, + const std::string& source_namespace_id, + const std::optional& next_pg_oid) { + CreateNamespaceRequest request { + .namespaceName = namespace_name, + .namespaceId = namespace_id, + .sourceNamespaceId = source_namespace_id, + .creatorRoleName = creator_role_name, + .nextPgOid = next_pg_oid + }; + CreateNamespaceResponse response = catalog_manager_->CreateNamespace(request); + if (!response.status.IsSucceeded()) { + return STATUS_SUBSTITUTE(RuntimeError, + "Failed to create namespace $0 due to code $1 and message $2", namespace_name, response.status.code, response.status.errorMessage); + } + return Status::OK(); +} + +Status SqlCatalogClient::DeleteNamespace(const std::string& namespace_name, + const std::string& namespace_id) { + DeleteNamespaceRequest request {.namespaceName = namespace_name, .namespaceId = namespace_id}; + DeleteNamespaceResponse response = catalog_manager_->DeleteNamespace(request); + if (!response.status.IsSucceeded()) { + return STATUS_SUBSTITUTE(RuntimeError, + "Failed to delete namespace $0 due to code $1 and message $2", namespace_name, response.status.code, response.status.errorMessage); + } + return Status::OK(); +} + +Status SqlCatalogClient::CreateTable( + const std::string& namespace_name, + const std::string& table_name, + const PgObjectId& table_id, + PgSchema& schema, + bool is_pg_catalog_table, + bool is_shared_table, + bool if_not_exist) { + CreateTableRequest request { + .namespaceName = namespace_name, + .namespaceOid = table_id.database_oid, + .tableName = table_name, + .tableOid = table_id.object_oid, + .schema = schema, + .isSysCatalogTable = is_pg_catalog_table, + .isSharedTable = is_shared_table, + .isNotExist = if_not_exist + }; + CreateTableResponse response = catalog_manager_->CreateTable(request); + if (!response.status.IsSucceeded()) { + return STATUS_SUBSTITUTE(RuntimeError, + "Failed to create table $0 in database $1 due to code $2 and message $3", table_name, namespace_name, + response.status.code, response.status.errorMessage); + } + return Status::OK(); +} + +Status SqlCatalogClient::CreateIndexTable( + const std::string& namespace_name, + const std::string& table_name, + const PgObjectId& table_id, + const PgObjectId& base_table_id, + PgSchema& schema, + bool is_unique_index, + bool skip_index_backfill, + bool is_pg_catalog_table, + bool is_shared_table, + bool if_not_exist) { + CreateIndexTableRequest request { + .namespaceName = namespace_name, + .namespaceOid = table_id.database_oid, + .tableName = table_name, + .tableOid = table_id.object_oid, + // index and the base table should be in the same namespace, i.e., database + .baseTableOid = base_table_id.object_oid, + .schema = schema, + .isUnique = is_unique_index, + .skipIndexBackfill = skip_index_backfill, + .isSysCatalogTable = is_pg_catalog_table, + .isSharedTable = is_shared_table, + .isNotExist = if_not_exist + }; + CreateIndexTableResponse response = catalog_manager_->CreateIndexTable(request); + if (!response.status.IsSucceeded()) { + return STATUS_SUBSTITUTE(RuntimeError, + "Failed to create table $0 in database $1 due to code $2 and message $3", table_name, namespace_name, + response.status.code, response.status.errorMessage); + } + return Status::OK(); +} + +Status SqlCatalogClient::DeleteTable(const PgOid database_oid, const PgOid table_oid, bool wait) { + DeleteTableRequest request { + .namespaceOid = database_oid, + .tableOid = table_oid + }; + DeleteTableResponse response = catalog_manager_->DeleteTable(request); + if (!response.status.IsSucceeded()) { + return STATUS_SUBSTITUTE(RuntimeError, + "Failed to delete table $0 due to code $1 and message $2", table_oid, response.status.code, response.status.errorMessage); + } + return Status::OK(); +} + +Status SqlCatalogClient::DeleteIndexTable(const PgOid database_oid, const PgOid table_oid, PgOid *base_table_oid, bool wait) { + DeleteIndexRequest request { + .namespaceOid = database_oid, + .tableOid = table_oid + }; + DeleteIndexResponse response = catalog_manager_->DeleteIndex(request); + if (!response.status.IsSucceeded()) { + return STATUS_SUBSTITUTE(RuntimeError, + "Failed to delete index $0 due to code $1 and message $2", table_oid, response.status.code, response.status.errorMessage); + } + *base_table_oid = response.baseIndexTableOid; + // TODO: add wait logic once we refactor the catalog manager APIs to be asynchronous for state/response + return Status::OK(); +} + +Status SqlCatalogClient::OpenTable(const PgOid database_oid, const PgOid table_oid, std::shared_ptr* table) { + GetTableSchemaRequest request { + .namespaceOid = database_oid, + .tableOid = table_oid + }; + GetTableSchemaResponse response = catalog_manager_->GetTableSchema(request); + if (!response.status.IsSucceeded()) { + return STATUS_SUBSTITUTE(RuntimeError, + "Failed to get schema for table $0 due to code $1 and message $2", table_oid, response.status.code, response.status.errorMessage); + } + + table->swap(response.tableInfo); + return Status::OK(); +} + +Status SqlCatalogClient::ReservePgOids(const PgOid database_oid, + const uint32_t next_oid, + const uint32_t count, + uint32_t* begin_oid, + uint32_t* end_oid) { + ReservePgOidsRequest request { + .namespaceId = GetPgsqlNamespaceId(database_oid), + .nextOid = next_oid, + .count = count + }; + ReservePgOidsResponse response = catalog_manager_->ReservePgOid(request); + if (!response.status.IsSucceeded()) { + return STATUS_SUBSTITUTE(RuntimeError, + "Failed to reserve PG Oids for database $0 due to code $1 and message $2", database_oid, response.status.code, response.status.errorMessage); + } + *begin_oid = response.beginOid; + *end_oid = response.endOid; + return Status::OK(); +} + +Status SqlCatalogClient::GetCatalogVersion(uint64_t *pg_catalog_version) { + GetCatalogVersionRequest request; + GetCatalogVersionResponse response = catalog_manager_->GetCatalogVersion(request); + if(!response.status.IsSucceeded()) { + return STATUS_SUBSTITUTE(RuntimeError, + "Failed to get catalog version due to code $0 and message $1", response.status.code, response.status.errorMessage); + } + *pg_catalog_version = response.catalogVersion; + + return Status::OK(); +} + +} // namespace catalog +} // namespace sql +} // namespace k2pg diff --git a/src/k2/connector/yb/pggate/sql_catalog_client.h b/src/k2/connector/yb/pggate/catalog/sql_catalog_client.h similarity index 93% rename from src/k2/connector/yb/pggate/sql_catalog_client.h rename to src/k2/connector/yb/pggate/catalog/sql_catalog_client.h index 3fa137b9..2aaa8699 100644 --- a/src/k2/connector/yb/pggate/sql_catalog_client.h +++ b/src/k2/connector/yb/pggate/catalog/sql_catalog_client.h @@ -31,10 +31,11 @@ Copyright(c) 2020 Futurewei Cloud #include "yb/entities/schema.h" #include "yb/entities/value.h" #include "yb/pggate/pg_env.h" -#include "yb/pggate/sql_catalog_manager.h" +#include "yb/pggate/catalog/sql_catalog_manager.h" namespace k2pg { namespace sql { +namespace catalog { using yb::Status; using k2pg::gate::PgObjectId; @@ -81,11 +82,11 @@ class SqlCatalogClient { // Delete the specified table. // Set 'wait' to true if the call must wait for the table to be fully deleted before returning. - CHECKED_STATUS DeleteTable(const PgOid database_oid, const PgOid table_id, bool wait = true); + CHECKED_STATUS DeleteTable(const PgOid database_oid, const PgOid table_oid, bool wait = true); - CHECKED_STATUS DeleteIndexTable(const PgOid database_oid, const PgOid table_id, PgOid *base_table_id, bool wait = true); + CHECKED_STATUS DeleteIndexTable(const PgOid database_oid, const PgOid table_oid, PgOid *base_table_oid, bool wait = true); - CHECKED_STATUS OpenTable(const PgOid database_oid, const PgOid table_id, std::shared_ptr* table); + CHECKED_STATUS OpenTable(const PgOid database_oid, const PgOid table_oid, std::shared_ptr* table); Result> OpenTable(const PgOid database_oid, const PgOid table_id) { std::shared_ptr result; @@ -98,12 +99,13 @@ class SqlCatalogClient { uint32_t next_oid, uint32_t count, uint32_t* begin_oid, uint32_t* end_oid); - CHECKED_STATUS GetCatalogVersion(uint64_t *ysql_catalog_version); + CHECKED_STATUS GetCatalogVersion(uint64_t *pg_catalog_version); private: std::shared_ptr catalog_manager_; }; +} // namespace catalog } // namespace sql } // namespace k2pg diff --git a/src/k2/connector/yb/pggate/catalog/sql_catalog_defaults.cc b/src/k2/connector/yb/pggate/catalog/sql_catalog_defaults.cc new file mode 100644 index 00000000..5cd083fb --- /dev/null +++ b/src/k2/connector/yb/pggate/catalog/sql_catalog_defaults.cc @@ -0,0 +1,45 @@ +/* +MIT License + +Copyright(c) 2020 Futurewei Cloud + + Permission is hereby granted, + free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : + + The above copyright notice and this permission notice shall be included in all copies + or + substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", + WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER + LIABILITY, + WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ +#include "yb/pggate/catalog/sql_catalog_defaults.h" + +#include + +namespace k2pg { +namespace sql { +namespace catalog { + +const std::string CatalogConsts::default_cluster_id = "PG_DEFAULT_CLUSTER"; +const std::string CatalogConsts::skv_collection_name_sql_primary = "K2RESVD_COLLECTION_SQL_PRIMARY"; +const std::string CatalogConsts::skv_schema_name_cluster_info = "K2RESVD_SCHEMA_SQL_CLUSTER_INFO"; +const std::string CatalogConsts::skv_schema_name_namespace_info = "K2RESVD_SCHEMA_SQL_NAMESPACE_INFO"; +const std::string CatalogConsts::skv_schema_name_sys_catalog_tablehead = "K2RESVD_SCHEMA_SQL_SYS_CATALOG_TABLEHEAD"; +const std::string CatalogConsts::skv_schema_name_sys_catalog_tablecolumn = "K2RESVD_SCHEMA_SQL_SYS_CATALOG_TABLECOLUMN"; +const std::string CatalogConsts::skv_schema_name_sys_catalog_indexcolumn = "K2RESVD_SCHEMA_SQL_SYS_CATALOG_INDEXCOLUMN"; + +const std::string CatalogConsts::TABLE_ID_COLUMN_NAME = "TableId"; +const std::string CatalogConsts::INDEX_ID_COLUMN_NAME = "IndexId"; +const std::string CatalogConsts::INDEXED_TABLE_ID_COLUMN_NAME = "IndexedTableId"; + +} // namespace catalog +} // namespace sql +} // namespace k2pg diff --git a/src/k2/connector/yb/pggate/catalog/sql_catalog_defaults.h b/src/k2/connector/yb/pggate/catalog/sql_catalog_defaults.h new file mode 100644 index 00000000..043a7079 --- /dev/null +++ b/src/k2/connector/yb/pggate/catalog/sql_catalog_defaults.h @@ -0,0 +1,51 @@ +/* +MIT License + +Copyright(c) 2020 Futurewei Cloud + + Permission is hereby granted, + free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : + + The above copyright notice and this permission notice shall be included in all copies + or + substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", + WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER + LIABILITY, + WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ + +#ifndef CHOGORI_SQL_DEFAULTS_H +#define CHOGORI_SQL_DEFAULTS_H + +#include + +namespace k2pg { +namespace sql { +namespace catalog { + +struct CatalogConsts { + static const std::string default_cluster_id; + static const std::string skv_collection_name_sql_primary; + static const std::string skv_schema_name_cluster_info; + static const std::string skv_schema_name_namespace_info; + static const std::string skv_schema_name_sys_catalog_tablehead; + static const std::string skv_schema_name_sys_catalog_tablecolumn; + static const std::string skv_schema_name_sys_catalog_indexcolumn; + + static const std::string TABLE_ID_COLUMN_NAME; + static const std::string INDEX_ID_COLUMN_NAME; + static const std::string INDEXED_TABLE_ID_COLUMN_NAME; +}; + +} // namespace catalog +} // namespace sql +} // namespace k2pg + +#endif //CHOGORI_SQL_DEFAULTS_H \ No newline at end of file diff --git a/src/k2/connector/yb/pggate/catalog/sql_catalog_entity.cc b/src/k2/connector/yb/pggate/catalog/sql_catalog_entity.cc new file mode 100644 index 00000000..09be2caf --- /dev/null +++ b/src/k2/connector/yb/pggate/catalog/sql_catalog_entity.cc @@ -0,0 +1,67 @@ +/* +MIT License + +Copyright(c) 2020 Futurewei Cloud + + Permission is hereby granted, + free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : + + The above copyright notice and this permission notice shall be included in all copies + or + substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", + WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER + LIABILITY, + WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ + +#include "yb/pggate/catalog/sql_catalog_entity.h" + +#include + +namespace k2pg { +namespace sql { +namespace catalog { + +ClusterInfo::ClusterInfo() { +}; + +ClusterInfo::ClusterInfo(string cluster_id, uint64_t catalog_version, bool initdb_done) : + cluster_id_(cluster_id), catalog_version_(catalog_version), initdb_done_(initdb_done) { +}; + +ClusterInfo::~ClusterInfo() { +}; + +SessionTransactionContext::SessionTransactionContext(std::shared_ptr txn) { + txn_ = txn; + finished_ = false; +} + +SessionTransactionContext::~SessionTransactionContext() { + if (!finished_) { + // abort the transaction if it has been committed or aborted + EndTransaction(false); + finished_ = true; + } +} + +void SessionTransactionContext::EndTransaction(bool should_commit) { + std::future txn_result_future = txn_->endTxn(should_commit); + k2::EndResult txn_result = txn_result_future.get(); + if (!txn_result.status.is2xxOK()) { + LOG(FATAL) << "Failed to commit transaction due to error code " << txn_result.status.code + << " and message: " << txn_result.status.message; + throw std::runtime_error("Failed to end transaction, should_commit: " + should_commit); + } +} + +} // namespace catalog +} // namespace sql +} // namespace k2pg diff --git a/src/k2/connector/yb/pggate/catalog/sql_catalog_entity.h b/src/k2/connector/yb/pggate/catalog/sql_catalog_entity.h new file mode 100644 index 00000000..d0e4316e --- /dev/null +++ b/src/k2/connector/yb/pggate/catalog/sql_catalog_entity.h @@ -0,0 +1,227 @@ +/* +MIT License + +Copyright(c) 2020 Futurewei Cloud + + Permission is hereby granted, + free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : + + The above copyright notice and this permission notice shall be included in all copies + or + substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", + WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER + LIABILITY, + WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ + +#ifndef CHOGORI_SQL_CATALOG_PERSISTENCE_H +#define CHOGORI_SQL_CATALOG_PERSISTENCE_H + +#include +#include + +#include "yb/pggate/k23si_txn.h" + +namespace k2pg { +namespace sql { +namespace catalog { + +using std::string; +using k2pg::gate::K23SITxn; +// use the pair to reference a table +typedef std::pair TableNameKey; + +class ClusterInfo { + public: + ClusterInfo(); + + ClusterInfo(string cluster_id, uint64_t catalog_version, bool initdb_done); + + ~ClusterInfo(); + + void SetClusterId(string cluster_id) { + cluster_id_ = std::move(cluster_id); + } + + const string& GetClusterId() { + return cluster_id_; + } + + void SetCatalogVersion(uint64_t catalog_version) { + catalog_version_ = catalog_version; + } + + uint64_t GetCatalogVersion() { + return catalog_version_; + } + + void SetInitdbDone(bool initdb_done) { + initdb_done_ = initdb_done; + } + + bool IsInitdbDone() { + return initdb_done_; + } + + private: + // cluster id, could be randomly generated or from a configuration parameter + std::string cluster_id_; + + // Right now, the YB logic in PG uses a single global catalog caching version defined in + // src/include/pg_yb_utils.h + // + // extern uint64_t yb_catalog_cache_version; + // + // to check if the catalog needs to be refreshed or not. To not break the above caching + // logic, we need to store the catalog_version as a global variable here. + // + // TODO: update both YB logic in PG, PG gate APIs, and catalog manager to be more fine-grained to + // reduce frequency and/or duration of cache refreshes. One good example is to use a separate + // catalog version for a database, however, we do need to consider the catalog version change + // for shared system tables in PG if we go this path. + // + // Only certain system catalogs (such as pg_database) are shared. + uint64_t catalog_version_; + + // whether initdb, i.e., PG bootstrap procedure to create template DBs, has been done or not + bool initdb_done_ = false; +}; + +class NamespaceInfo { + public: + NamespaceInfo() = default; + ~NamespaceInfo() = default; + + void SetNamespaceId(string id) { + namespace_id_ = std::move(id); + } + + const string& GetNamespaceId() const { + return namespace_id_; + } + + void SetNamespaceName(string name) { + namespace_name_ = std::move(name); + } + + const string& GetNamespaceName() const { + return namespace_name_; + } + + void SetNamespaceOid(uint32_t pg_oid) { + namespace_oid_ = pg_oid; + } + + uint32_t GetNamespaceOid() { + return namespace_oid_; + } + + void SetNextPgOid(uint32_t next_pg_oid) { + assert(next_pg_oid > next_pg_oid_); + next_pg_oid_ = next_pg_oid; + } + + uint32_t GetNextPgOid() { + return next_pg_oid_; + } + + private: + // encoded id, for example, uuid + string namespace_id_; + + // name + string namespace_name_; + + // object id assigned by PG + uint32_t namespace_oid_; + + // next PG Oid that is available for object id assignment for this namespace + uint32_t next_pg_oid_; +}; + +class SessionTransactionContext { + public: + SessionTransactionContext(std::shared_ptr txn); + ~SessionTransactionContext(); + + std::shared_ptr GetTxn() { + return txn_; + } + + void Commit() { + EndTransaction(true); + finished_ = true; + } + + void Abort() { + EndTransaction(false); + finished_ = true; + } + + private: + void EndTransaction(bool should_commit); + + std::shared_ptr txn_; + bool finished_; +}; + +// mapping to the status code defined in yb's status.h (some are not applicable and thus, not included here) +typedef enum RStatusCode { + OK = 0, + NOT_FOUND = 1, + CORRUPTION = 2, + NOT_SUPPORTED = 3, + INVALID_ARGUMENT = 4, + IO_ERROR = 5, + ALREADY_PRESENT = 6, + RUNTIME_ERROR = 7, + NETWORK_ERROR = 8, + ILLEGAL_STATE = 9, + NOT_AUTHORIZED = 10, + ABORTED = 11, + REMOTE_ERROR = 12, + SERVICE_UNAVAILABLE = 13, + TIMED_OUT = 14, + UNINITIALIZED = 15, + CONFIGURATION_ERROR = 16, + INCOMPLETE = 17, + END_OF_FILE = 18, + INVALID_COMMAND = 19, + QUERY_ERROR = 20, + INTERNAL_ERROR = 21, + EXPIRED = 22, +} StatusCode; + +// response status +struct RStatus { + RStatus() = default; + ~RStatus() = default; + + StatusCode code; + std::string errorMessage; + + void Succeed() { + code = StatusCode::OK; + } + + bool IsSucceeded() { + return code == StatusCode::OK; + } +}; + +static const inline RStatus StatusOK{.code = StatusCode::OK, .errorMessage=""}; + +} // namespace catalog +} // namespace sql +} // namespace k2pg + +#endif //CHOGORI_SQL_CATALOG_PERSISTENCE_H + + diff --git a/src/k2/connector/yb/pggate/catalog/sql_catalog_manager.cc b/src/k2/connector/yb/pggate/catalog/sql_catalog_manager.cc new file mode 100644 index 00000000..1d63043e --- /dev/null +++ b/src/k2/connector/yb/pggate/catalog/sql_catalog_manager.cc @@ -0,0 +1,861 @@ +/* +MIT License + +Copyright(c) 2020 Futurewei Cloud + + Permission is hereby granted, + free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : + + The above copyright notice and this permission notice shall be included in all copies + or + substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", + WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER + LIABILITY, + WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ + +#include "yb/pggate/catalog/sql_catalog_manager.h" + +#include +#include +#include +#include + +#include + +namespace k2pg { +namespace sql { +namespace catalog { + + using yb::Status; + using k2pg::gate::K2Adapter; + + SqlCatalogManager::SqlCatalogManager(std::shared_ptr k2_adapter) : + cluster_id_(CatalogConsts::default_cluster_id), k2_adapter_(k2_adapter) { + cluster_info_handler_ = std::make_shared(k2_adapter); + namespace_info_handler_ = std::make_shared(k2_adapter); + table_info_handler_ = std::make_shared(k2_adapter_); + } + + SqlCatalogManager::~SqlCatalogManager() { + } + + Status SqlCatalogManager::Start() { + CHECK(!initted_.load(std::memory_order_acquire)); + + std::shared_ptr ci_context = NewTransactionContext(); + // load cluster info + GetClusterInfoResult ciresp = cluster_info_handler_->ReadClusterInfo(ci_context, cluster_id_); + if (ciresp.status.IsSucceeded()) { + if (ciresp.clusterInfo != nullptr) { + init_db_done_.store(ciresp.clusterInfo->IsInitdbDone(), std::memory_order_relaxed); + catalog_version_.store(ciresp.clusterInfo->GetCatalogVersion(), std::memory_order_relaxed); + LOG(INFO) << "Loaded cluster info record succeeded"; + } else { + ClusterInfo cluster_info(cluster_id_, catalog_version_, init_db_done_); + CreateClusterInfoResult clresp = cluster_info_handler_->CreateClusterInfo(ci_context, cluster_info); + if (clresp.status.IsSucceeded()) { + LOG(INFO) << "Created cluster info record succeeded"; + } else { + ci_context->Abort(); + LOG(FATAL) << "Failed to create cluster info record due to " << clresp.status.errorMessage; + return STATUS_FORMAT(IOError, "Failed to create cluster info record to error code $0 and message $1", + clresp.status.code, clresp.status.errorMessage); + } + } + } else { + ci_context->Abort(); + LOG(FATAL) << "Failed to read cluster info record"; + return STATUS_FORMAT(IOError, "Failed to read cluster info record to error code $0 and message $1", + ciresp.status.code, ciresp.status.errorMessage); + } + // end the current transaction so that we use a different one for later operations + ci_context->Commit(); + + // load namespaces + CreateNamespaceTableResult cnresp = namespace_info_handler_->CreateNamespaceTableIfNecessary(); + if (cnresp.status.IsSucceeded()) { + std::shared_ptr ns_context = NewTransactionContext(); + ListNamespacesResult nsresp = namespace_info_handler_->ListNamespaces(ns_context); + ns_context->Commit(); + + if (nsresp.status.IsSucceeded()) { + if (!nsresp.namespaceInfos.empty()) { + for (auto ns_ptr : nsresp.namespaceInfos) { + // cache namespaces by namespace id and namespace name + namespace_id_map_[ns_ptr->GetNamespaceId()] = ns_ptr; + namespace_name_map_[ns_ptr->GetNamespaceName()] = ns_ptr; + } + } else { + LOG(INFO) << "namespaces are empty"; + } + } else { + LOG(FATAL) << "Failed to load namespaces due to " << nsresp.status.errorMessage; + return STATUS_FORMAT(IOError, "Failed to load namespaces due to error code $0 and message $1", + nsresp.status.code, nsresp.status.errorMessage); + } + } else { + LOG(FATAL) << "Failed to create or check namespace table due to " << cnresp.status.errorMessage; + return STATUS_FORMAT(IOError, "Failed to create or check namespace table due to error code $0 and message $1", + cnresp.status.code, cnresp.status.errorMessage); + } + + initted_.store(true, std::memory_order_release); + return Status::OK(); + } + + void SqlCatalogManager::Shutdown() { + LOG(INFO) << "SQL CatalogManager shutting down..."; + + bool expected = true; + if (initted_.compare_exchange_strong(expected, false, std::memory_order_acq_rel)) { + // TODO: add shut down steps + + } + + LOG(INFO) << "SQL CatalogManager shut down complete. Bye!"; + } + + GetInitDbResponse SqlCatalogManager::IsInitDbDone(const GetInitDbRequest& request) { + GetInitDbResponse response; + if (!init_db_done_) { + std::shared_ptr context = NewTransactionContext(); + GetClusterInfoResult result = cluster_info_handler_->ReadClusterInfo(context, cluster_id_); + context->Commit(); + if (result.status.IsSucceeded() && result.clusterInfo != nullptr) { + if (result.clusterInfo->IsInitdbDone()) { + init_db_done_.store(result.clusterInfo->IsInitdbDone(), std::memory_order_relaxed); + } + if (result.clusterInfo->GetCatalogVersion() > catalog_version_) { + catalog_version_.store(result.clusterInfo->GetCatalogVersion(), std::memory_order_relaxed); + } + } else { + response.status = std::move(result.status); + return response; + } + } + response.isInitDbDone = init_db_done_; + response.status.Succeed(); + return response; + } + + GetCatalogVersionResponse SqlCatalogManager::GetCatalogVersion(const GetCatalogVersionRequest& request) { + GetCatalogVersionResponse response; + std::shared_ptr context = NewTransactionContext(); + // TODO: use a background thread to fetch the ClusterInfo record periodically instead of fetching it for each call + GetClusterInfoResult result = cluster_info_handler_->ReadClusterInfo(context, cluster_id_); + if (result.status.IsSucceeded() && result.clusterInfo != nullptr) { + RStatus status = UpdateCatalogVersion(context, result.clusterInfo->GetCatalogVersion()); + if (!status.IsSucceeded()) { + response.status = std::move(status); + } else { + response.catalogVersion = catalog_version_; + response.status.Succeed(); + } + } else { + response.status = std::move(result.status); + } + context->Commit(); + return response; + } + + CreateNamespaceResponse SqlCatalogManager::CreateNamespace(const CreateNamespaceRequest& request) { + throw std::logic_error("Not implemented yet"); + } + + ListNamespacesResponse SqlCatalogManager::ListNamespaces(const ListNamespacesRequest& request) { + ListNamespacesResponse response; + std::shared_ptr context = NewTransactionContext(); + ListNamespacesResult result = namespace_info_handler_->ListNamespaces(context); + context->Commit(); + + if (result.status.IsSucceeded()) { + response.status.Succeed(); + if (!result.namespaceInfos.empty()) { + UpdateNamespaceCache(result.namespaceInfos); + for (auto ns_ptr : result.namespaceInfos) { + response.namespace_infos.push_back(ns_ptr); + } + } else { + LOG(WARNING) << "No namespaces are found"; + } + } else { + LOG(ERROR) << "Failed to list namespaces due to code " << result.status.code + << " and message " << result.status.errorMessage; + response.status = std::move(result.status); + } + + return response; + } + + GetNamespaceResponse SqlCatalogManager::GetNamespace(const GetNamespaceRequest& request) { + GetNamespaceResponse response; + std::shared_ptr context = NewTransactionContext(); + // TODO: use a background task to refresh the namespace caches to avoid fetching from SKV on each call + GetNamespaceResult result = namespace_info_handler_->GetNamespace(context, request.namespaceId); + context->Commit(); + if (result.status.IsSucceeded()) { + if (result.namespaceInfo != nullptr) { + response.namespace_info = result.namespaceInfo; + + // update namespace caches + namespace_id_map_[response.namespace_info->GetNamespaceId()] = response.namespace_info ; + namespace_name_map_[response.namespace_info->GetNamespaceName()] = response.namespace_info; + response.status.Succeed(); + } else { + LOG(WARNING) << "Cannot find namespace " << request.namespaceId; + response.status.code = StatusCode::NOT_FOUND; + response.status.errorMessage = "Cannot find namespace " + request.namespaceId; + } + } else { + response.status = std::move(result.status); + } + + return response; + } + + DeleteNamespaceResponse SqlCatalogManager::DeleteNamespace(const DeleteNamespaceRequest& request) { + throw std::logic_error("Not implemented yet"); + } + + CreateTableResponse SqlCatalogManager::CreateTable(const CreateTableRequest& request) { + CreateTableResponse response; + std::shared_ptr namespace_info = GetCachedNamespaceByName(request.namespaceName); + if (namespace_info == nullptr) { + // try to refresh namespaces from SKV in case that the requested namespace is created by another catalog manager instance + // this could be avoided by use a single or a quorum of catalog managers + std::shared_ptr ns_context = NewTransactionContext(); + ListNamespacesResult result = namespace_info_handler_->ListNamespaces(ns_context); + ns_context->Commit(); + if (result.status.IsSucceeded() && !result.namespaceInfos.empty()) { + // update namespace caches + UpdateNamespaceCache(result.namespaceInfos); + // recheck namespace + namespace_info = GetCachedNamespaceByName(request.namespaceName); + } + } + if (namespace_info == nullptr) { + LOG(FATAL) << "Cannot find namespace " << request.namespaceName; + response.status.code = StatusCode::NOT_FOUND; + response.status.errorMessage = "Cannot find namespace " + request.namespaceName; + return response; + } + // check if the Table has already existed or not + std::shared_ptr table_info = GetCachedTableInfoByName(namespace_info->GetNamespaceId(), request.tableName); + uint32_t schema_version = 0; + std::string table_id; + if (table_info != nullptr) { + // only create table when it does not exist + if (request.isNotExist) { + response.status.Succeed(); + response.tableInfo = table_info; + // return if the table already exists + return response; + } + // increase the schema version by one + // TODO: If SQL allows changing or rearranging primary key columns, we can't support that on SKV as + // different versions of the same schema. Need to figure out a way to handle this case + schema_version = request.schema.version() + 1; + table_id = table_info->table_id(); + } else { + // new table + schema_version = request.schema.version(); + if (schema_version == 0) { + schema_version++; + } + // generate a string format table id based database object oid and table oid + table_id = GetPgsqlTableId(request.namespaceOid, request.tableOid); + } + Schema table_schema = std::move(request.schema); + table_schema.set_version(schema_version); + std::shared_ptr new_table_info = std::make_shared(namespace_info->GetNamespaceId(), request.namespaceName, + table_info->table_id(), request.tableName, table_schema); + new_table_info->set_pg_oid(request.tableOid); + new_table_info->set_is_sys_table(request.isSysCatalogTable); + new_table_info->set_next_column_id(table_schema.max_col_id() + 1); + + // TODO: add logic for shared table + std::shared_ptr context = NewTransactionContext(); + CreateUpdateTableResult result = table_info_handler_->CreateOrUpdateTable(context, new_table_info->namespace_id(), new_table_info); + if (result.status.IsSucceeded()) { + // commit transaction + context->Commit(); + // update table caches + UpdateTableCache(new_table_info); + // increase catalog version + IncreaseCatalogVersion(); + // return response + response.status.Succeed(); + response.tableInfo = new_table_info; + } else { + // abort the transaction + context->Abort(); + response.status = std::move(result.status); + } + + return response; + } + + CreateIndexTableResponse SqlCatalogManager::CreateIndexTable(const CreateIndexTableRequest& request) { + CreateIndexTableResponse response; + std::shared_ptr namespace_info = GetCachedNamespaceByName(request.namespaceName); + if (namespace_info == nullptr) { + // try to refresh namespaces from SKV in case that the requested namespace is created by another catalog manager instance + // this could be avoided by use a single or a quorum of catalog managers + std::shared_ptr ns_context = NewTransactionContext(); + ListNamespacesResult result = namespace_info_handler_->ListNamespaces(ns_context); + ns_context->Commit(); + if (result.status.IsSucceeded() && !result.namespaceInfos.empty()) { + // update namespace caches + UpdateNamespaceCache(result.namespaceInfos); + // recheck namespace + namespace_info = GetCachedNamespaceByName(request.namespaceName); + } + } + if (namespace_info == nullptr) { + LOG(FATAL) << "Cannot find namespace " << request.namespaceName; + response.status.code = StatusCode::NOT_FOUND; + response.status.errorMessage = "Cannot find namespace " + request.namespaceName; + return response; + } + // generate table id from namespace oid and table oid + std::string base_table_id = GetPgsqlTableId(request.namespaceOid, request.baseTableOid); + std::string index_table_id = GetPgsqlTableId(request.namespaceOid, request.tableOid); + + // check if the base table exists or not + std::shared_ptr base_table_info = GetCachedTableInfoById(base_table_id); + std::shared_ptr context = NewTransactionContext(); + // try to fetch the table from SKV if not found + if (base_table_info == nullptr) { + GetTableResult table_result = table_info_handler_->GetTable(context, namespace_info->GetNamespaceId(), namespace_info->GetNamespaceName(), + base_table_id); + if (table_result.status.IsSucceeded() && table_result.tableInfo != nullptr) { + // update table cache + UpdateTableCache(table_result.tableInfo); + base_table_info = table_result.tableInfo; + } + } + + if (base_table_info == nullptr) { + // cannot find the base table + response.status.code = StatusCode::NOT_FOUND; + response.status.errorMessage = "Cannot find base table " + base_table_id + " for index " + request.tableName; + return response; + } + + bool need_create_index = false; + if (base_table_info->has_secondary_indexes()) { + const IndexMap& index_map = base_table_info->secondary_indexes(); + const auto itr = index_map.find(index_table_id); + // the index has already been defined + if (itr != index_map.end()) { + // return if 'create .. if not exist' clause is specified + if (request.isNotExist) { + const IndexInfo& index_info = itr->second; + response.indexInfo = std::make_shared(index_info); + response.status.Succeed(); + return response; + } else { + // BUGBUG: change to alter index instead of recreating one here + need_create_index = true; + } + } else { + need_create_index = true; + } + } else { + need_create_index = true; + } + + if (need_create_index) { + try { + // use default index permission, could be customized by user/api + IndexInfo new_index_info = BuildIndexInfo(base_table_info, index_table_id, request.tableName, request.tableOid, + request.schema, request.isUnique, IndexPermissions::INDEX_PERM_READ_WRITE_AND_DELETE); + + // persist the index table metadata to the system catalog SKV tables + table_info_handler_->PersistIndexTable(context, namespace_info->GetNamespaceId(), base_table_info, new_index_info); + + // create a SKV schema to insert the actual index data + table_info_handler_->CreateOrUpdateIndexSKVSchema(context, namespace_info->GetNamespaceId(), base_table_info, new_index_info); + + // update the base table with the new index + base_table_info->add_secondary_index(index_table_id, new_index_info); + + // update table cache + UpdateTableCache(base_table_info); + + // update index cache + std::shared_ptr new_index_info_ptr = std::make_shared(new_index_info); + AddIndexCache(new_index_info_ptr); + + // increase catalog version + IncreaseCatalogVersion(); + + if (!request.skipIndexBackfill) { + // TODO: add logic to backfill the index + } + response.indexInfo = new_index_info_ptr; + response.status.Succeed(); + } catch (const std::exception& e) { + response.status.code = StatusCode::RUNTIME_ERROR; + response.status.errorMessage = e.what(); + } + } + + return response; + } + + GetTableSchemaResponse SqlCatalogManager::GetTableSchema(const GetTableSchemaRequest& request) { + GetTableSchemaResponse response; + // generate table id from namespace oid and table oid + std::string table_id = GetPgsqlTableId(request.namespaceOid, request.tableOid); + // check the table schema from cache + std::shared_ptr table_info = GetCachedTableInfoById(table_id); + if (table_info != nullptr) { + response.tableInfo = table_info; + response.status.Succeed(); + } else { + std::string namespace_id = GetPgsqlNamespaceId(request.namespaceOid); + std::shared_ptr namespace_info = GetCachedNamespaceById(namespace_id); + if (namespace_info == nullptr) { + // try to refresh namespaces from SKV in case that the requested namespace is created by another catalog manager instance + // this could be avoided by use a single or a quorum of catalog managers + std::shared_ptr ns_context = NewTransactionContext(); + ListNamespacesResult result = namespace_info_handler_->ListNamespaces(ns_context); + ns_context->Commit(); + if (result.status.IsSucceeded() && !result.namespaceInfos.empty()) { + // update namespace caches + UpdateNamespaceCache(result.namespaceInfos); + // recheck namespace + namespace_info = GetCachedNamespaceById(namespace_id); + } + } + if (namespace_info == nullptr) { + LOG(FATAL) << "Cannot find namespace " << namespace_id; + response.status.code = StatusCode::NOT_FOUND; + response.status.errorMessage = "Cannot find namespace " + namespace_id; + return response; + } + + std::shared_ptr context = NewTransactionContext(); + // fetch the table from SKV + GetTableResult table_result = table_info_handler_->GetTable(context, namespace_info->GetNamespaceId(), namespace_info->GetNamespaceName(), + table_id); + if (table_result.status.IsSucceeded()) { + if (table_result.tableInfo != nullptr) { + response.status.Succeed(); + response.tableInfo = table_result.tableInfo; + // update table cache + UpdateTableCache(table_result.tableInfo); + } else { + response.status.code = StatusCode::NOT_FOUND; + response.status.errorMessage = "Cannot find table " + table_id; + response.tableInfo = nullptr; + } + context->Commit(); + } else { + response.status = std::move(table_result.status); + response.tableInfo = nullptr; + context->Abort(); + } + } + + return response; + } + + DeleteTableResponse SqlCatalogManager::DeleteTable(const DeleteTableRequest& request) { + DeleteTableResponse response; + std::string namespace_id = GetPgsqlNamespaceId(request.namespaceOid); + std::string table_id = GetPgsqlTableId(request.namespaceOid, request.tableOid); + response.namespaceId = namespace_id; + response.tableId = table_id; + + std::shared_ptr table_info = GetCachedTableInfoById(table_id); + if (table_info == nullptr) { + // try to find table from SKV by looking at namespace first + std::shared_ptr namespace_info = GetCachedNamespaceById(namespace_id); + if (namespace_info == nullptr) { + // try to refresh namespaces from SKV in case that the requested namespace is created by another catalog manager instance + // this could be avoided by use a single or a quorum of catalog managers + std::shared_ptr ns_context = NewTransactionContext(); + ListNamespacesResult result = namespace_info_handler_->ListNamespaces(ns_context); + ns_context->Commit(); + if (result.status.IsSucceeded() && !result.namespaceInfos.empty()) { + // update namespace caches + UpdateNamespaceCache(result.namespaceInfos); + // recheck namespace + namespace_info = GetCachedNamespaceById(namespace_id); + } + } + if (namespace_info == nullptr) { + LOG(FATAL) << "Cannot find namespace " << namespace_id; + response.status.code = StatusCode::NOT_FOUND; + response.status.errorMessage = "Cannot find namespace " + namespace_id; + return response; + } + + std::shared_ptr context = NewTransactionContext(); + // fetch the table from SKV + GetTableResult table_result = table_info_handler_->GetTable(context, namespace_info->GetNamespaceId(), namespace_info->GetNamespaceName(), + table_id); + if (table_result.status.IsSucceeded()) { + if (table_result.tableInfo != nullptr) { + // delete indexes and the table itself + // delete table data + DeleteTableResult delete_data_result = table_info_handler_->DeleteTableData(context, namespace_info->GetNamespaceId(), table_result.tableInfo); + if (!delete_data_result.status.IsSucceeded()) { + response.status = std::move(delete_data_result.status); + } else { + // delete table schema metadata + DeleteTableResult delete_metadata_result = table_info_handler_->DeleteTableMetadata(context, namespace_info->GetNamespaceId(), table_result.tableInfo); + if (!delete_metadata_result.status.IsSucceeded()) { + response.status = std::move(delete_metadata_result.status); + } else { + // clear table cache after table deletion + ClearTableCache(table_result.tableInfo); + response.status.Succeed(); + } + } + } else { + response.status.code = StatusCode::NOT_FOUND; + response.status.errorMessage = "Cannot find table " + table_id; + } + } else { + response.status = std::move(table_result.status); + } + context->Commit(); + } + + return response; + } + + DeleteIndexResponse SqlCatalogManager::DeleteIndex(const DeleteIndexRequest& request) { + DeleteIndexResponse response; + std::string namespace_id = GetPgsqlNamespaceId(request.namespaceOid); + std::string table_id = GetPgsqlTableId(request.namespaceOid, request.tableOid); + response.namespaceId = namespace_id; + std::shared_ptr namespace_info = GetCachedNamespaceById(namespace_id); + if (namespace_info == nullptr) { + // try to refresh namespaces from SKV in case that the requested namespace is created by another catalog manager instance + // this could be avoided by use a single or a quorum of catalog managers + std::shared_ptr ns_context = NewTransactionContext(); + ListNamespacesResult result = namespace_info_handler_->ListNamespaces(ns_context); + ns_context->Commit(); + if (result.status.IsSucceeded() && !result.namespaceInfos.empty()) { + // update namespace caches + UpdateNamespaceCache(result.namespaceInfos); + // recheck namespace + namespace_info = GetCachedNamespaceById(namespace_id); + } + } + if (namespace_info == nullptr) { + LOG(FATAL) << "Cannot find namespace " << namespace_id; + response.status.code = StatusCode::NOT_FOUND; + response.status.errorMessage = "Cannot find namespace " + namespace_id; + return response; + } + + std::shared_ptr context = NewTransactionContext(); + std::shared_ptr index_info = GetCachedIndexInfoById(table_id); + std::string base_table_id; + if (index_info == nullptr) { + GeBaseTableIdResult index_result = table_info_handler_->GeBaseTableId(context, namespace_id, table_id); + if (!index_result.status.IsSucceeded()) { + response.status = std::move(index_result.status); + context->Abort(); + return response; + } + base_table_id = index_result.baseTableId; + } else { + base_table_id = index_info->indexed_table_id(); + } + + std::shared_ptr base_table_info = GetCachedTableInfoById(base_table_id); + // try to fetch the table from SKV if not found + if (base_table_info == nullptr) { + GetTableResult table_result = table_info_handler_->GetTable(context, namespace_id, namespace_info->GetNamespaceName(), + base_table_id); + if (table_result.status.IsSucceeded()) { + if (table_result.tableInfo != nullptr) { + base_table_info = table_result.tableInfo; + // delete index data + table_info_handler_->DeleteIndexData(context, namespace_id, table_id); + // delete index metadata + DeleteIndexResult delete_result = table_info_handler_->DeleteIndexMetadata(context, namespace_id, table_id); + if (delete_result.status.IsSucceeded()) { + // remove index from the table_info object + base_table_info->drop_index(table_id); + // update table cache with the index removed, index cache is updated accordingly + UpdateTableCache(base_table_info); + response.baseIndexTableOid = base_table_info->pg_oid(); + response.status.Succeed(); + } else { + response.status = std::move(delete_result.status); + } + } else { + response.status.code = StatusCode::NOT_FOUND; + response.status.errorMessage = "Base table " + base_table_id + " cannot be found"; + } + } else { + response.status = std::move(table_result.status); + } + } else { + // delete index data + table_info_handler_->DeleteIndexData(context, namespace_id, table_id); + // delete index metadata + DeleteIndexResult delete_result = table_info_handler_->DeleteIndexMetadata(context, namespace_id, table_id); + if (delete_result.status.IsSucceeded()) { + // remove index from the table_info object + base_table_info->drop_index(table_id); + // update table cache with the index removed, index cache is updated accordingly + UpdateTableCache(base_table_info); + response.baseIndexTableOid = base_table_info->pg_oid(); + response.status.Succeed(); + } else { + response.status = std::move(delete_result.status); + } + } + context->Commit(); + + return response; + } + + ReservePgOidsResponse SqlCatalogManager::ReservePgOid(const ReservePgOidsRequest& request) { + ReservePgOidsResponse response; + std::shared_ptr ns_context = NewTransactionContext(); + GetNamespaceResult result = namespace_info_handler_->GetNamespace(ns_context, request.namespaceId); + if (result.status.IsSucceeded()) { + if (result.namespaceInfo != nullptr) { + uint32_t begin_oid = result.namespaceInfo->GetNextPgOid(); + if (begin_oid < request.nextOid) { + begin_oid = request.nextOid; + } + if (begin_oid == std::numeric_limits::max()) { + LOG(WARNING) << "No more object identifier is available for Postgres database " << request.namespaceId; + response.status.code = StatusCode::INVALID_ARGUMENT; + response.status.errorMessage = "No more object identifier is available for " + request.namespaceId; + ns_context->Abort(); + return response; + } + + uint32_t end_oid = begin_oid + request.count; + if (end_oid < begin_oid) { + end_oid = std::numeric_limits::max(); // Handle wraparound. + } + response.namespaceId = request.namespaceId; + response.beginOid = begin_oid; + response.endOid = end_oid; + + // update the namespace record on SKV + // We use read and write in the same transaction so that K23SI guarantees that concurrent SKV records on SKV + // won't override each other and won't lose the correctness of PgNextOid + std::shared_ptr updated_ns = std::move(result.namespaceInfo); + updated_ns->SetNextPgOid(end_oid); + AddOrUpdateNamespaceResult update_result = namespace_info_handler_->AddOrUpdateNamespace(ns_context, updated_ns); + if (!update_result.status.IsSucceeded()) { + response.status = std::move(update_result.status); + } else { + // update namespace caches after persisting to SKV successfully + namespace_id_map_[updated_ns->GetNamespaceId()] = updated_ns; + namespace_name_map_[updated_ns->GetNamespaceName()] = updated_ns; + response.status.Succeed(); + } + } else { + response.status.code = StatusCode::NOT_FOUND; + response.status.errorMessage = "Cannot find namespace " + request.namespaceId; + } + } else { + response.status = std::move(result.status); + } + ns_context->Commit(); + + return response; + } + + RStatus SqlCatalogManager::UpdateCatalogVersion(std::shared_ptr context, uint64_t new_version) { + std::lock_guard l(lock_); + // compare new_version with the local version + uint64_t local_catalog_version = catalog_version_.load(std::memory_order_acquire); + if (new_version < local_catalog_version) { + LOG(INFO) << "Catalog version update: version on SKV is too old. " + << "New: " << new_version << ", Old: " << local_catalog_version; + ClusterInfo cluster_info(cluster_id_, init_db_done_, local_catalog_version); + UpdateClusterInfoResult result = cluster_info_handler_->UpdateClusterInfo(context, cluster_info); + if (!result.status.IsSucceeded()) { + LOG(ERROR) << "ClusterInfo update failed due to error code " << result.status.code << " and message " + << result.status.errorMessage; + return result.status; + } + } else if (new_version > local_catalog_version) { + catalog_version_.store(new_version, std::memory_order_release); + } + return StatusOK; + } + + // update namespace caches + void SqlCatalogManager::UpdateNamespaceCache(std::vector> namespace_infos) { + std::lock_guard l(lock_); + namespace_id_map_.clear(); + namespace_name_map_.clear(); + for (auto ns_ptr : namespace_infos) { + namespace_id_map_[ns_ptr->GetNamespaceId()] = ns_ptr; + namespace_name_map_[ns_ptr->GetNamespaceName()] = ns_ptr; + } + } + + // update table caches + void SqlCatalogManager::UpdateTableCache(std::shared_ptr table_info) { + std::lock_guard l(lock_); + table_id_map_[table_info->table_id()] = table_info; + // TODO: add logic to remove table with old name if rename table is called + TableNameKey key = std::make_pair(table_info->namespace_id(), table_info->table_name()); + table_name_map_[key] = table_info; + // update the corresponding index cache + UpdateIndexCacheForTable(table_info); + } + + // remove table info from table cache and its related indexes from index cache + void SqlCatalogManager::ClearTableCache(std::shared_ptr table_info) { + std::lock_guard l(lock_); + ClearIndexCacheForTable(table_info->table_id()); + table_id_map_.erase(table_info->table_id()); + TableNameKey key = std::make_pair(table_info->namespace_id(), table_info->table_name()); + table_name_map_.erase(key); + } + + // clear index infos for a table in the index cache + void SqlCatalogManager::ClearIndexCacheForTable(std::string table_id) { + std::lock_guard l(lock_); + std::vector index_ids; + for (std::pair> pair : index_id_map_) { + // first find all indexes that belong to the table + if (table_id == pair.second->indexed_table_id()) { + index_ids.push_back(pair.first); + } + } + // delete the indexes in cache + for (std::string index_id : index_ids) { + index_id_map_.erase(index_id); + } + } + + void SqlCatalogManager::UpdateIndexCacheForTable(std::shared_ptr table_info) { + std::lock_guard l(lock_); + // clear existing index informaton first + ClearIndexCacheForTable(table_info->table_id()); + // add the new indexes to the index cache + if (table_info->has_secondary_indexes()) { + for (std::pair pair : table_info->secondary_indexes()) { + AddIndexCache(std::make_shared(pair.second)); + } + } + } + + void SqlCatalogManager::AddIndexCache(std::shared_ptr index_info) { + index_id_map_[index_info->table_id()] = index_info; + } + + std::shared_ptr SqlCatalogManager::GetCachedNamespaceById(std::string namespace_id) { + if (!namespace_id_map_.empty()) { + const auto itr = namespace_id_map_.find(namespace_id); + if (itr != namespace_id_map_.end()) { + return itr->second; + } + } + return nullptr; + } + + std::shared_ptr SqlCatalogManager::GetCachedNamespaceByName(std::string namespace_name) { + if (!namespace_name_map_.empty()) { + const auto itr = namespace_name_map_.find(namespace_name); + if (itr != namespace_name_map_.end()) { + return itr->second; + } + } + return nullptr; + } + + std::shared_ptr SqlCatalogManager::GetCachedTableInfoById(std::string table_id) { + if (!table_id_map_.empty()) { + const auto itr = table_id_map_.find(table_id); + if (itr != table_id_map_.end()) { + return itr->second; + } + } + return nullptr; + } + + std::shared_ptr SqlCatalogManager::GetCachedTableInfoByName(std::string namespace_id, std::string table_name) { + if (!table_id_map_.empty()) { + TableNameKey key = std::make_pair(namespace_id, table_name); + const auto itr = table_name_map_.find(key); + if (itr != table_name_map_.end()) { + return itr->second; + } + } + return nullptr; + } + + std::shared_ptr SqlCatalogManager::GetCachedIndexInfoById(std::string index_id) { + if (!index_id_map_.empty()) { + const auto itr = index_id_map_.find(index_id); + if (itr != index_id_map_.end()) { + return itr->second; + } + } + return nullptr; + } + + std::shared_ptr SqlCatalogManager::NewTransactionContext() { + std::future txn_future = k2_adapter_->beginTransaction(); + std::shared_ptr txn = std::make_shared(txn_future.get()); + std::shared_ptr context = std::make_shared(txn); + return context; + } + + void SqlCatalogManager::IncreaseCatalogVersion() { + catalog_version_++; + // need to update the catalog version on SKV + // the update frequency could be reduced once we have a single or a quorum of catalog managers + ClusterInfo cluster_info(cluster_id_, init_db_done_, catalog_version_); + std::shared_ptr context = NewTransactionContext(); + cluster_info_handler_->UpdateClusterInfo(context, cluster_info); + context->Commit(); + } + + IndexInfo SqlCatalogManager::BuildIndexInfo(std::shared_ptr base_table_info, std::string index_id, std::string index_name, uint32_t pg_oid, + const Schema& index_schema, bool is_unique, IndexPermissions index_permissions) { + std::vector columns; + for (ColumnId col_id: index_schema.column_ids()) { + int col_idx = index_schema.find_column_by_id(col_id); + if (col_idx == Schema::kColumnNotFound) { + throw std::runtime_error("Cannot find column with id " + col_id); + } + const ColumnSchema& col_schema = index_schema.column(col_idx); + std::pair pair = base_table_info->schema().FindColumnIdByName(col_schema.name()); + if (!pair.first) { + throw std::runtime_error("Cannot find column id in base table with name " + col_schema.name()); + } + ColumnId indexed_column_id = pair.second; + IndexColumn col(col_id, col_schema.name(), indexed_column_id); + columns.push_back(col); + } + IndexInfo index_info(index_id, index_name, pg_oid, base_table_info->table_id(), index_schema.version(), + is_unique, columns, index_permissions); + return index_info; + } + +} // namespace catalog +} // namespace sql +} // namespace k2pg + + + + diff --git a/src/k2/connector/yb/pggate/catalog/sql_catalog_manager.h b/src/k2/connector/yb/pggate/catalog/sql_catalog_manager.h new file mode 100644 index 00000000..63d4f75d --- /dev/null +++ b/src/k2/connector/yb/pggate/catalog/sql_catalog_manager.h @@ -0,0 +1,366 @@ +/* +MIT License + +Copyright(c) 2020 Futurewei Cloud + + Permission is hereby granted, + free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : + + The above copyright notice and this permission notice shall be included in all copies + or + substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", + WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER + LIABILITY, + WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + +------------- READ ME -------------- + +Catalog Manager is the service that manages database metadata. It is local for now, but will become a central one +to avoid metadata update conflicts and reduce the need to access SKV frequently. + +The database data on SKV consists of the following: + +1) NamespaceInfo and TableInfo (IndexInfo): where NamespaceInfo represents database information and TableInfo for table metadata. + Both information are needed by PG Gate as database/table schema metadata to avoid deriving them by using the complicated logic in + PG from PG system catalog tables. + +2) PG system catalog tables/indexes: each table has its own SKV schema and its data are managed by DMLs. Each system catalog + table has a TableInfo representation in 1). PG still has its own internal logic to update the system catalog tables. + +3) User tables/indexes: each table has its own SKV schema and its data are managed by DMLs. The table metadata are stored + in PG system catalog and is also represented as a TableInfo in 1). + +For a namespace, i.e., database here, its identifier consists +1) name (std::string): it could be changed by DDLs such as rename database +2) PG Oid (uint32_t): object ID assigned by PG +3) id (std::string): generated UUID string based on PG Oid. It is actually used to uniquely identifier a namespace + +Similary for a table: +1) name (std::string): table name and it could be renamed by DDLs +2) PG oid (uint32_t): object ID assigned by PG +3) id (std::string): generated UUID string based on database oid and table oid. It is used to identifer a table + +NamespaceInfo, TableInfo, and IndexInfo are cached in catalog manager. + +The catalog system has a primary SKV collection to store +1) cluster level information: the init_db_done flag and catalog version. The current pg uses a global catalog version + across different databases to be used as an variable to decide whether PG needs to refresh its internal cache. + We could refactor this to avoid a global catalog version and reduce the chance for cache refresh in PG later. + One SKV record in the primary collection is used for a cluster and it is accessed by the ClusterInfoHandler + +2) namespace information: database information is shared across different databases and thus, we need to store them + in the primary SKV record. It is accessed by the NamespaceInfoHandler + +3) Table information: PG gate needs to know the table schema and it is represented by the TableInfo object. + The TableInfoHandler is used to acccess the TableInfo (including IndexInfo) objects on SKV + +All tables in a namespace (database) are mapped to a SKV collection by namespace id, not namespace name to make the +rename database working. Similary, a table's SKV schema name is identified by the table id, not the table name to +avoid moving data around after renaming a table. + +To store a TableInfo with possible multiple IndexInfo on SKV (an IndexInfo represents a secondary index for a table), +we have the following three catalog tables with fixed schema defined in table_info_handler.h for each namespace (database) +1) table head: store table and index informaton. One entry for a table or an index +2) table column: store column information for a table. One entry for a table column +3) index column: store index column for an index. One entry for an index column + +The TableInfoHandler is used to persist or read a TableInfo/IndexInfo object on SKV. When a TableInfo is stored to the +above catalog tables, the actual SKV schema is obtained dynamically and is created by TableInfoHandler as well by +table id so that DMLs could insert, update, select, and delete from the SKV table. + +Please be aware that blocking APIs are used for the catalog manager for now. Will refactor them to include task submission +and task state checking APIs later by using thread pools. + +*/ + +#ifndef CHOGORI_SQL_CATALOG_MANAGER_H +#define CHOGORI_SQL_CATALOG_MANAGER_H + +#include + +#include "yb/common/env.h" +#include "yb/common/status.h" +#include "yb/common/concurrent/locks.h" +#include "yb/entities/schema.h" +#include "yb/entities/index.h" +#include "yb/entities/table.h" +#include "yb/pggate/k2_adapter.h" +#include "yb/pggate/catalog/sql_catalog_defaults.h" +#include "yb/pggate/catalog/cluster_info_handler.h" +#include "yb/pggate/catalog/namespace_info_handler.h" +#include "yb/pggate/catalog/table_info_handler.h" + +namespace k2pg { +namespace sql { +namespace catalog { + using yb::Status; + using yb::simple_spinlock; + using k2pg::gate::K2Adapter; + using k2pg::gate::PgObjectId; + + struct GetInitDbRequest { + }; + + struct GetInitDbResponse { + RStatus status; + bool isInitDbDone; + }; + + struct GetCatalogVersionRequest { + }; + + struct GetCatalogVersionResponse { + RStatus status; + uint64_t catalogVersion; + }; + + struct CreateNamespaceRequest { + string namespaceName; + string namespaceId; + string sourceNamespaceId; + string creatorRoleName; + // next oid to assign. Ignored when sourceNamespaceId is given and the nextPgOid from source namespace will be used + std::optional nextPgOid; + }; + + struct CreateNamespaceResponse { + RStatus status; + std::shared_ptr namespaceInfo; + }; + + struct ListNamespacesRequest { + }; + + struct ListNamespacesResponse { + RStatus status; + std::vector> namespace_infos; + }; + + struct GetNamespaceRequest { + string namespaceName; + string namespaceId; + }; + + struct GetNamespaceResponse { + RStatus status; + std::shared_ptr namespace_info; + }; + + struct DeleteNamespaceRequest { + string namespaceName; + string namespaceId; + }; + + struct DeleteNamespaceResponse { + RStatus status; + }; + + struct CreateTableRequest { + string namespaceName; + uint32_t namespaceOid; + string tableName; + uint32_t tableOid; + Schema schema; + bool isSysCatalogTable; + // should put shared table in primary collection + bool isSharedTable; + bool isNotExist; + }; + + struct CreateTableResponse { + RStatus status; + std::shared_ptr tableInfo; + }; + + struct CreateIndexTableRequest { + string namespaceName; + uint32_t namespaceOid; + string tableName; + uint32_t tableOid; + uint32_t baseTableOid; + Schema schema; + bool isUnique; + bool skipIndexBackfill; + bool isSysCatalogTable; + bool isSharedTable; + bool isNotExist; + }; + + struct CreateIndexTableResponse { + RStatus status; + std::shared_ptr indexInfo; + }; + + struct GetTableSchemaRequest { + uint32_t namespaceOid; + uint32_t tableOid; + }; + + struct GetTableSchemaResponse { + RStatus status; + std::shared_ptr tableInfo; + }; + + struct DeleteTableRequest { + uint32_t namespaceOid; + uint32_t tableOid; + }; + + struct DeleteTableResponse { + RStatus status; + string namespaceId; + string tableId; + }; + + struct DeleteIndexRequest { + uint32_t namespaceOid; + uint32_t tableOid; + }; + + struct DeleteIndexResponse { + RStatus status; + string namespaceId; + uint32_t baseIndexTableOid; + }; + + struct ReservePgOidsRequest { + std::string namespaceId; + uint32_t nextOid; + uint32_t count; + }; + + struct ReservePgOidsResponse { + RStatus status; + std::string namespaceId; + // the beginning of the oid reserver, which could be higher than requested + uint32_t beginOid; + // the end (exclusive) oid reserved + uint32_t endOid; + }; + + class SqlCatalogManager : public std::enable_shared_from_this { + + public: + typedef std::shared_ptr SharedPtr; + + SqlCatalogManager(std::shared_ptr k2_adapter); + ~SqlCatalogManager(); + + CHECKED_STATUS Start(); + + virtual void Shutdown(); + + // use synchronous APIs for the first attempt here + // TODO: change them to asynchronous with status check later by introducing threadpool task processing + + GetInitDbResponse IsInitDbDone(const GetInitDbRequest& request); + + GetCatalogVersionResponse GetCatalogVersion(const GetCatalogVersionRequest& request); + + CreateNamespaceResponse CreateNamespace(const CreateNamespaceRequest& request); + + ListNamespacesResponse ListNamespaces(const ListNamespacesRequest& request); + + GetNamespaceResponse GetNamespace(const GetNamespaceRequest& request); + + DeleteNamespaceResponse DeleteNamespace(const DeleteNamespaceRequest& request); + + CreateTableResponse CreateTable(const CreateTableRequest& request); + + CreateIndexTableResponse CreateIndexTable(const CreateIndexTableRequest& request); + + GetTableSchemaResponse GetTableSchema(const GetTableSchemaRequest& request); + + DeleteTableResponse DeleteTable(const DeleteTableRequest& request); + + DeleteIndexResponse DeleteIndex(const DeleteIndexRequest& request); + + ReservePgOidsResponse ReservePgOid(const ReservePgOidsRequest& request); + + protected: + std::atomic initted_{false}; + + mutable simple_spinlock lock_; + + RStatus UpdateCatalogVersion(std::shared_ptr context, uint64_t new_version); + + void UpdateNamespaceCache(std::vector> namespace_infos); + + void UpdateTableCache(std::shared_ptr table_info); + + void ClearTableCache(std::shared_ptr table_info); + + void ClearIndexCacheForTable(std::string table_id); + + void UpdateIndexCacheForTable(std::shared_ptr table_info); + + void AddIndexCache(std::shared_ptr index_info); + + std::shared_ptr GetCachedNamespaceById(std::string namespace_id); + + std::shared_ptr GetCachedNamespaceByName(std::string namespace_name); + + std::shared_ptr GetCachedTableInfoById(std::string table_id); + + std::shared_ptr GetCachedTableInfoByName(std::string namespace_id, std::string table_name); + + std::shared_ptr GetCachedIndexInfoById(std::string index_id); + + std::shared_ptr NewTransactionContext(); + + void IncreaseCatalogVersion(); + + IndexInfo BuildIndexInfo(std::shared_ptr base_table_info, std::string index_id, std::string index_name, uint32_t pg_oid, + const Schema& index_schema, bool is_unique, IndexPermissions index_permissions); + + private: + // cluster identifier + std::string cluster_id_; + + std::shared_ptr k2_adapter_; + + // flag to indicate whether init_db is done or not + std::atomic init_db_done_{false}; + + // catalog version, 0 stands for uninitialized + std::atomic catalog_version_{0}; + + // handler to access ClusterInfo record including init_db_done flag and catalog version + std::shared_ptr cluster_info_handler_; + + // handler to access the namespace info record, which consists of database information + // and it is a shared table across all databases + std::shared_ptr namespace_info_handler_; + + // handler to access table and index information + std::shared_ptr table_info_handler_; + + // namespace information cache based on namespace id + std::unordered_map> namespace_id_map_; + + // namespace information cache based on namespace name + std::unordered_map> namespace_name_map_; + + // a table is uniquely referenced by its id, which is generated based on its + // database (namespace) PgOid and table PgOid, as a result, no namespace name is required here + std::unordered_map> table_id_map_; + + // to reference a table by its name, we have to use both namespaceId and table name + std::unordered_map, boost::hash> table_name_map_; + + // index id to quickly search for the index information and base table id + std::unordered_map> index_id_map_; + }; + +} // namespace catalog +} // namespace sql +} // namespace k2pg + +#endif //CHOGORI_SQL_CATALOG_MANAGER_H diff --git a/src/k2/connector/yb/pggate/catalog/table_info_handler.cc b/src/k2/connector/yb/pggate/catalog/table_info_handler.cc new file mode 100644 index 00000000..6d1a2222 --- /dev/null +++ b/src/k2/connector/yb/pggate/catalog/table_info_handler.cc @@ -0,0 +1,941 @@ +/* +MIT License + +Copyright(c) 2020 Futurewei Cloud + + Permission is hereby granted, + free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : + + The above copyright notice and this permission notice shall be included in all copies + or + substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", + WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER + LIABILITY, + WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ + +#include "yb/pggate/catalog/table_info_handler.h" + +#include + +#include + +namespace k2pg { +namespace sql { +namespace catalog { + +TableInfoHandler::TableInfoHandler(std::shared_ptr k2_adapter) + : BaseHandler(k2_adapter), + tablehead_schema_name_(CatalogConsts::skv_schema_name_sys_catalog_tablehead), + tablecolumn_schema_name_(CatalogConsts::skv_schema_name_sys_catalog_tablecolumn), + indexcolumn_schema_name_(CatalogConsts::skv_schema_name_sys_catalog_indexcolumn) { + tablehead_schema_ptr = std::make_shared(sys_catalog_tablehead_schema); + tablecolumn_schema_ptr = std::make_shared(sys_catalog_tablecolumn_schema); + indexcolumn_schema_ptr = std::make_shared(sys_catalog_indexcolumn_schema); +} + +TableInfoHandler::~TableInfoHandler() { +} + +CreateSysTablesResult TableInfoHandler::CheckAndCreateSystemTables(std::shared_ptr context, std::string collection_name) { + CreateSysTablesResult response; + // TODO: use sequential calls for now, could be optimized later for concurrent SKV api calls + CheckSysTableResult result = CheckAndCreateSysTable(context, collection_name, tablehead_schema_name_, tablehead_schema_ptr); + if (!result.status.IsSucceeded()) { + response.status = std::move(result.status); + return response; + } + + result = CheckAndCreateSysTable(context, collection_name, tablecolumn_schema_name_, tablecolumn_schema_ptr); + if (!result.status.IsSucceeded()) { + response.status = std::move(result.status); + return response; + } + + result = CheckAndCreateSysTable(context, collection_name, indexcolumn_schema_name_, indexcolumn_schema_ptr); + if (!result.status.IsSucceeded()) { + response.status = std::move(result.status); + return response; + } + + response.status.Succeed(); + return response; +} + +CheckSysTableResult TableInfoHandler::CheckAndCreateSysTable(std::shared_ptr context, std::string collection_name, + std::string schema_name, std::shared_ptr schema) { + // check if the schema already exists or not, which is an indication of whether if we have created the table or not + std::future schema_result_future = k2_adapter_->GetSchema(collection_name, schema_name, 1); + k2::GetSchemaResult schema_result = schema_result_future.get(); + CheckSysTableResult response; + // TODO: double check if this check is valid for schema + if (schema_result.status == k2::dto::K23SIStatus::KeyNotFound) { + LOG(INFO) << schema_name << " table does not exist in " << collection_name; + // create the table schema since it does not exist + RStatus schema_result = CreateSKVSchema(collection_name, schema); + response.status = std::move(schema_result); + } else { + response.status.Succeed(); + } + return response; +} + +CreateUpdateTableResult TableInfoHandler::CreateOrUpdateTable(std::shared_ptr context, std::string collection_name, std::shared_ptr table) { + CreateUpdateTableResult response; + // persist system catalog entries to sys tables + PersistSysTableResult sys_table_result = PersistSysTable(context, collection_name, table); + if (!sys_table_result.status.IsSucceeded()) { + response.status = std::move(sys_table_result.status); + return response; + } + // persist SKV table and index schemas + CreateUpdateSKVSchemaResult skv_schema_result = CreateOrUpdateTableSKVSchema(context, collection_name, table); + if (skv_schema_result.status.IsSucceeded()) { + response.status.Succeed(); + } else { + response.status = std::move(skv_schema_result.status); + } + return response; +} + +GetTableResult TableInfoHandler::GetTable(std::shared_ptr context, std::string namespace_id, std::string namespace_name, std::string table_id) { + GetTableResult response; + // use namespace_id, not namespace_name as the collection name + // in this we could implement rename database easily by sticking to the same namespace_id + // after the namespace_name change + // same purpose for using table_id instead of table_name + k2::dto::SKVRecord table_head_record = FetchTableHeadSKVRecord(context, namespace_id, table_id); + std::vector table_column_records = FetchTableColumnSchemaSKVRecords(context, namespace_id, table_id); + std::shared_ptr table_info = BuildTableInfo(namespace_id, namespace_name, table_head_record, table_column_records); + // check all the indexes whose IndexedTableId is table_id + std::vector index_records = FetchIndexHeadSKVRecords(context, namespace_id, table_id); + if (!index_records.empty()) { + // table has indexes defined + for (auto& index_record : index_records) { + // Fetch and build each index table + IndexInfo index_info = FetchAndBuildIndexInfo(context, namespace_id, index_record); + // populate index to table_info + table_info->add_secondary_index(index_info.table_id(), index_info); + } + } + response.status.Succeed(); + response.tableInfo = table_info; + return response; +} + +ListTablesResult TableInfoHandler::ListTables(std::shared_ptr context, std::string namespace_id, std::string namespace_name, bool isSysTableIncluded) { + ListTablesResult response; + + return response; +} + +CheckSKVSchemaResult TableInfoHandler::CheckSKVSchema(std::shared_ptr context, std::string collection_name, std::string schema_name, uint32_t version) { + CheckSKVSchemaResult response; + // use the same schema version in PG for SKV schema version + std::future schema_result_future = k2_adapter_->GetSchema(collection_name, schema_name, version); + k2::GetSchemaResult schema_result = schema_result_future.get(); + if (schema_result.status == k2::dto::K23SIStatus::KeyNotFound) { + response.schema = nullptr; + response.status.Succeed(); + } else if (schema_result.status.is2xxOK()) { + if(schema_result.schema != nullptr) { + response.schema = std::move(schema_result.schema); + response.status.Succeed(); + } else { + response.schema = nullptr; + response.status.Succeed(); + } + } else { + response.status.code = StatusCode::INTERNAL_ERROR; + response.status.errorMessage = std::move(schema_result.status.message); + } + + return response; +} + +CreateUpdateSKVSchemaResult TableInfoHandler::CreateOrUpdateTableSKVSchema(std::shared_ptr context, std::string collection_name, std::shared_ptr table) { + CreateUpdateSKVSchemaResult response; + // use table id (string) instead of table name as the schema name + CheckSKVSchemaResult check_result = CheckSKVSchema(context, collection_name, table->table_id(), table->schema().version()); + if (check_result.status.IsSucceeded()) { + if (check_result.schema == nullptr) { + // build the SKV schema from TableInfo, i.e., PG table schema + std::shared_ptr tablecolumn_schema = DeriveSKVTableSchema(table); + RStatus table_status = CreateSKVSchema(collection_name, tablecolumn_schema); + if (!table_status.IsSucceeded()) { + response.status = std::move(table_status); + return response; + } + + if (table->has_secondary_indexes()) { + std::vector> index_schemas = DeriveIndexSchemas(table); + for (std::shared_ptr index_schema : index_schemas) { + // use sequential SKV writes for now, could optimize this later + RStatus index_status = CreateSKVSchema(collection_name, index_schema); + if (!index_status.IsSucceeded()) { + response.status = std::move(table_status); + return response; + } + } + } + } + response.status.Succeed(); + } else { + response.status.code = check_result.status.code; + response.status.errorMessage = std::move(check_result.status.errorMessage); + } + + return response; +} + +CreateUpdateSKVSchemaResult TableInfoHandler::CreateOrUpdateIndexSKVSchema(std::shared_ptr context, std::string collection_name, + std::shared_ptr table, const IndexInfo& index_info) { + CreateUpdateSKVSchemaResult response; + std::shared_ptr index_schema = DeriveIndexSchema(index_info, table->schema()); + response.status = CreateSKVSchema(collection_name, index_schema); + return response; +} + +PersistSysTableResult TableInfoHandler::PersistSysTable(std::shared_ptr context, std::string collection_name, std::shared_ptr table) { + PersistSysTableResult response; + // use sequential SKV writes for now, could optimize this later + k2::dto::SKVRecord tablelist_table_record = DeriveTableHeadRecord(collection_name, table); + RStatus table_status = PersistSKVRecord(context, tablelist_table_record); + if (!table_status.IsSucceeded()) { + response.status = std::move(table_status); + return response; + } + std::vector table_column_records = DeriveTableColumnRecords(collection_name, table); + for (auto& table_column_record : table_column_records) { + RStatus column_status = PersistSKVRecord(context, table_column_record); + if (!column_status.IsSucceeded()) { + response.status = std::move(column_status); + return response; + } + } + if (table->has_secondary_indexes()) { + for( const auto& pair : table->secondary_indexes()) { + PersistIndexTableResult index_result = PersistIndexTable(context, collection_name, table, pair.second); + if (!index_result.status.IsSucceeded()) { + response.status = std::move(index_result.status); + return response; + } + } + } + response.status.Succeed(); + return response; +} + +PersistIndexTableResult TableInfoHandler::PersistIndexTable(std::shared_ptr context, std::string collection_name, std::shared_ptr table, + const IndexInfo& index_info) { + PersistIndexTableResult response; + k2::dto::SKVRecord tablelist_index_record = DeriveIndexHeadRecord(collection_name, index_info, table->is_sys_table(), table->next_column_id()); + RStatus table_status = PersistSKVRecord(context, tablelist_index_record); + if (!table_status.IsSucceeded()) { + response.status = std::move(table_status); + return response; + } + std::vector index_column_records = DeriveIndexColumnRecords(collection_name, index_info, table->schema()); + for (auto& index_column_record : index_column_records) { + RStatus index_status = PersistSKVRecord(context, index_column_record); + if (!index_status.IsSucceeded()) { + response.status = std::move(index_status); + return response; + } + } + response.status.Succeed(); + return response; +} + +// Delete table_info and the related index_info from tablehead, tablecolumn, and indexcolumn system tables +DeleteTableResult TableInfoHandler::DeleteTableMetadata(std::shared_ptr context, std::string collection_name, std::shared_ptr table) { + DeleteTableResult response; + // first delete indexes + std::vector index_records = FetchIndexHeadSKVRecords(context, collection_name, table->table_id()); + if (!index_records.empty()) { + for (auto& record : index_records) { + // get table id for the index + std::string index_id = record.deserializeNext().value(); + // delete index columns and the index head + DeleteIndexResult index_result = DeleteIndexMetadata(context, collection_name, index_id); + if (!index_result.status.IsSucceeded()) { + response.status = std::move(index_result.status); + return response; + } + } + } + + // then delete the table metadata itself + // first, fetch the table columns + std::vector table_columns = FetchTableColumnSchemaSKVRecords(context, collection_name, table->table_id()); + RStatus columns_result = BatchDeleteSKVRecords(context, table_columns); + if (!columns_result.IsSucceeded()) { + response.status = std::move(columns_result); + } else { + // fetch table head + k2::dto::SKVRecord table_head = FetchTableHeadSKVRecord(context, collection_name, table->table_id()); + // then delete table head record + RStatus head_result = DeleteSKVRecord(context, table_head); + if (!head_result.IsSucceeded()) { + response.status = std::move(head_result); + } else { + response.status.Succeed(); + } + } + + return response; +} + +// Delete the actual table records from SKV that are stored with the SKV schema name to be table_id as in table_info +DeleteTableResult TableInfoHandler::DeleteTableData(std::shared_ptr context, std::string collection_name, std::shared_ptr table) { + DeleteTableResult response; + // TODO: add a task to delete the actual data from SKV + + response.status.Succeed(); + return response; +} + +// Delete index_info from tablehead and indexcolumn system tables +DeleteIndexResult TableInfoHandler::DeleteIndexMetadata(std::shared_ptr context, std::string collection_name, std::string& index_id) { + DeleteIndexResult response; + // fetch index columns first + std::vector index_columns = FetchIndexColumnSchemaSKVRecords(context, collection_name, index_id); + // delete index columns first + RStatus columns_result = BatchDeleteSKVRecords(context, index_columns); + if (!columns_result.IsSucceeded()) { + response.status = std::move(columns_result); + } else { + // fetch index head + k2::dto::SKVRecord index_head = FetchTableHeadSKVRecord(context, collection_name, index_id); + // then delete index head record + RStatus head_result = DeleteSKVRecord(context, index_head); + if (!head_result.IsSucceeded()) { + response.status = std::move(head_result); + } else { + response.status.Succeed(); + } + } + return response; +} + +// Delete the actual index records from SKV that are stored with the SKV schema name to be table_id as in index_info +DeleteIndexResult TableInfoHandler::DeleteIndexData(std::shared_ptr context, std::string collection_name, std::string& index_id) { + DeleteIndexResult response; + // TODO: add a task to delete the actual data from SKV + + response.status.Succeed(); + return response; +} + +GeBaseTableIdResult TableInfoHandler::GeBaseTableId(std::shared_ptr context, std::string collection_name, std::string index_id) { + GeBaseTableIdResult response; + try { + // exception would be thrown if the record could not be found + k2::dto::SKVRecord index_head = FetchTableHeadSKVRecord(context, collection_name, index_id); + // TableId + index_head.skipNext(); + // TableName + index_head.skipNext(); + // TableOid + index_head.skipNext(); + // IsSysTable + index_head.skipNext(); + // IsTransactional + index_head.skipNext(); + // IsIndex + index_head.skipNext(); + // IsUnique + index_head.skipNext(); + // IndexedTableId + response.baseTableId = index_head.deserializeNext().value(); + }catch (const std::exception& e) { + response.status.code = StatusCode::RUNTIME_ERROR; + response.status.errorMessage = e.what(); + } + + return response; +} + +void TableInfoHandler::AddDefaultPartitionKeys(std::shared_ptr schema) { + // "TableId" + k2::dto::SchemaField table_id_field; + table_id_field.type = k2::dto::FieldType::STRING; + table_id_field.name = CatalogConsts::TABLE_ID_COLUMN_NAME; + schema->fields.push_back(table_id_field); + schema->partitionKeyFields.push_back(0); + // "IndexId" + k2::dto::SchemaField index_id_field; + index_id_field.type = k2::dto::FieldType::STRING; + index_id_field.name = CatalogConsts::INDEX_ID_COLUMN_NAME; + schema->fields.push_back(index_id_field); + schema->partitionKeyFields.push_back(1); +} + +std::shared_ptr TableInfoHandler::DeriveSKVTableSchema(std::shared_ptr table) { + std::shared_ptr schema = std::make_shared(); + schema->name = table->table_id(); + schema->version = table->schema().version(); + // add two partitionkey fields + AddDefaultPartitionKeys(schema); + uint32_t count = 2; + for (ColumnSchema col_schema : table->schema().columns()) { + k2::dto::SchemaField field; + field.type = ToK2Type(col_schema.type()); + field.name = col_schema.name(); + switch (col_schema.sorting_type()) { + case ColumnSchema::SortingType::kAscending: { + field.descending = false; + field.nullLast = false; + } break; + case ColumnSchema::SortingType::kDescending: { + field.descending = true; + field.nullLast = false; + } break; + case ColumnSchema::SortingType::kAscendingNullsLast: { + field.descending = false; + field.nullLast = true; + } break; + case ColumnSchema::SortingType::kDescendingNullsLast: { + field.descending = true; + field.nullLast = true; + } break; + default: break; + } + schema->fields.push_back(field); + if (col_schema.is_primary()) { + schema->partitionKeyFields.push_back(count); + } + count++; + } + + return schema; +} + +std::vector> TableInfoHandler::DeriveIndexSchemas(std::shared_ptr table) { + std::vector> response; + const IndexMap& index_map = table->secondary_indexes(); + for (const auto& pair : index_map) { + response.push_back(DeriveIndexSchema(pair.second, table->schema())); + } + return response; +} + +std::shared_ptr TableInfoHandler::DeriveIndexSchema(const IndexInfo& index_info, const Schema& base_tablecolumn_schema) { + std::shared_ptr schema = std::make_shared(); + schema->name = index_info.table_id(); + schema->version = index_info.version(); + // add two partitionkey fields: base table id + index table id + AddDefaultPartitionKeys(schema); + uint32_t count = 2; + for (IndexColumn indexcolumn_schema : index_info.columns()) { + k2::dto::SchemaField field; + field.name = indexcolumn_schema.column_name; + int column_idx = base_tablecolumn_schema.find_column_by_id(indexcolumn_schema.indexed_column_id); + if (column_idx == Schema::kColumnNotFound) { + throw std::invalid_argument("Cannot find base column " + indexcolumn_schema.indexed_column_id); + } + const ColumnSchema& col_schema = base_tablecolumn_schema.column(column_idx); + field.type = ToK2Type(col_schema.type()); + switch (col_schema.sorting_type()) { + case ColumnSchema::SortingType::kAscending: { + field.descending = false; + field.nullLast = false; + } break; + case ColumnSchema::SortingType::kDescending: { + field.descending = true; + field.nullLast = false; + } break; + case ColumnSchema::SortingType::kAscendingNullsLast: { + field.descending = false; + field.nullLast = true; + } break; + case ColumnSchema::SortingType::kDescendingNullsLast: { + field.descending = true; + field.nullLast = true; + } break; + default: break; + } + schema->fields.push_back(field); + // all index columns should be treated as primary keys + schema->partitionKeyFields.push_back(count); + count++; + } + + return schema; +} + +k2::dto::SKVRecord TableInfoHandler::DeriveTableHeadRecord(std::string collection_name, std::shared_ptr table) { + k2::dto::SKVRecord record; + // TableId + record.serializeNext(table->table_id()); + // TableName + record.serializeNext(table->table_name()); + // TableOid + record.serializeNext(table->pg_oid()); + // IsSysTable + record.serializeNext(table->is_sys_table()); + // IsTransactional + record.serializeNext(table->schema().table_properties().is_transactional()); + // IsIndex + record.serializeNext(false); + // IsUnique (for index) + record.serializeNext(false); + // IndexedTableId + record.skipNext(); + // IndexPermission + record.skipNext(); + // NextColumnId + record.serializeNext(table->next_column_id()); + // SchemaVersion + record.serializeNext(table->schema().version()); + + return record; +} + +k2::dto::SKVRecord TableInfoHandler::DeriveIndexHeadRecord(std::string collection_name, const IndexInfo& index, bool is_sys_table, int32_t next_column_id) { + k2::dto::SKVRecord record; + // TableId + record.serializeNext(index.table_id()); + // TableName + record.serializeNext(index.table_name()); + // TableOid + record.serializeNext(index.pg_oid()); + // IsSysTable + record.serializeNext(is_sys_table); + // IsTransactional + record.serializeNext(true); + // IsIndex + record.serializeNext(true); + // IsUnique (for index) + record.serializeNext(index.is_unique()); + // IndexedTableId + record.serializeNext(index.indexed_table_id()); + // IndexPermission + record.serializeNext(index.index_permissions()); + // NextColumnId + record.serializeNext(next_column_id); + // SchemaVersion + record.serializeNext(index.version()); + + return record; +} + +std::vector TableInfoHandler::DeriveTableColumnRecords(std::string collection_name, std::shared_ptr table) { + std::vector response; + for (std::size_t i = 0; i != table->schema().columns().size(); ++i) { + ColumnSchema col_schema = table->schema().columns()[i]; + int32_t column_id = table->schema().column_ids()[i]; + k2::dto::SKVRecord record; + // TableId + record.serializeNext(table->table_id()); + // ColumnId + record.serializeNext(column_id); + // ColumnName + record.serializeNext(col_schema.name()); + // ColumnType + record.serializeNext(col_schema.type()->id()); + // IsNullable + record.serializeNext(col_schema.is_nullable()); + // IsPrimary + record.serializeNext(col_schema.is_primary()); + // IsPartition + record.serializeNext(col_schema.is_partition()); + // Order + record.serializeNext(col_schema.order()); + // SortingType + record.serializeNext(col_schema.sorting_type()); + + response.push_back(std::move(record)); + } + return response; +} + +std::vector TableInfoHandler::DeriveIndexColumnRecords(std::string collection_name, const IndexInfo& index, const Schema& base_tablecolumn_schema) { + std::vector response; + int count = 0; + for (IndexColumn index_column : index.columns()) { + int column_idx = base_tablecolumn_schema.find_column_by_id(index_column.indexed_column_id); + if (column_idx == Schema::kColumnNotFound) { + throw std::invalid_argument("Cannot find base column " + index_column.indexed_column_id); + } + const ColumnSchema& col_schema = base_tablecolumn_schema.column(column_idx); + + k2::dto::SKVRecord record; + // TableId + record.serializeNext(index.table_id()); + // ColumnId + record.serializeNext(index_column.column_id); + // ColumnName + record.serializeNext(index_column.column_name); + // ColumnType + record.serializeNext(col_schema.type()->id()); + // IsNullable + record.serializeNext(false); + // IsPrimary + record.serializeNext(true); + // IsPartition + record.serializeNext(true); + // Order + record.serializeNext(count++); + // SortingType + record.serializeNext(col_schema.sorting_type()); + + response.push_back(std::move(record)); + } + return response; +} + +k2::dto::FieldType TableInfoHandler::ToK2Type(std::shared_ptr type) { + k2::dto::FieldType field_type = k2::dto::FieldType::NOT_KNOWN; + switch (type->id()) { + case DataType::INT8: { + field_type = k2::dto::FieldType::INT16T; + } break; + case DataType::INT16: { + field_type = k2::dto::FieldType::INT16T; + } break; + case DataType::INT32: { + field_type = k2::dto::FieldType::INT32T; + } break; + case DataType::INT64: { + field_type = k2::dto::FieldType::INT64T; + } break; + case DataType::STRING: { + field_type = k2::dto::FieldType::STRING; + } break; + case DataType::BOOL: { + field_type = k2::dto::FieldType::BOOL; + } break; + case DataType::FLOAT: { + field_type = k2::dto::FieldType::FLOAT; + } break; + case DataType::DOUBLE: { + field_type = k2::dto::FieldType::DOUBLE; + } break; + case DataType::BINARY: { + field_type = k2::dto::FieldType::STRING; + } break; + case DataType::TIMESTAMP: { + field_type = k2::dto::FieldType::INT64T; + } break; + case DataType::DECIMAL: { + field_type = k2::dto::FieldType::DECIMAL64; + } break; + default: + throw std::invalid_argument("Unsupported type " + type->id()); + } + return field_type; +} + +DataType TableInfoHandler::ToSqlType(k2::dto::FieldType type) { + // utility method, not used yet + DataType sql_type = DataType::NOT_SUPPORTED; + switch (type) { + case k2::dto::FieldType::INT16T: { + sql_type = DataType::INT16; + } break; + case k2::dto::FieldType::INT32T: { + sql_type = DataType::INT32; + } break; + case k2::dto::FieldType::INT64T: { + sql_type = DataType::INT64; + } break; + case k2::dto::FieldType::STRING: { + sql_type = DataType::STRING; + } break; + case k2::dto::FieldType::BOOL: { + sql_type = DataType::BOOL; + } break; + case k2::dto::FieldType::FLOAT: { + sql_type = DataType::FLOAT; + } break; + case k2::dto::FieldType::DOUBLE: { + sql_type = DataType::DOUBLE; + } break; + case k2::dto::FieldType::DECIMAL64: { + sql_type = DataType::DECIMAL; + } break; + default: { + std::ostringstream oss; + oss << "Unsupported Type: " << type; + throw std::invalid_argument(oss.str()); + } + } + return sql_type; +} + +k2::dto::SKVRecord TableInfoHandler::FetchTableHeadSKVRecord(std::shared_ptr context, std::string collection_name, std::string table_id) { + k2::dto::SKVRecord record(collection_name, tablehead_schema_ptr); + // table_id is the primary key + record.serializeNext(table_id); + std::future> result_future = context->GetTxn()->read(std::move(record)); + k2::ReadResult result = result_future.get(); + // TODO: add error handling and retry logic in catalog manager + if (result.status == k2::dto::K23SIStatus::KeyNotFound) { + throw std::runtime_error("Cannot find entry " + table_id + " in " + collection_name); + } else if (!result.status.is2xxOK()) { + std::ostringstream oss; + oss << "Error fetching entry " << table_id << " in " << collection_name << " due to " << result.status.message; + throw std::runtime_error(oss.str()); + } + return std::move(result.value); +} + +std::vector TableInfoHandler::FetchIndexHeadSKVRecords(std::shared_ptr context, std::string collection_name, std::string base_table_id) { + std::future create_result_future = k2_adapter_->CreateScanRead(collection_name, tablehead_schema_name_); + CreateScanReadResult create_result = create_result_future.get(); + if (!create_result.status.is2xxOK()) { + LOG(FATAL) << "Failed to create scan read due to error code " << create_result.status.code + << " and message: " << create_result.status.message; + std::ostringstream oss; + oss << "Failed to create scan read for " << base_table_id << " in " << collection_name << " due to " << create_result.status.message; + throw std::runtime_error(oss.str()); + } + + std::vector records; + std::shared_ptr query = create_result.query; + std::vector values; + std::vector exps; + // find all the indexes for the base table, i.e., by IndexedTableId + values.emplace_back(k2::dto::expression::makeValueReference(CatalogConsts::INDEXED_TABLE_ID_COLUMN_NAME)); + values.emplace_back(k2::dto::expression::makeValueLiteral(base_table_id)); + k2::dto::expression::Expression filterExpr = k2::dto::expression::makeExpression(k2::dto::expression::Operation::EQ, std::move(values), std::move(exps)); + query->setFilterExpression(std::move(filterExpr)); + do { + std::future query_result_future = context->GetTxn()->scanRead(query); + k2::QueryResult query_result = query_result_future.get(); + if (!query_result.status.is2xxOK()) { + LOG(FATAL) << "Failed to run scan read due to error code " << query_result.status.code + << " and message: " << query_result.status.message; + std::ostringstream oss; + oss << "Failed to create scan read for " << base_table_id << " in " << collection_name << " due to " << query_result.status.message; + throw std::runtime_error(oss.str()); + } + + if (!query_result.records.empty()) { + for (k2::dto::SKVRecord& record : query_result.records) { + records.push_back(std::move(record)); + } + } + // if the query is not done, the query itself is updated with the pagination token for the next call + } while (!query->isDone()); + + return records; +} + +std::vector TableInfoHandler::FetchTableColumnSchemaSKVRecords(std::shared_ptr context, std::string collection_name, std::string table_id) { + std::future create_result_future = k2_adapter_->CreateScanRead(collection_name, tablecolumn_schema_name_); + CreateScanReadResult create_result = create_result_future.get(); + if (!create_result.status.is2xxOK()) { + LOG(FATAL) << "Failed to create scan read due to error code " << create_result.status.code + << " and message: " << create_result.status.message; + std::ostringstream oss; + oss << "Failed to create scan read for " << table_id << " in " << collection_name << " due to " << create_result.status.message; + throw std::runtime_error(oss.str()); + } + + std::vector records; + auto& query = create_result.query; + std::vector values; + std::vector exps; + // find all the columns for a table by TableId + values.emplace_back(k2::dto::expression::makeValueReference(CatalogConsts::TABLE_ID_COLUMN_NAME)); + values.emplace_back(k2::dto::expression::makeValueLiteral(table_id)); + k2::dto::expression::Expression filterExpr = k2::dto::expression::makeExpression(k2::dto::expression::Operation::EQ, std::move(values), std::move(exps)); + query->setFilterExpression(std::move(filterExpr)); + do { + std::future query_result_future = context->GetTxn()->scanRead(query); + k2::QueryResult query_result = query_result_future.get(); + if (!query_result.status.is2xxOK()) { + LOG(FATAL) << "Failed to run scan read due to error code " << query_result.status.code + << " and message: " << query_result.status.message; + std::ostringstream oss; + oss << "Failed to run scan read for " << table_id << " in " << collection_name << " due to " << query_result.status.message; + throw std::runtime_error(oss.str()); + } + + if (!query_result.records.empty()) { + std::copy(make_move_iterator(query_result.records.begin()), + make_move_iterator(query_result.records.end()), + records.begin()); + } + // if the query is not done, the query itself is updated with the pagination token for the next call + } while (!query->isDone()); + + return records; +} + +std::vector TableInfoHandler::FetchIndexColumnSchemaSKVRecords(std::shared_ptr context, std::string collection_name, std::string table_id) { + std::future create_result_future = k2_adapter_->CreateScanRead(collection_name, indexcolumn_schema_name_); + CreateScanReadResult create_result = create_result_future.get(); + if (!create_result.status.is2xxOK()) { + LOG(FATAL) << "Failed to create scan read due to error code " << create_result.status.code + << " and message: " << create_result.status.message; + std::ostringstream oss; + oss << "Failed to create scan read for " << table_id << " in " << collection_name << " due to " << create_result.status.message; + throw std::runtime_error(oss.str()); + } + + std::vector records; + std::shared_ptr query = create_result.query; + std::vector values; + std::vector exps; + // find all the columns for an index table by TableId + values.emplace_back(k2::dto::expression::makeValueReference(CatalogConsts::TABLE_ID_COLUMN_NAME)); + values.emplace_back(k2::dto::expression::makeValueLiteral(table_id)); + k2::dto::expression::Expression filterExpr = k2::dto::expression::makeExpression(k2::dto::expression::Operation::EQ, std::move(values), std::move(exps)); + query->setFilterExpression(std::move(filterExpr)); + do { + std::future query_result_future = context->GetTxn()->scanRead(query); + k2::QueryResult query_result = query_result_future.get(); + if (!query_result.status.is2xxOK()) { + LOG(FATAL) << "Failed to run scan read due to error code " << query_result.status.code + << " and message: " << query_result.status.message; + std::ostringstream oss; + oss << "Failed to run scan read for " << table_id << " in " << collection_name << " due to " << query_result.status.message; + throw std::runtime_error(oss.str()); + } + + if (!query_result.records.empty()) { + for (k2::dto::SKVRecord& record : query_result.records) { + records.push_back(std::move(record)); + } + } + // if the query is not done, the query itself is updated with the pagination token for the next call + } while (!query->isDone()); + + return records; +} + +std::shared_ptr TableInfoHandler::BuildTableInfo(std::string namespace_id, std::string namespace_name, + k2::dto::SKVRecord& table_head, std::vector& table_columns) { + // deserialize table head + // TableId + std::string table_id = table_head.deserializeNext().value(); + // TableName + std::string table_name = table_head.deserializeNext().value(); + // TableOid + uint32_t table_oid = table_head.deserializeNext().value(); + // IsSysTable + bool is_sys_table = table_head.deserializeNext().value(); + // IsTransactional + bool is_transactional = table_head.deserializeNext().value(); + // IsIndex + bool is_index = table_head.deserializeNext().value(); + if (is_index) { + throw std::runtime_error("Table " + table_id + " should not be an index"); + } + // IsUnique + table_head.skipNext(); + // IndexedTableId + table_head.skipNext(); + // IndexPermission + table_head.skipNext(); + // NextColumnId + int32_t next_column_id = table_head.deserializeNext().value(); + // SchemaVersion + uint32_t version = table_head.deserializeNext().value(); + + TableProperties table_properties; + table_properties.SetTransactional(is_transactional); + + vector cols; + int key_columns = 0; + vector ids; + // deserialize table columns + for (auto& column : table_columns) { + // TableId + std::string tb_id = column.deserializeNext().value(); + // ColumnId + int32_t col_id = column.deserializeNext().value(); + // ColumnName + std::string col_name = column.deserializeNext().value(); + // ColumnType, we persist SQL type directly as an integer + int16_t col_type = column.deserializeNext().value(); + // IsNullable + bool is_nullable = column.deserializeNext().value(); + // IsPrimary + bool is_primary = column.deserializeNext().value(); + // IsPartition + bool is_partition = column.deserializeNext().value(); + // Order + int32 order = column.deserializeNext().value(); + // SortingType + int16_t sorting_type = column.deserializeNext().value(); + ColumnSchema col_schema(col_name, static_cast(col_type), is_nullable, is_primary, is_partition, order, static_cast(sorting_type)); + cols.push_back(std::move(col_schema)); + ids.push_back(col_id); + if (is_primary) { + key_columns++; + } + } + Schema table_schema(cols, ids, key_columns, table_properties); + table_schema.set_version(version); + std::shared_ptr table_info = std::make_shared(namespace_id, namespace_name, table_id, table_name, table_schema); + table_info->set_pg_oid(table_oid); + table_info->set_next_column_id(next_column_id); + table_info->set_is_sys_table(is_sys_table); + return table_info; +} + +IndexInfo TableInfoHandler::FetchAndBuildIndexInfo(std::shared_ptr context, std::string collection_name, k2::dto::SKVRecord& index_head) { + // deserialize index head + // TableId + std::string table_id = index_head.deserializeNext().value(); + // TableName + std::string table_name = index_head.deserializeNext().value(); + // TableOid + uint32_t table_oid = index_head.deserializeNext().value(); + // IsSysTable + index_head.skipNext(); + // IsTransactional + index_head.skipNext(); + // IsIndex + bool is_index = index_head.deserializeNext().value(); + if (!is_index) { + throw std::runtime_error("Table " + table_id + " should be an index"); + } + // IsUnique + bool is_unique = index_head.deserializeNext().value(); + // IndexedTableId + std::string indexed_table_id = index_head.deserializeNext().value(); + // IndexPermission + IndexPermissions index_perm = static_cast(index_head.deserializeNext().value()); + // NextColumnId + index_head.skipNext(); + // SchemaVersion + uint32_t version = index_head.deserializeNext().value(); + + // Fetch index columns + std::vector index_columns = FetchIndexColumnSchemaSKVRecords(context, collection_name, table_id); + + // deserialize index columns + std::vector columns; + for (auto& column : index_columns) { + // TableId + std::string tb_id = column.deserializeNext().value(); + // ColumnId + int32_t col_id = column.deserializeNext().value(); + // ColumnName + std::string col_name = column.deserializeNext().value(); + // IndexedColumnId + int32_t indexed_col_id = column.deserializeNext().value(); + // TODO: add support for expression in index + IndexColumn index_column(col_id, col_name, indexed_col_id); + columns.push_back(std::move(index_column)); + } + + IndexInfo index_info(table_id, table_name, table_oid, indexed_table_id, version, is_unique, columns, index_perm); + return index_info; +} + +} // namespace catalog +} // namespace sql +} // namespace k2pg \ No newline at end of file diff --git a/src/k2/connector/yb/pggate/catalog/table_info_handler.h b/src/k2/connector/yb/pggate/catalog/table_info_handler.h new file mode 100644 index 00000000..adaf71b6 --- /dev/null +++ b/src/k2/connector/yb/pggate/catalog/table_info_handler.h @@ -0,0 +1,225 @@ +/* +MIT License + +Copyright(c) 2020 Futurewei Cloud + + Permission is hereby granted, + free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : + + The above copyright notice and this permission notice shall be included in all copies + or + substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", + WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER + LIABILITY, + WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ + +#ifndef CHOGORI_SQL_TABLE_INFO_HANDLER_H +#define CHOGORI_SQL_TABLE_INFO_HANDLER_H + +#include +#include + +#include "yb/pggate/catalog/base_handler.h" + +namespace k2pg { +namespace sql { +namespace catalog { + +using k2pg::gate::CreateScanReadResult; + +struct CreateSysTablesResult { + RStatus status; +}; + +struct CheckSysTableResult { + RStatus status; +}; + +struct CreateUpdateTableResult { + RStatus status; +}; + +struct GetTableResult { + RStatus status; + std::shared_ptr tableInfo; +}; + +struct ListTablesResult { + RStatus status; + std::vector> tableInfos; +}; + +struct CheckSKVSchemaResult { + RStatus status; + std::shared_ptr schema; +}; + +struct CreateUpdateSKVSchemaResult { + RStatus status; +}; + +struct PersistSysTableResult { + RStatus status; +}; + +struct PersistIndexTableResult { + RStatus status; +}; + +struct DeleteTableResult { + RStatus status; +}; + +struct DeleteIndexResult { + RStatus status; +}; + +struct GeBaseTableIdResult { + RStatus status; + std::string baseTableId; +}; + +class TableInfoHandler : public BaseHandler { + public: + typedef std::shared_ptr SharedPtr; + + TableInfoHandler(std::shared_ptr k2_adapter); + ~TableInfoHandler(); + + // schema to store table information for a namespace + static inline k2::dto::Schema sys_catalog_tablehead_schema { + .name = CatalogConsts::skv_schema_name_sys_catalog_tablehead, + .version = 1, + .fields = std::vector { + {k2::dto::FieldType::STRING, "TableId", false, false}, + {k2::dto::FieldType::STRING, "TableName", false, false}, + {k2::dto::FieldType::INT64T, "TableOid", false, false}, + {k2::dto::FieldType::BOOL, "IsSysTable", false, false}, + {k2::dto::FieldType::BOOL, "IsTransactional", false, false}, + {k2::dto::FieldType::BOOL, "IsIndex", false, false}, + {k2::dto::FieldType::BOOL, "IsUnique", false, false}, + {k2::dto::FieldType::STRING, "IndexedTableId", false, false}, + {k2::dto::FieldType::INT16T, "IndexPermission", false, false}, + {k2::dto::FieldType::INT32T, "NextColumnId", false, false}, + {k2::dto::FieldType::INT32T, "SchemaVersion", false, false}}, + .partitionKeyFields = std::vector { 0 }, + .rangeKeyFields = std::vector {} + }; + + // schema to store table column schema information + static inline k2::dto::Schema sys_catalog_tablecolumn_schema { + .name = CatalogConsts::skv_schema_name_sys_catalog_tablecolumn, + .version = 1, + .fields = std::vector { + {k2::dto::FieldType::STRING, "TableId", false, false}, + {k2::dto::FieldType::INT32T, "ColumnId", false, false}, + {k2::dto::FieldType::STRING, "ColumnName", false, false}, + {k2::dto::FieldType::INT16T, "ColumnType", false, false}, + {k2::dto::FieldType::BOOL, "IsNullable", false, false}, + {k2::dto::FieldType::BOOL, "IsPrimary", false, false}, + {k2::dto::FieldType::BOOL, "IsPartition", false, false}, + {k2::dto::FieldType::INT32T, "Order", false, false}, + {k2::dto::FieldType::INT16T, "SortingType", false, false}}, + .partitionKeyFields = std::vector { 0 , 1}, + .rangeKeyFields = std::vector {} + }; + + // schema to store index column schema information + static inline k2::dto::Schema sys_catalog_indexcolumn_schema { + .name = CatalogConsts::skv_schema_name_sys_catalog_indexcolumn, + .version = 1, + .fields = std::vector { + {k2::dto::FieldType::STRING, "TableId", false, false}, + {k2::dto::FieldType::INT32T, "ColumnId", false, false}, + {k2::dto::FieldType::STRING, "ColumnName", false, false}, + {k2::dto::FieldType::INT32T, "IndexedColumnId", false, false}}, + .partitionKeyFields = std::vector { 0, 1 }, + .rangeKeyFields = std::vector {} + }; + + CreateSysTablesResult CheckAndCreateSystemTables(std::shared_ptr context, std::string collection_name); + + CreateUpdateTableResult CreateOrUpdateTable(std::shared_ptr context, std::string collection_name, std::shared_ptr table); + + GetTableResult GetTable(std::shared_ptr context, std::string namespace_id, std::string namespace_name, std::string table_id); + + ListTablesResult ListTables(std::shared_ptr context, std::string namespace_id, std::string namespace_name, bool isSysTableIncluded); + + CheckSKVSchemaResult CheckSKVSchema(std::shared_ptr context, std::string collection_name, std::string schema_name, uint32_t version); + + CreateUpdateSKVSchemaResult CreateOrUpdateTableSKVSchema(std::shared_ptr context, std::string collection_name, std::shared_ptr table); + + CreateUpdateSKVSchemaResult CreateOrUpdateIndexSKVSchema(std::shared_ptr context, std::string collection_name, + std::shared_ptr table, const IndexInfo& index_info); + + PersistSysTableResult PersistSysTable(std::shared_ptr context, std::string collection_name, std::shared_ptr table); + + PersistIndexTableResult PersistIndexTable(std::shared_ptr context, std::string collection_name, std::shared_ptr table, const IndexInfo& index_info); + + DeleteTableResult DeleteTableMetadata(std::shared_ptr context, std::string collection_name, std::shared_ptr table); + + DeleteTableResult DeleteTableData(std::shared_ptr context, std::string collection_name, std::shared_ptr table); + + DeleteIndexResult DeleteIndexMetadata(std::shared_ptr context, std::string collection_name, std::string& index_id); + + DeleteIndexResult DeleteIndexData(std::shared_ptr context, std::string collection_name, std::string& index_id); + + GeBaseTableIdResult GeBaseTableId(std::shared_ptr context, std::string collection_name, std::string index_id); + + private: + CheckSysTableResult CheckAndCreateSysTable(std::shared_ptr context, std::string collection_name, std::string schema_name, + std::shared_ptr schema); + + std::shared_ptr DeriveSKVTableSchema(std::shared_ptr table); + + std::vector> DeriveIndexSchemas(std::shared_ptr table); + + std::shared_ptr DeriveIndexSchema(const IndexInfo& index_info, const Schema& base_tablecolumn_schema); + + k2::dto::SKVRecord DeriveTableHeadRecord(std::string collection_name, std::shared_ptr table); + + k2::dto::SKVRecord DeriveIndexHeadRecord(std::string collection_name, const IndexInfo& index, bool is_sys_table, int32_t next_column_id); + + std::vector DeriveTableColumnRecords(std::string collection_name, std::shared_ptr table); + + std::vector DeriveIndexColumnRecords(std::string collection_name, const IndexInfo& index, const Schema& base_tablecolumn_schema); + + k2::dto::FieldType ToK2Type(std::shared_ptr type); + + DataType ToSqlType(k2::dto::FieldType type); + + k2::dto::SKVRecord FetchTableHeadSKVRecord(std::shared_ptr context, std::string collection_name, std::string table_id); + + std::vector FetchIndexHeadSKVRecords(std::shared_ptr context, std::string collection_name, std::string base_table_id); + + std::vector FetchTableColumnSchemaSKVRecords(std::shared_ptr context, std::string collection_name, std::string table_id); + + std::vector FetchIndexColumnSchemaSKVRecords(std::shared_ptr context, std::string collection_name, std::string table_id); + + std::shared_ptr BuildTableInfo(std::string namespace_id, std::string namespace_name, k2::dto::SKVRecord& table_head, std::vector& table_columns); + + IndexInfo FetchAndBuildIndexInfo(std::shared_ptr context, std::string collection_name, k2::dto::SKVRecord& index_head); + + void AddDefaultPartitionKeys(std::shared_ptr schema); + + std::string tablehead_schema_name_; + std::string tablecolumn_schema_name_; + std::string indexcolumn_schema_name_; + std::shared_ptr tablehead_schema_ptr; + std::shared_ptr tablecolumn_schema_ptr; + std::shared_ptr indexcolumn_schema_ptr; +}; + +} // namespace catalog +} // namespace sql +} // namespace k2pg + +#endif //CHOGORI_SQL_TABLE_INFO_HANDLER_H \ No newline at end of file diff --git a/src/k2/connector/yb/pggate/k2_adapter.cc b/src/k2/connector/yb/pggate/k2_adapter.cc index cd389f81..3f7415bc 100644 --- a/src/k2/connector/yb/pggate/k2_adapter.cc +++ b/src/k2/connector/yb/pggate/k2_adapter.cc @@ -26,6 +26,19 @@ Status K2Adapter::Shutdown() { return Status::OK(); } +std::future K2Adapter::GetSchema(const std::string& collectionName, const std::string& schemaName, uint64_t schemaVersion) { + return k23si_->getSchema(collectionName, schemaName, schemaVersion); +} + +std::future K2Adapter::CreateSchema(const std::string& collectionName, std::shared_ptr schema) { + return k23si_->createSchema(collectionName, *schema.get()); +} + +std::future K2Adapter::CreateScanRead(const std::string& collectionName, + const std::string& schemaName) { + return k23si_->createScanRead(collectionName, schemaName); +} + std::future K2Adapter::Exec(std::shared_ptr k23SITxn, std::shared_ptr op) { // TODO: add implementation // 1) check the request in op and construct the SKV request based on the op type, i.e., READ or WRITE diff --git a/src/k2/connector/yb/pggate/k2_adapter.h b/src/k2/connector/yb/pggate/k2_adapter.h index 99c31ca1..effecb16 100644 --- a/src/k2/connector/yb/pggate/k2_adapter.h +++ b/src/k2/connector/yb/pggate/k2_adapter.h @@ -49,6 +49,13 @@ class K2Adapter { CHECKED_STATUS Shutdown(); + std::future GetSchema(const std::string& collectionName, const std::string& schemaName, uint64_t schemaVersion); + + std::future CreateSchema(const std::string& collectionName, std::shared_ptr schema); + + std::future CreateScanRead(const std::string& collectionName, + const std::string& schemaName); + std::future Exec(std::shared_ptr k23SITxn, std::shared_ptr op); std::future BatchExec(std::shared_ptr k23SITxn, const std::vector>& ops); diff --git a/src/k2/connector/yb/pggate/pg_dml.cc b/src/k2/connector/yb/pggate/pg_dml.cc index d16d7d75..f5aee0dc 100644 --- a/src/k2/connector/yb/pggate/pg_dml.cc +++ b/src/k2/connector/yb/pggate/pg_dml.cc @@ -347,7 +347,7 @@ Result PgDml::GetNextRow(PgTuple *pg_tuple) { Result PgDml::BuildYBTupleId(const PgAttrValueDescriptor *attrs, int32_t nattrs) { // TODO: generate the row id by calling K2 Adapter to use SKV client to // generate the id in string format from the primary keys - throw new std::logic_error("Not implemented yet"); + throw std::logic_error("Not implemented yet"); } bool PgDml::has_aggregate_targets() { diff --git a/src/k2/connector/yb/pggate/pg_env.h b/src/k2/connector/yb/pggate/pg_env.h index 5a4b90e2..c0b8439d 100644 --- a/src/k2/connector/yb/pggate/pg_env.h +++ b/src/k2/connector/yb/pggate/pg_env.h @@ -69,7 +69,7 @@ struct PgObjectId { return database_oid != kPgInvalidOid && object_oid != kPgInvalidOid; } - TableId GetYBTableId() const { + TableId GetPgTableId() const { return GetPgsqlTableId(database_oid, object_oid); } diff --git a/src/k2/connector/yb/pggate/pg_gate_api.cc b/src/k2/connector/yb/pggate/pg_gate_api.cc index ae3d6a62..0a49eff9 100644 --- a/src/k2/connector/yb/pggate/pg_gate_api.cc +++ b/src/k2/connector/yb/pggate/pg_gate_api.cc @@ -10,9 +10,9 @@ namespace k2pg { namespace gate { -using yb::Status; -using k2pg::sql::SqlCatalogManager; +using yb::Status; +using k2pg::sql::catalog::SqlCatalogManager; namespace { // Using a raw pointer here to fully control object initialization and destruction. @@ -904,4 +904,4 @@ void YBCShutdownPgGateBackend() { } // extern "C" } // namespace gate -} // namespace k2pg \ No newline at end of file +} // namespace k2pg diff --git a/src/k2/connector/yb/pggate/pg_gate_impl.h b/src/k2/connector/yb/pggate/pg_gate_impl.h index b51ffe82..2a4e53a8 100644 --- a/src/k2/connector/yb/pggate/pg_gate_impl.h +++ b/src/k2/connector/yb/pggate/pg_gate_impl.h @@ -65,8 +65,8 @@ #include "yb/pggate/pg_statement.h" #include "yb/pggate/pg_txn_handler.h" #include "yb/pggate/k2_adapter.h" -#include "yb/pggate/sql_catalog_client.h" -#include "yb/pggate/sql_catalog_manager.h" +#include "yb/pggate/catalog/sql_catalog_client.h" +#include "yb/pggate/catalog/sql_catalog_manager.h" namespace k2pg { namespace gate { @@ -76,7 +76,8 @@ using yb::MetricEntity; using yb::MetricRegistry; using yb::Status; using k2pg::sql::PgExpr; -using k2pg::sql::SqlCatalogManager; +using k2pg::sql::catalog::SqlCatalogClient; +using k2pg::sql::catalog::SqlCatalogManager; //-------------------------------------------------------------------------------------------------- // Implements support for CAPI. @@ -509,4 +510,4 @@ class PgGateApiImpl { } // namespace gate } // namespace k2pg -#endif //CHOGORI_GATE_API_H \ No newline at end of file +#endif //CHOGORI_GATE_API_H diff --git a/src/k2/connector/yb/pggate/pg_op_api.h b/src/k2/connector/yb/pggate/pg_op_api.h index af8f56cc..dab5c469 100644 --- a/src/k2/connector/yb/pggate/pg_op_api.h +++ b/src/k2/connector/yb/pggate/pg_op_api.h @@ -269,7 +269,6 @@ namespace gate { RequestStatus status = RequestStatus::PGSQL_STATUS_OK; bool skipped; string error_message; - int32_t rows_data_sidecar; std::unique_ptr paging_state; int32_t rows_affected_count; diff --git a/src/k2/connector/yb/pggate/pg_select.cc b/src/k2/connector/yb/pggate/pg_select.cc index d6809d55..b26ee95d 100644 --- a/src/k2/connector/yb/pggate/pg_select.cc +++ b/src/k2/connector/yb/pggate/pg_select.cc @@ -136,7 +136,7 @@ Status PgSelectIndex::PrepareQuery(std::shared_ptr read_req) { // case. DSCHECK(prepare_params_.querying_colocated_table, InvalidArgument, "Read request invalid"); read_req_ = read_req; - read_req_->table_name = index_id_.GetYBTableId(); + read_req_->table_name = index_id_.GetPgTableId(); sql_op_ = nullptr; } else { auto read_op = target_desc_->NewPgsqlSelect(client_id_, stmt_id_); diff --git a/src/k2/connector/yb/pggate/pg_session.cc b/src/k2/connector/yb/pggate/pg_session.cc index cfc40836..9d858341 100644 --- a/src/k2/connector/yb/pggate/pg_session.cc +++ b/src/k2/connector/yb/pggate/pg_session.cc @@ -222,9 +222,9 @@ Status PgSession::DeleteDBSequences(int64_t db_oid) { return Status::OK(); } -void PgSession::InvalidateTableCache(const PgObjectId& table_id) { - const TableId yb_table_id = table_id.GetYBTableId(); - table_cache_.erase(yb_table_id); +void PgSession::InvalidateTableCache(const PgObjectId& table_obj_id) { + const TableId pg_table_id = table_obj_id.GetPgTableId(); + table_cache_.erase(pg_table_id); } void PgSession::StartOperationsBuffering() { @@ -406,7 +406,7 @@ Result PgSession::RunHelper::Flush() { Result> PgSession::LoadTable(const PgObjectId& table_id) { VLOG(3) << "Loading table descriptor for " << table_id; - const TableId yb_table_id = table_id.GetYBTableId(); + const TableId yb_table_id = table_id.GetPgTableId(); std::shared_ptr table; auto cached_table = table_cache_.find(yb_table_id); diff --git a/src/k2/connector/yb/pggate/pg_session.h b/src/k2/connector/yb/pggate/pg_session.h index e8a6cca0..ddea15c8 100644 --- a/src/k2/connector/yb/pggate/pg_session.h +++ b/src/k2/connector/yb/pggate/pg_session.h @@ -64,13 +64,13 @@ #include "yb/pggate/pg_op_api.h" #include "yb/pggate/pg_gate_api.h" #include "yb/pggate/pg_txn_handler.h" -#include "yb/pggate/sql_catalog_client.h" +#include "yb/pggate/catalog/sql_catalog_client.h" namespace k2pg { namespace gate { using k2pg::sql::IndexPermissions; -using k2pg::sql::SqlCatalogClient; +using k2pg::sql::catalog::SqlCatalogClient; using yb::ObjectIdGenerator; using yb::MonoDelta; using yb::Status; diff --git a/src/k2/connector/yb/pggate/sql_catalog_client.cc b/src/k2/connector/yb/pggate/sql_catalog_client.cc deleted file mode 100644 index 4542e76f..00000000 --- a/src/k2/connector/yb/pggate/sql_catalog_client.cc +++ /dev/null @@ -1,193 +0,0 @@ -/* -MIT License - -Copyright(c) 2020 Futurewei Cloud - - Permission is hereby granted, - free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : - - The above copyright notice and this permission notice shall be included in all copies - or - substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", - WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, - DAMAGES OR OTHER - LIABILITY, - WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. -*/ - -#include "yb/pggate/sql_catalog_client.h" - -namespace k2pg { -namespace sql { - -Status SqlCatalogClient::IsInitDbDone(bool* isDone) { - return catalog_manager_->IsInitDbDone(isDone); -} - -Status SqlCatalogClient::CreateNamespace(const std::string& namespace_name, - const std::string& creator_role_name, - const std::string& namespace_id, - const std::string& source_namespace_id, - const std::optional& next_pg_oid) { - std::shared_ptr request = std::make_shared(); - request->namespaceName = std::move(namespace_name); - if(!namespace_id.empty()) { - request->namespaceId = std::move(namespace_id); - } - if (!creator_role_name.empty()) { - request->creatorRoleName = std::move(creator_role_name); - } - if (!source_namespace_id.empty()) { - request->sourceNamespaceId = std::move(source_namespace_id); - } - if (next_pg_oid) { - request->nextPgOid = next_pg_oid; - } - std::shared_ptr response; - catalog_manager_->CreateNamespace(request, &response); - if (!response->errorMessage.empty()) { - return STATUS_SUBSTITUTE(RuntimeError, - "Failed to create namespace $0 due to error: $1", namespace_name, response->errorMessage); - } - return Status::OK(); -} - -Status SqlCatalogClient::DeleteNamespace(const std::string& namespace_name, - const std::string& namespace_id) { - std::shared_ptr request = std::make_shared(); - request->namespaceName = std::move(namespace_name); - if (!namespace_id.empty()) { - request->namespaceId = std::move(namespace_id); - } - std::shared_ptr response = std::make_shared(); - catalog_manager_->DeleteNamespace(request, &response); - if (!response->errorMessage.empty()) { - return STATUS_SUBSTITUTE(RuntimeError, - "Failed to delete namespace $0 due to error: $1", namespace_name, response->errorMessage); - } - return Status::OK(); -} - -Status SqlCatalogClient::CreateTable( - const std::string& namespace_name, - const std::string& table_name, - const PgObjectId& table_id, - PgSchema& schema, - bool is_pg_catalog_table, - bool is_shared_table, - bool if_not_exist) { - std::shared_ptr request = std::make_shared(); - request->namespaceName = std::move(namespace_name); - request->namespaceId = table_id.database_oid; - request->tableName = std::move(table_name); - request->tableId = table_id.object_oid; - request->schema = std::move(schema); - request->isSysCatalogTable = is_pg_catalog_table; - request->isSharedTable = is_shared_table; - std::shared_ptr response = std::make_shared(); - catalog_manager_->CreateTable(request, &response); - if (!response->errorMessage.empty()) { - return STATUS_SUBSTITUTE(RuntimeError, - "Failed to create table $0 in database $1 due to error: $2", table_name, namespace_name, response->errorMessage); - } - return Status::OK(); -} - -Status SqlCatalogClient::CreateIndexTable( - const std::string& namespace_name, - const std::string& table_name, - const PgObjectId& table_id, - const PgObjectId& base_table_id, - PgSchema& schema, - bool is_unique_index, - bool skip_index_backfill, - bool is_pg_catalog_table, - bool is_shared_table, - bool if_not_exist) { - - // TODO: add implementation - return Status::OK(); -} - -Status SqlCatalogClient::DeleteTable(const PgOid database_oid, const PgOid table_id, bool wait) { - std::shared_ptr request = std::make_shared(); - request->namespaceId = database_oid; - request->tableId = table_id; - request->isIndexTable = false; - std::shared_ptr response = std::make_shared(); - catalog_manager_->DeleteTable(request, &response); - if (!response->errorMessage.empty()) { - return STATUS_SUBSTITUTE(RuntimeError, - "Failed to delete table $0 due to error: $1", table_id, response->errorMessage); - } - return Status::OK(); -} - -Status SqlCatalogClient::DeleteIndexTable(const PgOid database_oid, const PgOid table_id, PgOid *base_table_id, bool wait) { - std::shared_ptr request = std::make_shared(); - request->namespaceId = database_oid; - request->tableId = table_id; - request->isIndexTable = true; - std::shared_ptr response = std::make_shared(); - catalog_manager_->DeleteTable(request, &response); - if (!response->errorMessage.empty()) { - return STATUS_SUBSTITUTE(RuntimeError, - "Failed to delete index table $0 due to error: $1", table_id, response->errorMessage); - } - *base_table_id = response->indexedTableId; - return Status::OK(); -} - -Status SqlCatalogClient::OpenTable(const PgOid database_oid, const PgOid table_id, std::shared_ptr* table) { - std::shared_ptr request = std::make_shared(); - request->namespaceId = database_oid; - request->tableId = table_id; - std::shared_ptr response = std::make_shared(); - catalog_manager_->GetTableSchema(request, &response); - if (!response->errorMessage.empty()) { - return STATUS_SUBSTITUTE(RuntimeError, - "Failed to get schema for table $0 due to error: $1", table_id, response->errorMessage); - } - std::shared_ptr result = std::make_shared(response->namespaceName, response->tableName, response->schema); - // TODO: double check wether we treat indexInfo as secondary Index or the index table itself - if (response->indexInfo != std::nullopt) { - PgObjectId pgObjectId(database_oid, table_id); - result->add_secondary_index(pgObjectId.GetYBTableId(), response->indexInfo.value()); - } - table->swap(result); - return Status::OK(); -} - -Status SqlCatalogClient::ReservePgOids(const PgOid database_oid, - const uint32_t next_oid, - const uint32_t count, - uint32_t* begin_oid, - uint32_t* end_oid) { - std::shared_ptr request = std::make_shared(); - request->namespaceId = database_oid; - request->nextOid = next_oid; - request->count = count; - std::shared_ptr response = std::make_shared(); - catalog_manager_->ReservePgOid(request, &response); - if (!response->errorMessage.empty()) { - return STATUS_SUBSTITUTE(RuntimeError, - "Failed to reserve PG Oids for database $0 due to error: $1", database_oid, response->errorMessage); - } - *begin_oid = response->beginOid; - *end_oid = response->endOid; - return Status::OK(); -} - -Status SqlCatalogClient::GetCatalogVersion(uint64_t *catalog_version) { - *catalog_version = catalog_manager_->GetCatalogVersion(); - return Status::OK(); -} - -} // namespace sql -} // namespace k2pg diff --git a/src/k2/connector/yb/pggate/sql_catalog_manager.cc b/src/k2/connector/yb/pggate/sql_catalog_manager.cc deleted file mode 100644 index 295aa684..00000000 --- a/src/k2/connector/yb/pggate/sql_catalog_manager.cc +++ /dev/null @@ -1,127 +0,0 @@ -/* -MIT License - -Copyright(c) 2020 Futurewei Cloud - - Permission is hereby granted, - free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : - - The above copyright notice and this permission notice shall be included in all copies - or - substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", - WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, - DAMAGES OR OTHER - LIABILITY, - WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. -*/ - -#include "yb/pggate/sql_catalog_manager.h" - -#include -#include -#include -#include - -#include - -#include "yb/common/status.h" -#include "yb/common/env.h" - -namespace k2pg { -namespace sql { - using yb::Status; - using k2pg::gate::K2Adapter; - - SqlCatalogManager::SqlCatalogManager(std::shared_ptr k2_adapter) : k2_adapter_(k2_adapter) { - } - - SqlCatalogManager::~SqlCatalogManager() { - } - - Status SqlCatalogManager::Start() { - CHECK(!initted_.load(std::memory_order_acquire)); - // TODO: initialization steps - - initted_.store(true, std::memory_order_release); - return Status::OK(); - } - - void SqlCatalogManager::Shutdown() { - LOG(INFO) << "SQL CatalogManager shutting down..."; - - bool expected = true; - if (initted_.compare_exchange_strong(expected, false, std::memory_order_acq_rel)) { - // TODO: shut down steps - - } - - LOG(INFO) << "SQL CatalogManager shut down complete. Bye!"; - } - - Status SqlCatalogManager::IsInitDbDone(bool* isDone) { - *isDone = init_db_done_; - return Status::OK(); - } - - void SqlCatalogManager::SetCatalogVersion(uint64_t new_version) { - std::lock_guard l(lock_); - uint64_t ysql_catalog_version_ = catalog_version_.load(std::memory_order_acquire); - if (new_version > ysql_catalog_version_) { - catalog_version_.store(new_version, std::memory_order_release); - } else if (new_version < ysql_catalog_version_) { - LOG(DFATAL) << "Ignoring ysql catalog version update: new version too old. " - << "New: " << new_version << ", Old: " << ysql_catalog_version_; - } - } - - uint64_t SqlCatalogManager::GetCatalogVersion() const { - return catalog_version_; - } - - Status SqlCatalogManager::CreateNamespace(const std::shared_ptr request, std::shared_ptr* response) { - return Status::OK(); - } - - Status SqlCatalogManager::ListNamespaces(const std::shared_ptr request, std::shared_ptr* response) { - return Status::OK(); - } - - Status SqlCatalogManager::GetNamespace(const std::shared_ptr request, std::shared_ptr* response) { - return Status::OK(); - } - - Status SqlCatalogManager::DeleteNamespace(const std::shared_ptr request, std::shared_ptr *response) { - return Status::OK(); - } - - Status SqlCatalogManager::CreateTable(const std::shared_ptr request, std::shared_ptr* response) { - return Status::OK(); - } - - Status SqlCatalogManager::GetTableSchema(const std::shared_ptr request, std::shared_ptr* response) { - return Status::OK(); - } - - Status SqlCatalogManager::ListTables(const std::shared_ptr request, std::shared_ptr* response) { - return Status::OK(); - } - - Status SqlCatalogManager::DeleteTable(const std::shared_ptr request, std::shared_ptr * response) { - return Status::OK(); - } - - Status SqlCatalogManager::ReservePgOid(const std::shared_ptr request, std::shared_ptr* response) { - return Status::OK(); - } -} // namespace sql -} // namespace k2pg - - - - diff --git a/src/k2/connector/yb/pggate/sql_catalog_manager.h b/src/k2/connector/yb/pggate/sql_catalog_manager.h deleted file mode 100644 index 3647fca8..00000000 --- a/src/k2/connector/yb/pggate/sql_catalog_manager.h +++ /dev/null @@ -1,214 +0,0 @@ -/* -MIT License - -Copyright(c) 2020 Futurewei Cloud - - Permission is hereby granted, - free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : - - The above copyright notice and this permission notice shall be included in all copies - or - substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", - WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, - DAMAGES OR OTHER - LIABILITY, - WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. -*/ - -#ifndef CHOGORI_SQL_CATALOG_MANAGER_H -#define CHOGORI_SQL_CATALOG_MANAGER_H - -#include "yb/common/env.h" -#include "yb/common/status.h" -#include "yb/common/concurrent/locks.h" -#include "yb/entities/schema.h" -#include "yb/entities/index.h" -#include "yb/pggate/k2_adapter.h" - -namespace k2pg { -namespace sql { - using yb::Env; - using yb::Status; - using yb::simple_spinlock; - using k2pg::gate::K2Adapter; - - struct CreateNamespaceRequest { - string namespaceName; - string namespaceId; - string sourceNamespaceId; - string creatorRoleName; - // next oid to assign. Ignored when sourceNamespaceId is given and the nextPgOid from source namespace will be used - std::optional nextPgOid; - }; - - struct CreateNamespaceResponse { - string namespaceId; - string errorMessage; - }; - - struct ListNamespacesRequest { - }; - - struct ListNamespaceResponse { - std::vector namespaceNames; - string errorMessage; - }; - - struct GetNamespaceRequest { - string namespaceName; - string namespaceId; - }; - - struct GetNamespaceResponse { - string namespaceName; - string namespaceId; - string errorMessage; - }; - - struct DeleteNamespaceRequest { - string namespaceName; - string namespaceId; - }; - - struct DeleteNamespaceResponse { - string namespaceName; - string namespaceId; - string errorMessage; - }; - - struct CreateTableRequest { - string namespaceName; - uint32_t namespaceId; - string tableName; - uint32_t tableId; - Schema schema; - bool isSysCatalogTable; - bool isSharedTable; - - // for index table - std::optional indexInfo; - }; - - struct CreateTableResponse { - uint32_t namespaceId; - uint32_t tableId; - string errorMessage; - }; - - struct GetTableSchemaRequest { - uint32_t namespaceId; - uint32_t tableId; - }; - - struct GetTableSchemaResponse { - uint32_t namespaceId; - string namespaceName; - uint32_t tableId; - string tableName; - Schema schema; - uint32_t version; - std::optional indexInfo; - string errorMessage; - }; - - struct ListTablesRequest { - string namespaceName; - // use string match for table name - string nameFilter; - bool excludeSystemTables = false; - }; - - struct ListTablesResponse { - string namespaceName; - std::vector tableNames; - string errorMessage; - }; - - struct DeleteTableRequest { - uint32_t namespaceId; - uint32_t tableId; - bool isIndexTable; - }; - - struct DeleteTableResponse { - uint32_t namespaceId; - uint32_t tableId; - uint32_t indexedTableId; - string errorMessage; - }; - - struct ReservePgOidsRequest { - uint32_t namespaceId; - uint32_t nextOid; - uint32_t count; - }; - - struct ReservePgOidsResponse { - uint32_t namespaceId; - // the beginning of the oid reserver, which could be higher than requested - uint32_t beginOid; - // the end (exclusive) oid reserved - uint32_t endOid; - string errorMessage; - }; - - class SqlCatalogManager : public std::enable_shared_from_this{ - - public: - typedef std::shared_ptr SharedPtr; - - SqlCatalogManager(std::shared_ptr k2_adapter); - ~SqlCatalogManager(); - - CHECKED_STATUS Start(); - - virtual void Shutdown(); - - CHECKED_STATUS IsInitDbDone(bool* isDone); - - void SetCatalogVersion(uint64_t new_version); - - uint64_t GetCatalogVersion() const; - - CHECKED_STATUS CreateNamespace(const std::shared_ptr request, std::shared_ptr* response); - - CHECKED_STATUS ListNamespaces(const std::shared_ptr request, std::shared_ptr* response); - - CHECKED_STATUS GetNamespace(const std::shared_ptr request, std::shared_ptr* response); - - CHECKED_STATUS DeleteNamespace(const std::shared_ptr request, std::shared_ptr *response); - - CHECKED_STATUS CreateTable(const std::shared_ptr request, std::shared_ptr* response); - - CHECKED_STATUS GetTableSchema(const std::shared_ptr request, std::shared_ptr* response); - - CHECKED_STATUS ListTables(const std::shared_ptr request, std::shared_ptr* response); - - CHECKED_STATUS DeleteTable(const std::shared_ptr request, std::shared_ptr * response); - - CHECKED_STATUS ReservePgOid(const std::shared_ptr request, std::shared_ptr* response); - - protected: - std::atomic initted_{false}; - - mutable simple_spinlock lock_; - - private: - std::shared_ptr k2_adapter_; - - std::atomic init_db_done_{false}; - - std::atomic catalog_version_{0}; - - }; - -} // namespace sql -} // namespace k2pg - -#endif //CHOGORI_SQL_CATALOG_MANAGER_H