Skip to content

Commit

Permalink
feat: create table with map type columns
Browse files Browse the repository at this point in the history
  • Loading branch information
@aceforeverd
aceforeverd committed Jan 24, 2024
1 parent 1956cc0 commit 7ce7503
Show file tree
Hide file tree
Showing 13 changed files with 371 additions and 131 deletions.
3 changes: 3 additions & 0 deletions hybridse/include/node/sql_node.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1888,6 +1888,9 @@ class ColumnDefNode : public SqlNode {

std::string GetColumnName() const { return column_name_; }

const ColumnSchemaNode *schema() const { return schema_; }

// deprecated, use ColumnDefNode::schema instead
DataType GetColumnType() const { return schema_->type(); }

const ExprNode* GetDefaultValue() const { return schema_->default_value(); }
Expand Down
12 changes: 7 additions & 5 deletions hybridse/src/codegen/block_ir_builder.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -290,16 +290,18 @@ bool BlockIRBuilder::BuildReturnStmt(const ::hybridse::node::FnReturnStmt *node,
}
::llvm::Value *value = value_wrapper.GetValue(&builder);
if (TypeIRBuilder::IsStructPtr(value->getType())) {
StructTypeIRBuilder *struct_builder =
StructTypeIRBuilder::CreateStructTypeIRBuilder(block->getModule(),
value->getType());
auto struct_builder = StructTypeIRBuilder::CreateStructTypeIRBuilder(block->getModule(), value->getType());
if (!struct_builder.ok()) {
status.code = kCodegenError;
status.msg = struct_builder.status().ToString();
return false;

Check warning on line 297 in hybridse/src/codegen/block_ir_builder.cc

View check run for this annotation

Codecov / codecov/patch

hybridse/src/codegen/block_ir_builder.cc#L293-L297 

Added lines #L293 - L297 were not covered by tests
}
NativeValue ret_value;
if (!var_ir_builder.LoadRetStruct(&ret_value, status)) {
LOG(WARNING) << "fail to load ret struct address";
return false;
}
if (!struct_builder->CopyFrom(block, value,
ret_value.GetValue(&builder))) {
if (!struct_builder.value()->CopyFrom(block, value, ret_value.GetValue(&builder))) {

Check warning on line 304 in hybridse/src/codegen/block_ir_builder.cc

View check run for this annotation

Codecov / codecov/patch

hybridse/src/codegen/block_ir_builder.cc#L304 

Added line #L304 was not covered by tests
return false;
}
value = builder.getInt1(true);
Expand Down
2 changes: 1 addition & 1 deletion hybridse/src/codegen/buf_ir_builder.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -529,7 +529,7 @@ absl::StatusOr<llvm::Function*> BufNativeEncoderIRBuilder::GetOrBuildAppendMapFn
auto bs = ctx_->CreateBranchNot(is_null, [&]() -> base::Status {
auto row_ptr = BuildGetPtrOffset(sub_builder, i8_ptr, str_body_offset);
CHECK_TRUE(row_ptr.ok(), common::kCodegenError, row_ptr.status().ToString());
auto sz = map_builder.Encode(ctx_, map_ptr, row_ptr.value());
auto sz = map_builder.Encode(ctx_, row_ptr.value(), map_ptr);
CHECK_TRUE(sz.ok(), common::kCodegenError, sz.status().ToString());
sub_builder->CreateStore(sz.value(), encode_sz_alloca);
return {};
Expand Down
6 changes: 5 additions & 1 deletion hybridse/src/codegen/ir_base_builder_test.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -360,7 +360,11 @@ void ModuleFunctionBuilderWithFullInfo<Ret, Args...>::ExpandApplyArg(
if (TypeIRBuilder::IsStructPtr(expect_ty)) {
auto struct_builder =
StructTypeIRBuilder::CreateStructTypeIRBuilder(function->getEntryBlock().getModule(), expect_ty);
struct_builder->CreateDefault(&function->getEntryBlock(),
if (!struct_builder.ok()) {
LOG(WARNING) << struct_builder.status();
return;

Check warning on line 365 in hybridse/src/codegen/ir_base_builder_test.h

View check run for this annotation

Codecov / codecov/patch

hybridse/src/codegen/ir_base_builder_test.h#L364-L365 

Added lines #L364 - L365 were not covered by tests
}
struct_builder.value()->CreateDefault(&function->getEntryBlock(),
&alloca);
arg = builder.CreateSelect(
is_null, alloca, builder.CreatePointerCast(arg, expect_ty));
Expand Down
212 changes: 132 additions & 80 deletions hybridse/src/codegen/map_ir_builder.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ absl::StatusOr<NativeValue> MapIRBuilder::ExtractElement(CodeGenContextBase* ctx
ctx->GetBuilder()->getInt1Ty()->getPointerTo() // output is null ptr
},
false);
fn = llvm::Function::Create(fnt, llvm::Function::ExternalLinkage, fn_name, ctx->GetModule());
fn = llvm::Function::Create(fnt, llvm::GlobalValue::ExternalLinkage, fn_name, ctx->GetModule());

FunctionScopeGuard fg(fn, ctx);

Expand Down Expand Up @@ -362,6 +362,53 @@ absl::StatusOr<NativeValue> MapIRBuilder::MapKeys(CodeGenContextBase* ctx, const
return out;
}

absl::StatusOr<llvm::Function*> MapIRBuilder::BuildEncodeByteSizeFn(CodeGenContextBase* ctx) const {
std::string fn_name = absl::StrCat("calc_encode_map_sz_", GetIRTypeName(struct_type_));
llvm::Function* fn = ctx->GetModule()->getFunction(fn_name);
auto builder = ctx->GetBuilder();
if (fn == nullptr) {
llvm::FunctionType* fnt = llvm::FunctionType::get(builder->getInt32Ty(), // return size
{
struct_type_->getPointerTo(),
},
false);
fn = llvm::Function::Create(fnt, llvm::GlobalValue::ExternalLinkage, fn_name, ctx->GetModule());
FunctionScopeGuard fg(fn, ctx);

llvm::Value* raw = fn->arg_begin();
// map_size + [key_ele_sz * map_size] + [val_ele_sz * map_sz] + [sizeof(bool) * map_size]
llvm::Value* final_size = CodecSizeForPrimitive(builder, builder->getInt32Ty());

auto elements = Load(ctx, raw);
if (!elements.ok()) {
return elements.status();

Check warning on line 384 in hybridse/src/codegen/map_ir_builder.cc

View check run for this annotation

Codecov / codecov/patch

hybridse/src/codegen/map_ir_builder.cc#L384 

Added line #L384 was not covered by tests
}

if (elements->size() != FIELDS_CNT) {
return absl::FailedPreconditionError(
absl::Substitute("element count error, expect $0, got $1", FIELDS_CNT, elements->size()));

Check warning on line 389 in hybridse/src/codegen/map_ir_builder.cc

View check run for this annotation

Codecov / codecov/patch

hybridse/src/codegen/map_ir_builder.cc#L388-L389 

Added lines #L388 - L389 were not covered by tests
}
auto& elements_vec = elements.value();
auto& map_size = elements_vec[0];
auto& key_vec = elements_vec[1];
auto& value_vec = elements_vec[2];
auto& value_null_vec = elements_vec[3];

auto keys_sz = CalEncodeSizeForArray(ctx, key_vec, map_size);
CHECK_ABSL_STATUSOR(keys_sz);
auto values_sz = CalEncodeSizeForArray(ctx, value_vec, map_size);
CHECK_ABSL_STATUSOR(values_sz);
auto values_null_sz = CalEncodeSizeForArray(ctx, value_null_vec, map_size);
CHECK_ABSL_STATUSOR(values_null_sz);

builder->CreateRet(builder->CreateAdd(
final_size,
builder->CreateAdd(keys_sz.value(), builder->CreateAdd(values_sz.value(), values_null_sz.value()))));
}

return fn;
}

absl::StatusOr<llvm::Value*> MapIRBuilder::CalEncodeByteSize(CodeGenContextBase* ctx, llvm::Value* raw) const {
auto builder = ctx->GetBuilder();
if (!raw->getType()->isPointerTy() || raw->getType()->getPointerElementType() != struct_type_) {
Expand All @@ -370,33 +417,11 @@ absl::StatusOr<llvm::Value*> MapIRBuilder::CalEncodeByteSize(CodeGenContextBase*
GetLlvmObjectString(raw->getType())));

Check warning on line 417 in hybridse/src/codegen/map_ir_builder.cc

View check run for this annotation

Codecov / codecov/patch

hybridse/src/codegen/map_ir_builder.cc#L415-L417 

Added lines #L415 - L417 were not covered by tests
}

// map_size + [key_ele_sz * map_size] + [val_ele_sz * map_sz] + [sizeof(bool) * map_size]
llvm::Value* final_size = CodecSizeForPrimitive(builder, builder->getInt32Ty());
auto fns = BuildEncodeByteSizeFn(ctx);

auto elements = Load(ctx, raw);
if (!elements.ok()) {
return elements.status();
}
CHECK_ABSL_STATUSOR(fns);

if (elements->size() != FIELDS_CNT) {
return absl::FailedPreconditionError(
absl::Substitute("element count error, expect $0, got $1", FIELDS_CNT, elements->size()));
}
auto& elements_vec = elements.value();
auto& map_size = elements_vec[0];
auto& key_vec = elements_vec[1];
auto& value_vec = elements_vec[2];
auto& value_null_vec = elements_vec[3];

auto keys_sz = CalEncodeSizeForArray(ctx, key_vec, map_size);
CHECK_ABSL_STATUSOR(keys_sz);
auto values_sz = CalEncodeSizeForArray(ctx, value_vec, map_size);
CHECK_ABSL_STATUSOR(values_sz);
auto values_null_sz = CalEncodeSizeForArray(ctx, value_null_vec, map_size);
CHECK_ABSL_STATUSOR(values_null_sz);

return builder->CreateAdd(
final_size, builder->CreateAdd(keys_sz.value(), builder->CreateAdd(values_sz.value(), values_null_sz.value())));
return builder->CreateCall(fns.value(), {raw});
}

absl::StatusOr<llvm::Value*> MapIRBuilder::CalEncodeSizeForArray(CodeGenContextBase* ctx, llvm::Value* arr_ptr,
Expand Down Expand Up @@ -429,7 +454,7 @@ absl::StatusOr<llvm::Value*> MapIRBuilder::CalEncodeSizeForArray(CodeGenContextB
builder->getInt32Ty() // arr size
},
false);
fn = llvm::Function::Create(fnt, llvm::Function::ExternalLinkage, fn_name, ctx->GetModule());
fn = llvm::Function::Create(fnt, llvm::GlobalValue::ExternalLinkage, fn_name, ctx->GetModule());

FunctionScopeGuard fg(fn, ctx);
auto sub_builder = ctx->GetBuilder();
Expand Down Expand Up @@ -508,10 +533,82 @@ absl::StatusOr<llvm::Value*> MapIRBuilder::TypeEncodeByteSize(CodeGenContextBase
return absl::UnimplementedError(absl::StrCat("encode type ", GetLlvmObjectString(ele_type)));

Check warning on line 533 in hybridse/src/codegen/map_ir_builder.cc

View check run for this annotation

Codecov / codecov/patch

hybridse/src/codegen/map_ir_builder.cc#L533 

Added line #L533 was not covered by tests
}

absl::StatusOr<llvm::Value*> MapIRBuilder::Encode(CodeGenContextBase* ctx, llvm::Value* map_ptr,
llvm::Value* row_ptr) const {
absl::StatusOr<llvm::Function*> MapIRBuilder::BuildEncodeFn(CodeGenContextBase* ctx) const {
std::string fn_name = absl::StrCat("encode_map_", GetIRTypeName(struct_type_));
llvm::Function* fn = ctx->GetModule()->getFunction(fn_name);

auto builder = ctx->GetBuilder();
if (fn == nullptr) {
llvm::FunctionType* fnt = llvm::FunctionType::get(builder->getInt32Ty(), // encoded byte size
{
builder->getInt8PtrTy(), // row ptr
struct_type_->getPointerTo(), // map ptr
},
false);
fn = llvm::Function::Create(fnt, llvm::GlobalValue::ExternalLinkage, fn_name, ctx->GetModule());

FunctionScopeGuard fg(fn, ctx);

llvm::Value* row_ptr = fn->arg_begin();
llvm::Value* map_ptr = fn->arg_begin() + 1;
llvm::Value* written = builder->getInt32(0);

auto elements = Load(ctx, map_ptr);
if (!elements.ok()) {
return elements.status();

Check warning on line 558 in hybridse/src/codegen/map_ir_builder.cc

View check run for this annotation

Codecov / codecov/patch

hybridse/src/codegen/map_ir_builder.cc#L558 

Added line #L558 was not covered by tests
}

if (elements->size() != FIELDS_CNT) {
return absl::FailedPreconditionError(
absl::Substitute("element count error, expect $0, got $1", FIELDS_CNT, elements->size()));

Check warning on line 563 in hybridse/src/codegen/map_ir_builder.cc

View check run for this annotation

Codecov / codecov/patch

hybridse/src/codegen/map_ir_builder.cc#L562-L563 

Added lines #L562 - L563 were not covered by tests
}

auto& elements_vec = elements.value();
auto& map_size = elements_vec[0];
auto& key_vec = elements_vec[1];
auto& value_vec = elements_vec[2];
auto& value_null_vec = elements_vec[3];

// *(int32*) row_ptr = map_size
{
CHECK_ABSL_STATUS(BuildStoreOffset(builder, row_ptr, builder->getInt32(0), map_size));

written = builder->CreateAdd(written, builder->getInt32(4));
}
{
// *(key_type[map_size]) (row_ptr + 4) = key_vec
auto row_ptr_with_offset = BuildGetPtrOffset(builder, row_ptr, written);
CHECK_ABSL_STATUSOR(row_ptr_with_offset);
auto s = EncodeArray(ctx, row_ptr_with_offset.value(), key_vec, map_size);
CHECK_ABSL_STATUSOR(s);
written = builder->CreateAdd(written, s.value());
}
{
// *(value_type[map_size]) (row_ptr + ?) = value_vec
auto row_ptr_with_offset = BuildGetPtrOffset(builder, row_ptr, written);
CHECK_ABSL_STATUSOR(row_ptr_with_offset);
auto s = EncodeArray(ctx, row_ptr_with_offset.value(), value_vec, map_size);
CHECK_ABSL_STATUSOR(s);
written = builder->CreateAdd(written, s.value());
}
{
// *(bool[map_size]) (row_ptr + ?) = value_null_vec
auto row_ptr_with_offset = BuildGetPtrOffset(builder, row_ptr, written);
CHECK_ABSL_STATUSOR(row_ptr_with_offset);
// TODO(someone): alignment issue, bitwise operation for better performance ?
auto s = EncodeArray(ctx, row_ptr_with_offset.value(), value_null_vec, map_size);
CHECK_ABSL_STATUSOR(s);
written = builder->CreateAdd(written, s.value());
}

builder->CreateRet(written);
}
return fn;
}

absl::StatusOr<llvm::Value*> MapIRBuilder::Encode(CodeGenContextBase* ctx, llvm::Value* row_ptr,
llvm::Value* map_ptr) const {
auto builder = ctx->GetBuilder();
llvm::Value* written = builder->getInt32(0);

if (row_ptr->getType() != builder->getInt8Ty()->getPointerTo()) {
return absl::FailedPreconditionError(
Expand All @@ -525,55 +622,10 @@ absl::StatusOr<llvm::Value*> MapIRBuilder::Encode(CodeGenContextBase* ctx, llvm:
GetLlvmObjectString(map_ptr->getType()->getPointerElementType())));

Check warning on line 622 in hybridse/src/codegen/map_ir_builder.cc

View check run for this annotation

Codecov / codecov/patch

hybridse/src/codegen/map_ir_builder.cc#L620-L622 

Added lines #L620 - L622 were not covered by tests
}

auto elements = Load(ctx, map_ptr);
if (!elements.ok()) {
return elements.status();
}

if (elements->size() != FIELDS_CNT) {
return absl::FailedPreconditionError(
absl::Substitute("element count error, expect $0, got $1", FIELDS_CNT, elements->size()));
}

auto& elements_vec = elements.value();
auto& map_size = elements_vec[0];
auto& key_vec = elements_vec[1];
auto& value_vec = elements_vec[2];
auto& value_null_vec = elements_vec[3];

// *(int32*) row_ptr = map_size
{
CHECK_ABSL_STATUS(BuildStoreOffset(builder, row_ptr, builder->getInt32(0), map_size));

written = builder->CreateAdd(written, builder->getInt32(4));
}
{
// *(key_type[map_size]) (row_ptr + 4) = key_vec
auto row_ptr_with_offset = BuildGetPtrOffset(builder, row_ptr, written);
CHECK_ABSL_STATUSOR(row_ptr_with_offset);
auto s = EncodeArray(ctx, row_ptr_with_offset.value(), key_vec, map_size);
CHECK_ABSL_STATUSOR(s);
written = builder->CreateAdd(written, s.value());
}
{
// *(value_type[map_size]) (row_ptr + ?) = value_vec
auto row_ptr_with_offset = BuildGetPtrOffset(builder, row_ptr, written);
CHECK_ABSL_STATUSOR(row_ptr_with_offset);
auto s = EncodeArray(ctx, row_ptr_with_offset.value(), value_vec, map_size);
CHECK_ABSL_STATUSOR(s);
written = builder->CreateAdd(written, s.value());
}
{
// *(bool[map_size]) (row_ptr + ?) = value_null_vec
auto row_ptr_with_offset = BuildGetPtrOffset(builder, row_ptr, written);
CHECK_ABSL_STATUSOR(row_ptr_with_offset);
// TODO(someone): alignment issue, bitwise operation for better performance ?
auto s = EncodeArray(ctx, row_ptr_with_offset.value(), value_null_vec, map_size);
CHECK_ABSL_STATUSOR(s);
written = builder->CreateAdd(written, s.value());
}
auto fns = BuildEncodeFn(ctx);
CHECK_ABSL_STATUSOR(fns);

return written;
return builder->CreateCall(fns.value(), {row_ptr, map_ptr});
}

absl::StatusOr<llvm::Value*> MapIRBuilder::EncodeArray(CodeGenContextBase* ctx_, llvm::Value* row_ptr,
Expand Down Expand Up @@ -737,7 +789,7 @@ absl::StatusOr<llvm::Value*> MapIRBuilder::DecodeArrayValue(CodeGenContextBase*
},
false);

fn = llvm::Function::Create(fnt, llvm::Function::ExternalLinkage, fn_name, ctx->GetModule());
fn = llvm::Function::Create(fnt, llvm::GlobalValue::ExternalLinkage, fn_name, ctx->GetModule());

FunctionScopeGuard fg(fn, ctx);
auto* sub_builder = ctx->GetBuilder();
Expand Down Expand Up @@ -855,7 +907,7 @@ absl::StatusOr<llvm::Function*> MapIRBuilder::GetOrBuildEncodeArrFunction(CodeGe
builder->getInt32Ty(), {builder->getInt8Ty()->getPointerTo(), ele_type->getPointerTo(), builder->getInt32Ty()},
false);

fn = llvm::Function::Create(fnt, llvm::Function::ExternalLinkage, fn_name, ctx->GetModule());
fn = llvm::Function::Create(fnt, llvm::GlobalValue::ExternalLinkage, fn_name, ctx->GetModule());

// enter function
FunctionScopeGuard fg(fn, ctx);
Expand Down
8 changes: 5 additions & 3 deletions hybridse/src/codegen/map_ir_builder.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@
#ifndef HYBRIDSE_SRC_CODEGEN_MAP_IR_BUILDER_H_
#define HYBRIDSE_SRC_CODEGEN_MAP_IR_BUILDER_H_

#include <utility>

#include "codegen/struct_ir_builder.h"

namespace hybridse {
Expand All @@ -43,9 +41,13 @@ class MapIRBuilder final : public StructTypeIRBuilder {

absl::StatusOr<llvm::Value*> CalEncodeByteSize(CodeGenContextBase* ctx, llvm::Value*) const;

absl::StatusOr<llvm::Function*> BuildEncodeByteSizeFn(CodeGenContextBase* ctx) const;

// Encode the `map_ptr` into `row_ptr`, returns byte size written on success
// `row_ptr` is ensured to have enough space
absl::StatusOr<llvm::Value*> Encode(CodeGenContextBase*, llvm::Value* map_ptr, llvm::Value* row_ptr) const;
absl::StatusOr<llvm::Value*> Encode(CodeGenContextBase*, llvm::Value* row_ptr, llvm::Value* map_ptr) const;

absl::StatusOr<llvm::Function*> BuildEncodeFn(CodeGenContextBase*) const;

// Decode the stored map value at address row_ptr
absl::StatusOr<llvm::Value*> Decode(CodeGenContextBase*, llvm::Value* row_ptr) const;
Expand Down
27 changes: 15 additions & 12 deletions hybridse/src/codegen/struct_ir_builder.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,31 +31,34 @@ StructTypeIRBuilder::StructTypeIRBuilder(::llvm::Module* m)
StructTypeIRBuilder::~StructTypeIRBuilder() {}

bool StructTypeIRBuilder::StructCopyFrom(::llvm::BasicBlock* block, ::llvm::Value* src, ::llvm::Value* dist) {
StructTypeIRBuilder* struct_builder = CreateStructTypeIRBuilder(block->getModule(), src->getType());
bool ok = struct_builder->CopyFrom(block, src, dist);
delete struct_builder;
return ok;
auto struct_builder = CreateStructTypeIRBuilder(block->getModule(), src->getType());
if (struct_builder.ok()) {
return struct_builder.value()->CopyFrom(block, src, dist);

Check warning on line 36 in hybridse/src/codegen/struct_ir_builder.cc

View check run for this annotation

Codecov / codecov/patch

hybridse/src/codegen/struct_ir_builder.cc#L34-L36 

Added lines #L34 - L36 were not covered by tests
}
return false;

Check warning on line 38 in hybridse/src/codegen/struct_ir_builder.cc

View check run for this annotation

Codecov / codecov/patch

hybridse/src/codegen/struct_ir_builder.cc#L38 

Added line #L38 was not covered by tests
}

StructTypeIRBuilder* StructTypeIRBuilder::CreateStructTypeIRBuilder(::llvm::Module* m, ::llvm::Type* type) {
absl::StatusOr<std::unique_ptr<StructTypeIRBuilder>> StructTypeIRBuilder::CreateStructTypeIRBuilder(
::llvm::Module* m, ::llvm::Type* type) {
node::DataType base_type;
if (!GetBaseType(type, &base_type)) {
return nullptr;
return absl::UnimplementedError(
absl::StrCat("fail to create struct type ir builder for ", GetLlvmObjectString(type)));

Check warning on line 46 in hybridse/src/codegen/struct_ir_builder.cc

View check run for this annotation

Codecov / codecov/patch

hybridse/src/codegen/struct_ir_builder.cc#L45-L46 

Added lines #L45 - L46 were not covered by tests
}

switch (base_type) {
case node::kTimestamp:
return new TimestampIRBuilder(m);
return std::make_unique<TimestampIRBuilder>(m);
case node::kDate:
return new DateIRBuilder(m);
return std::make_unique<DateIRBuilder>(m);
case node::kVarchar:
return new StringIRBuilder(m);
return std::make_unique<StringIRBuilder>(m);
default: {
LOG(WARNING) << "fail to create struct type ir builder for " << DataTypeName(base_type);
return nullptr;
break;

Check warning on line 57 in hybridse/src/codegen/struct_ir_builder.cc

View check run for this annotation

Codecov / codecov/patch

hybridse/src/codegen/struct_ir_builder.cc#L57 

Added line #L57 was not covered by tests
}
}
return nullptr;
return absl::UnimplementedError(
absl::StrCat("fail to create struct type ir builder for ", GetLlvmObjectString(type)));

Check warning on line 61 in hybridse/src/codegen/struct_ir_builder.cc

View check run for this annotation

Codecov / codecov/patch

hybridse/src/codegen/struct_ir_builder.cc#L60-L61 

Added lines #L60 - L61 were not covered by tests
}

absl::StatusOr<NativeValue> StructTypeIRBuilder::CreateNull(::llvm::BasicBlock* block) {
Expand Down
Loading

0 comments on commit 7ce7503

Please sign in to comment.