Added faster alternative to InVirtualSet() (backport #874) [release/4…

….11.x] (#987)

* Added faster alternative to InVirtualSet() (#874)

* commit for new added test

* commit 2

* update

* Fixed all the issues related to filtering of data in Virtual Table

* Binder Type enhancement

* trying to fix BestIndex

* Fixed the single column issue in VT

* Performance Tests Added

* ValueExp_Issue_Id_Change due to conflicts while merging with main

* Added flag for binder info to check whether the binder is for a parameter inside InVirtualSet() or IdSet()....useful for BindIdSet()

* Tests Added

* Crash for following expression stopped by providing null checks

SELECT x FROM  (with cte(x) as(select ECInstanceId from meta.ECClassDef) select x from cte), test.IdSet('[1,2,3,4,5]') where id = x group by x

* Kepping concurrentQueryImpl as close to as it was with minimal changes

* schema name changed to ECVLib and also file name changed

* cleanup

* more cleanup

* Performanvce Tests Updated

* some comments resolved

* Comments regarding constant name of IdSet table resolved

* binderInfo refactoring

* added flag to call _onbeforefirststep() once in PragmaECSQLStatement and renamed _OnBeforeStep() to _ONBeforeFirstStep()

* changes as per suggestions by Affan

* Performance test updated

* Tests updated to prevent failure in pipeline

* tests updated

* update in logic in IModelJsNative.cpp and concurrentquery

* performance tests indentation updated

* final update

* OnBeforeFirstStep() logic updated by using m_isFirstStep flag

* More Tests added

* Added flag checking to m_isFirstStep flag so that when actually flag is false we reset it to true and vice versa

* removing m_isFirstStep and identifying first step using statement state

* Comment updated

* Fixed the issue with the query SELECT e.i FROM aps.TestElement e INNER JOIN ECVLib.IdSet(?) v ON e.ECInstanceId = v.id

* More tests added

* More Performance Tests added

* indentation issue solved

* Update in logic to get statement state

* Changelog updated

* comments updated

* Using EqualsI instead of EqualsIAscii

* Comments resolved and tests added

* Made IdSet experimental feature

---------

Co-authored-by: affank <[email protected]>
(cherry picked from commit 001246a)

# Conflicts:
#	iModelJsNodeAddon/api_package/ts/package-lock.json

* manually update package-lock.json

---------

Co-authored-by: Soham Bhattacharjee <[email protected]>

iModelCore/BeSQLite/BeSQLite.cpp

@@ -42,6 +42,7 @@ USING_NAMESPACE_BENTLEY_SQLITE
 extern "C" int checkNoActiveStatements(SqlDbP db);
 #endif
 
+extern "C" int getStatementState(SqlStatementP pStmt);
 extern "C" int sqlite3_shathree_init(sqlite3 *, char **, const sqlite3_api_routines *);
 
 BEGIN_BENTLEY_SQLITE_NAMESPACE
@@ -919,6 +920,17 @@ void Statement::DumpResults()
 
     Reset();
     }
+
+/*---------------------------------------------------------------------------------------
+* @bsimethod
++---------------+---------------+---------------+---------------+---------------+------*/
+bool Statement::TryGetStatementState(StatementState& state)
+    {
+    if(!IsPrepared())
+        return false;
+    state = (StatementState)getStatementState(m_stmt);
+    return true;
+    }
 
 /*---------------------------------------------------------------------------------**//**
 * @bsimethod

iModelCore/BeSQLite/PublicAPI/BeSQLite/BeSQLite.h

@@ -434,6 +434,18 @@ enum class DbDeserializeOptions {
 };
 ENUM_IS_FLAGS(DbDeserializeOptions)
 
+//=======================================================================================
+// @bsiclass
+//=======================================================================================
+enum class StatementState
+{
+    // The first four values are in accordance with sqlite3.c
+    Init          = 0,   //!< Prepared statement under construction
+    Ready         = 1,   //!< Ready to run but not yet started
+    Run           = 2,   //!< Run in progress
+    Halt          = 3,   //!< Finished.  Need reset() or finalize()
+};
+
 //=======================================================================================
 // @bsiclass
 //=======================================================================================
@@ -985,6 +997,10 @@ struct Statement : NonCopyableClass
     //! Dump query results to stdout, for debugging purposes
     BE_SQLITE_EXPORT void DumpResults();
 
+    //! Tries to get the state in which a particular statement is. Returns true if it is successful in getting the state of the statement otherwise returns false
+    //! If the returned value is true, the state value is stored in the passed reference argument.
+    BE_SQLITE_EXPORT bool TryGetStatementState(StatementState&);
+
     SqlStatementP GetSqlStatementP() const {return m_stmt;}  // for direct use of sqlite3 api
     operator SqlStatementP(){return m_stmt;}                 // for direct use of sqlite3 api
 };

iModelCore/BeSQLite/SQLite/bentley-sqlite.c

@@ -99,3 +99,12 @@ int checkNoActiveStatements(sqlite3* db)
     return SQLITE_ERROR;
     }
 #endif
+
+/*---------------------------------------------------------------------------------**//**
+* @bsimethod
++---------------+---------------+---------------+---------------+---------------+------*/
+int getStatementState(sqlite3_stmt *pStmt)
+    {
+    Vdbe* stmt = (Vdbe*)pStmt;
+    return stmt->eVdbeState;
+    }

iModelCore/ECDb/CHANGES.md

@@ -5,7 +5,18 @@ This document including important changes to syntax or file format.
 | Module  | Version   |
 | ------- | --------- |
 | Profile | `4.0.0.5` |
-| ECSQL   | `2.0.0.0` |
+| ECSQL   | `2.0.1.1` |
+
+## ## `01/22/2025`: Added IdSet Virtual Table in ECSQL
+* ECSql version change `2.0.1.0` -> `2.0.1.1`.
+* Added IdSet Virtual Table in ECSQL
+* Example: `Select test.str_prop, test.int_prop, v.id from ts.A test RIGHT OUTER JOIN ECVLib.IdSet(:idSet_param) v on test.ECInstanceId = v.id`.
+
+## ## `01/10/2025`: Added support for CTE subquery with alias
+* ECSql version change `2.0.0.0` -> `2.0.1.0`.
+* Added support for CTE subquery with alias
+* Example: `select a.x from (with tmp(x) as (SELECT e.i FROM aps.TestElement e order by e.i LIMIT 1) select x from tmp) a`.
+
 
 ## ## `09/05/2024`: Remove class names ALIAS support and Disqualify_polymorphic_constraint(+) support in UPDATE & DELETE statements
 * ECSql version change `1.2.14.0` -> `2.0.0.0`.

iModelCore/ECDb/ECDb/BuiltInVTabs.cpp

@@ -151,8 +151,184 @@ DbResult ClassPropsModule::Connect(DbVirtualTable*& out, Config& conf, int argc,
     return BE_SQLITE_OK;
 }
 
+/*IdSet Virtual Table*/
+
+//---------------------------------------------------------------------------------------
+// @bsimethod
+//---------------------------------------------------------------------------------------
+DbResult IdSetModule::IdSetTable::IdSetCursor::Next() {
+    ++m_index;
+    return BE_SQLITE_OK;
+}
+
+//---------------------------------------------------------------------------------------
+// @bsimethod
+//---------------------------------------------------------------------------------------
+DbResult IdSetModule::IdSetTable::IdSetCursor::GetRowId(int64_t& rowId) {
+    rowId = (int64_t)(*m_index);
+    return BE_SQLITE_OK;
+}
+
+//---------------------------------------------------------------------------------------
+// @bsimethod
+//---------------------------------------------------------------------------------------
+DbResult IdSetModule::IdSetTable::IdSetCursor::GetColumn(int i, Context& ctx) {
+    if ((Columns)i == Columns::Json_array_ids) {
+        ctx.SetResultText(m_text.c_str(), (int)m_text.size(), Context::CopyData::No);
+    } else if ((Columns)i == Columns::Id && m_index != m_idSet.end()) {
+        ctx.SetResultInt64((int64_t)(*m_index));
+    }
+    return BE_SQLITE_OK;
+}
+
+//---------------------------------------------------------------------------------------
+// @bsimethod
+//---------------------------------------------------------------------------------------
+DbResult IdSetModule::IdSetTable::IdSetCursor::FilterJSONStringIntoArray(BeJsDocument& doc) {
+    if(!doc.isArray()) {
+        GetTable().SetError("IdSet vtab: The argument should be a valid JSON array of ids");
+        return BE_SQLITE_ERROR;
+    }
+    bool flag = doc.ForEachArrayMember([&](BeJsValue::ArrayIndex a, BeJsConst k1) {
+                                            if(BE_SQLITE_OK != FilterJSONBasedOnType(k1)) {
+                                                GetTable().SetError(SqlPrintfString("IdSet vtab: The element with index %u is invalid", a));
+                                                return true;
+                                            }
+                                            return false; 
+                                        });
+    if(flag)
+        return BE_SQLITE_ERROR;
+    return BE_SQLITE_OK;
+}
+
+
+//---------------------------------------------------------------------------------------
+// @bsimethod
+//---------------------------------------------------------------------------------------
+DbResult IdSetModule::IdSetTable::IdSetCursor::FilterJSONBasedOnType(BeJsConst& val) {
+    if(val.isNull()) {
+        return BE_SQLITE_ERROR;
+    }
+    else if(val.isNumeric()) {
+        uint64_t id = val.GetUInt64();
+        if(id == 0)
+            return BE_SQLITE_ERROR;
+        m_idSet.insert(id);
+    }
+    else if(val.isString()) {
+        uint64_t id;
+        BentleyStatus status = BeStringUtilities::ParseUInt64(id, val.ToUtf8CP());
+        if(status != BentleyStatus::SUCCESS)
+            return BE_SQLITE_ERROR;
+        if(id == 0)
+            return BE_SQLITE_ERROR;
+        m_idSet.insert(id);
+    }
+    else
+        return BE_SQLITE_ERROR;
+    return BE_SQLITE_OK;
+}
+
+//---------------------------------------------------------------------------------------
+// @bsimethod
+//---------------------------------------------------------------------------------------
+DbResult IdSetModule::IdSetTable::IdSetCursor::Filter(int idxNum, const char *idxStr, int argc, DbValue* argv) {
+    int recompute = false;
+    if( idxNum & 1 ){
+        if(argv[0].GetValueType() == DbValueType::TextVal) {
+            Utf8String valueGiven = argv[0].GetValueText();
+            if(!valueGiven.EqualsIAscii(m_text)) {
+                m_text = valueGiven;
+                recompute = true;
+            }
+        } else {
+            Reset();
+        }
+    } else {
+        Reset();
+    }
+    if(recompute) {
+        m_idSet.clear();
+        BeJsDocument doc;
+        doc.Parse(m_text.c_str());
+
+        if(FilterJSONStringIntoArray(doc) != BE_SQLITE_OK) {
+            Reset();
+            m_index = m_idSet.begin();
+            return BE_SQLITE_ERROR;
+        }
+    }
+    m_index = m_idSet.begin();
+    return BE_SQLITE_OK;
+}
+
+//---------------------------------------------------------------------------------------
+// @bsimethod
+//---------------------------------------------------------------------------------------
+void IdSetModule::IdSetTable::IdSetCursor::Reset() {
+    m_text = "[]";
+    m_idSet.clear();
+}
+//---------------------------------------------------------------------------------------
+// @bsimethod
+//---------------------------------------------------------------------------------------
+DbResult IdSetModule::IdSetTable::BestIndex(IndexInfo& indexInfo) {
+    int i, j;              /* Loop over constraints */
+    int idxNum = 0;        /* The query plan bitmask */
+    int unusableMask = 0;  /* Mask of unusable constraints */
+    int nArg = 0;          /* Number of arguments that seriesFilter() expects */
+    int aIdx[2];           /* Constraints on start, stop, and step */
+    const int SQLITE_SERIES_CONSTRAINT_VERIFY = 0;
+    aIdx[0] = aIdx[1] = -1;
+    int nConstraint = indexInfo.GetConstraintCount();
+
+    for(i=0; i<nConstraint; i++) {
+        auto pConstraint = indexInfo.GetConstraint(i);
+        int iCol;    /* 0 for start, 1 for stop, 2 for step */
+        int iMask;   /* bitmask for those column */
+        if( pConstraint->GetColumn()< (int)IdSetCursor::Columns::Json_array_ids) continue;
+        iCol = pConstraint->GetColumn() - (int)IdSetCursor::Columns::Json_array_ids;
+        iMask = 1 << iCol;
+        if (!pConstraint->IsUsable()){
+            unusableMask |=  iMask;
+            continue;
+        } else if (pConstraint->GetOp() == IndexInfo::Operator::EQ ){
+            idxNum |= iMask;
+            aIdx[iCol] = i;
+        }
+    }
+    for( i = 0; i < 1; i++) {
+        if( (j = aIdx[i]) >= 0 ) {
+            indexInfo.GetConstraintUsage(j)->SetArgvIndex(++nArg);
+            indexInfo.GetConstraintUsage(j)->SetOmit(!SQLITE_SERIES_CONSTRAINT_VERIFY);
+        }
+    }
+
+    if ((unusableMask & ~idxNum)!=0 ) {
+        return BE_SQLITE_CONSTRAINT;
+    }
+
+    indexInfo.SetEstimatedCost(0);
+    indexInfo.SetEstimatedRows(100);
+    indexInfo.SetIdxNum(idxNum);
+    return BE_SQLITE_OK;
+
+}
+
+//---------------------------------------------------------------------------------------
+// @bsimethod
+//---------------------------------------------------------------------------------------
+DbResult IdSetModule::Connect(DbVirtualTable*& out, Config& conf, int argc, const char* const* argv) {
+    out = new IdSetTable(*this);
+    conf.SetTag(Config::Tags::Innocuous);
+    return BE_SQLITE_OK;
+}
+
 DbResult RegisterBuildInVTabs(ECDbR ecdb) {
-    auto rc = (new ClassPropsModule(ecdb))->Register();
-    return rc;
+    DbResult rc = (new ClassPropsModule(ecdb))->Register();
+    DbResult rcIdSet = (new IdSetModule(ecdb))->Register();
+    if(rc != BE_SQLITE_OK || rcIdSet != BE_SQLITE_OK)
+        return rc;
+    return BE_SQLITE_OK;
 }
 END_BENTLEY_SQLITE_EC_NAMESPACE

iModelCore/ECDb/ECDb/BuiltInVTabs.h

@@ -43,6 +43,67 @@ struct ClassPropsModule : BeSQLite::DbModule {
         DbResult Connect(DbVirtualTable*& out, Config& conf, int argc, const char* const* argv) final;
 };
 
+struct IdSetModule : ECDbModule {
+    constexpr static auto NAME = "IdSet";
+    struct IdSetTable : ECDbVirtualTable {
+        struct IdSetCursor : ECDbCursor {
+
+            enum class Columns{
+                Id = 0,
+                Json_array_ids = 1,
+            };
+
+            private:
+                Utf8String m_text;
+                bset<uint64_t> m_idSet;
+                bset<uint64_t>::const_iterator m_index;
+
+            public:
+                IdSetCursor(IdSetTable& vt): ECDbCursor(vt), m_index(m_idSet.begin()){}
+                bool Eof() final { return m_index == m_idSet.end() ; }
+                DbResult Next() final;
+                DbResult GetColumn(int i, Context& ctx) final;
+                DbResult GetRowId(int64_t& rowId) final;
+                DbResult Filter(int idxNum, const char *idxStr, int argc, DbValue* argv) final;
+                DbResult FilterJSONBasedOnType(BeJsConst& val);
+                DbResult FilterJSONStringIntoArray(BeJsDocument& val);
+                void Reset();
+        };
+        public:
+            IdSetTable(IdSetModule& module): ECDbVirtualTable(module) {}
+            DbResult Open(DbCursor*& cur) override {
+                cur = new IdSetCursor(*this);
+                return BE_SQLITE_OK;
+            }
+             DbResult BestIndex(IndexInfo& indexInfo) final;
+    };
+    public:
+        IdSetModule(ECDbR db): ECDbModule(
+            db,
+            NAME,
+            "CREATE TABLE x(id, json_array_ids hidden)",
+            R"xml(<?xml version="1.0" encoding="utf-8" ?>
+            <ECSchema
+                    schemaName="ECVLib"
+                    alias="ECVLib"
+                    version="1.0.0"
+                    xmlns="http://www.bentley.com/schemas/Bentley.ECXML.3.2">
+                <ECSchemaReference name="ECDbVirtual" version="01.00.00" alias="ecdbvir" />
+                <ECCustomAttributes>
+                    <VirtualSchema xmlns="ECDbVirtual.01.00.00"/>
+                </ECCustomAttributes>
+                <ECEntityClass typeName="IdSet" modifier="Abstract">
+                    <ECCustomAttributes>
+                        <VirtualType xmlns="ECDbVirtual.01.00.00"/>
+                    </ECCustomAttributes>
+                    <ECProperty propertyName="id"  typeName="long" extendedTypeName="Id"/>
+                </ECEntityClass>
+            </ECSchema>)xml"
+            ) {}
+        DbResult Connect(DbVirtualTable*& out, Config& conf, int argc, const char* const* argv) final;
+};
+
+
 DbResult RegisterBuildInVTabs(ECDbR);
 
 END_BENTLEY_SQLITE_EC_NAMESPACE

iModelCore/ECDb/ECDb/ConcurrentQueryManagerImpl.cpp

@@ -1905,8 +1905,34 @@ bool ECSqlParams::TryBindTo(ECSqlStatement& stmt, std::string& err) const {
             case ECSqlParam::Type::Id:
                 st = stmt.BindId(index, param.GetValueId());  break;
             case ECSqlParam::Type::IdSet: {
-                std::shared_ptr<IdSet<BeInt64Id>> idSet = std::make_shared<IdSet<BeInt64Id>>(param.GetValueIdSet());
-                st = stmt.BindVirtualSet(index, idSet);
+                BinderInfo const& binderInfo = stmt.GetBinderInfo(index);
+                if(binderInfo.GetType() == BinderInfo::BinderType::VirtualSet) {
+                    std::shared_ptr<IdSet<BeInt64Id>> idSet = std::make_shared<IdSet<BeInt64Id>>(param.GetValueIdSet());
+                    st = stmt.BindVirtualSet(index, idSet);
+                }
+                else if(binderInfo.GetType() == BinderInfo::BinderType::Array && binderInfo.IsForIdSet()) {
+                    bool allElementsAdded = true;
+                    IECSqlBinder& binder = stmt.GetBinder(index);
+                    IdSet<BeInt64Id> set(param.GetValueIdSet());
+                    for(auto& ids: set) {
+                        if(!ids.IsValid()) {
+                            allElementsAdded = false;
+                            break;
+                        }
+                        st = binder.AddArrayElement().BindInt64((int64_t) ids.GetValue());
+                        if(!st.IsSuccess()) {
+                            allElementsAdded = false;
+                            break;
+                        }
+                    }
+                    if(allElementsAdded) // If even one array element has failed to be added we set the status for the entire operation as ECSqlStatus::Error although for the time being we don't do anything with status even if it fails
+                        st = ECSqlStatus::Success;
+                    else
+                        st = ECSqlStatus::Error;
+                }
+                else
+                    st = ECSqlStatus::Error;
+
                 break;
             }
             case ECSqlParam::Type::Integer: