Add Whitelist support (#395)

* Work on whitelist and implemented #394

* Move code, refactor whiteliests

* work on whitelist

* Refactor procedures, implement more whitelisting

* Upd join room with whitelist

* Add whitelist functions

* Test

* Fix loading task

* fix rename

* test null

* Upd changelog

CHANGELOG.md

@@ -1,7 +1,9 @@
 # v1.7.0
 
-* Fix incorrect value error when user or token is not found, pr #390.
+* Fix incorrect value error when user or token is not found, pr #390 (@rickmoonex).
 * Implement optional _name_ for a room, pr #393.
+* Use MessagePack Extension to internally store names, issue #394.
+* Added whitelist support for both _"procedures"_ and _"rooms"_, pr #395.
 
 # v1.6.6
 

CMakeLists.txt

@@ -226,6 +226,7 @@ set(SOURCES
     src/ti/warn.c
     src/ti/watch.c
     src/ti/web.c
+    src/ti/whitelist.c
     src/ti/wrap.c
     src/ti/write.c
     src/ti/ws.c

inc/doc.h

@@ -183,6 +183,8 @@
 #define DOC_SET_TIME_ZONE           DOC_SEE("thingsdb-api/set_time_zone")
 #define DOC_USER_INFO               DOC_SEE("thingsdb-api/user_info")
 #define DOC_USERS_INFO              DOC_SEE("thingsdb-api/users_info")
+#define DOC_WHITELIST_ADD           DOC_SEE("thingsdb-api/whitelist_add")
+#define DOC_WHITELIST_DEL           DOC_SEE("thingsdb-api/whitelist_del")
 
 /* Procedures API */
 #define DOC_DEL_PROCEDURE           DOC_SEE("procedures-api/del_procedure")

inc/ti/fn/fn.h

@@ -71,6 +71,7 @@
 #include <ti/vtask.h>
 #include <ti/vtask.inline.h>
 #include <ti/warn.h>
+#include <ti/whitelist.h>
 #include <ti/wprop.t.h>
 #include <ti/wrap.h>
 #include <ti/wrap.inline.h>

inc/ti/fn/fnnewprocedure.h

@@ -51,7 +51,7 @@ static int do__f_new_procedure(ti_query_t * query, cleri_node_t * nd, ex_t * e)
     {
         ex_set(e, EX_LOOKUP_ERROR,
                 "procedure `%s` already exists",
-                procedure->name);
+                procedure->name->str);
         goto fail2;
     }
 

inc/ti/fn/fnwhitelistadd.h

@@ -0,0 +1,54 @@
+#include <ti/fn/fn.h>
+
+static int do__f_whitelist_add(ti_query_t * query, cleri_node_t * nd, ex_t * e)
+{
+    const int nargs = fn_get_nargs(nd);
+    cleri_node_t * child = nd->children;
+    ti_user_t * user;
+    ti_task_t * task;
+    ti_raw_t * raw;
+    int wid;
+
+    if (fn_not_thingsdb_scope("whitelist_add", query, e) ||
+        ti_access_check_err(ti.access_thingsdb, query->user, TI_AUTH_GRANT, e) ||
+        fn_nargs_range("whitelist_add", DOC_WHITELIST_ADD, 2, 3, nargs, e) ||
+        ti_do_statement(query, child, e) ||
+        fn_arg_str_slow("whitelist_add", DOC_WHITELIST_ADD, 1, query->rval, e))
+        return e->nr;
+
+    raw = (ti_raw_t *) query->rval;
+    user = ti_users_get_by_namestrn((const char *) raw->data, raw->n);
+    if (!user)
+        return ti_raw_printable_not_found(raw, "user", e);
+
+    ti_val_unsafe_drop(query->rval);
+    query->rval = NULL;
+
+    /* read whitelist */
+    if (ti_do_statement(query, (child = child->next->next), e) ||
+        fn_arg_str_slow("whitelist_add", DOC_WHITELIST_ADD, 2, query->rval, e))
+        return e->nr;
+
+    raw = (ti_raw_t *) query->rval;
+    wid = ti_whitelist_from_strn((const char *) raw->data, raw->n, e);
+    if (e->nr)
+        return e->nr;
+
+    ti_val_unsafe_drop(query->rval);
+    query->rval = NULL;
+
+    if (nargs == 3 && ti_do_statement(query, (child = child->next->next), e))
+        return e->nr;
+
+    if (ti_whitelist_add(&user->whitelists[wid], query->rval, e))
+        return e->nr;
+
+    task = ti_task_get_task(query->change, ti.thing0);
+    if (!task || ti_task_add_whitelist_add(task, user, query->rval, wid))
+        ex_set_mem(e);  /* task cleanup is not required */
+
+    ti_val_drop(query->rval);
+    query->rval = (ti_val_t *) ti_nil_get();
+
+    return e->nr;
+}

inc/ti/fn/fnwhitelistdel.h

@@ -0,0 +1,54 @@
+#include <ti/fn/fn.h>
+
+static int do__f_whitelist_del(ti_query_t * query, cleri_node_t * nd, ex_t * e)
+{
+    const int nargs = fn_get_nargs(nd);
+    cleri_node_t * child = nd->children;
+    ti_user_t * user;
+    ti_task_t * task;
+    ti_raw_t * raw;
+    int wid;
+
+    if (fn_not_thingsdb_scope("whitelist_del", query, e) ||
+        ti_access_check_err(ti.access_thingsdb, query->user, TI_AUTH_GRANT, e) ||
+        fn_nargs_range("whitelist_del", DOC_WHITELIST_DEL, 2, 3, nargs, e) ||
+        ti_do_statement(query, child, e) ||
+        fn_arg_str_slow("whitelist_del", DOC_WHITELIST_DEL, 1, query->rval, e))
+        return e->nr;
+
+    raw = (ti_raw_t *) query->rval;
+    user = ti_users_get_by_namestrn((const char *) raw->data, raw->n);
+    if (!user)
+        return ti_raw_printable_not_found(raw, "user", e);
+
+    ti_val_unsafe_drop(query->rval);
+    query->rval = NULL;
+
+    /* read whitelist */
+    if (ti_do_statement(query, (child = child->next->next), e) ||
+        fn_arg_str_slow("whitelist_del", DOC_WHITELIST_DEL, 2, query->rval, e))
+        return e->nr;
+
+    raw = (ti_raw_t *) query->rval;
+    wid = ti_whitelist_from_strn((const char *) raw->data, raw->n, e);
+    if (e->nr)
+        return e->nr;
+
+    ti_val_unsafe_drop(query->rval);
+    query->rval = NULL;
+
+    if (nargs == 3 && ti_do_statement(query, (child = child->next->next), e))
+        return e->nr;
+
+    if (ti_whitelist_del(&user->whitelists[wid], query->rval, e))
+        return e->nr;
+
+    task = ti_task_get_task(query->change, ti.thing0);
+    if (!task || ti_task_add_whitelist_del(task, user, query->rval, wid))
+        ex_set_mem(e);  /* task cleanup is not required */
+
+    ti_val_drop(query->rval);
+    query->rval = (ti_val_t *) ti_nil_get();
+
+    return e->nr;
+}

inc/ti/name.h

@@ -6,6 +6,7 @@
 
 #include <stdint.h>
 #include <ti/name.t.h>
+#include <util/mpack.h>
 
 ti_name_t * ti_name_create(const char * str, size_t n);
 void ti_name_destroy(ti_name_t * name);
@@ -23,4 +24,10 @@ static inline void ti_name_unsafe_drop(ti_name_t * name)
         ti_name_destroy(name);
 }
 
+/* added for issue #394 */
+static inline int ti_name_to_pk(ti_name_t * name, msgpack_packer * pk)
+{
+    return mp_pack_ext(pk, MPACK_EXT_NAME, name->str, name->n);
+}
+
 #endif /* TI_NAME_H_ */

inc/ti/procedure.h

@@ -17,7 +17,7 @@ typedef struct ti_procedure_s ti_procedure_t;
 #include <ti/val.h>
 
 ti_procedure_t * ti_procedure_create(
-        const char * name,
+        const char * str,
         size_t name_n,
         ti_closure_t * closure,
         uint64_t created_at);
@@ -37,8 +37,7 @@ ti_val_t * ti_procedure_as_mpval(
 struct ti_procedure_s
 {
     uint64_t created_at;        /* UNIX time-stamp in seconds */
-    char * name;                /* NULL terminated name */
-    size_t name_n;              /* size of the name */
+    ti_name_t * name;
     ti_raw_t * doc;             /* documentation, may be NULL */
     ti_raw_t * def;             /* formatted definition, may be NULL */
     ti_closure_t * closure;     /* closure */

inc/ti/procedures.h

@@ -16,7 +16,7 @@ int ti_procedures_to_pk(smap_t * procedures, msgpack_packer * pk);
 int ti_procedures_rename(
         smap_t * procedures,
         ti_procedure_t * procedure,
-        const char * name,
+        const char * str,
         size_t n);
 
 static inline void ti_procedures_clear(smap_t * procedures)
@@ -28,7 +28,7 @@ static inline ti_procedure_t * ti_procedures_pop(
         smap_t * procedures,
         ti_procedure_t * procedure)
 {
-    return smap_pop(procedures, procedure->name);
+    return smap_pop(procedures, procedure->name->str);
 }
 
 

inc/ti/task.h

@@ -178,6 +178,15 @@ int ti_task_add_mod_enum_ren(
         ti_name_t * oldname,
         ti_name_t * newname);
 int ti_task_add_del_enum(ti_task_t * task, ti_enum_t * enum_);
-
+int ti_task_add_whitelist_add(
+        ti_task_t * task,
+        ti_user_t * user,
+        ti_val_t * val,
+        int wid);
+int ti_task_add_whitelist_del(
+        ti_task_t * task,
+        ti_user_t * user,
+        ti_val_t * val,
+        int wid);
 
 #endif /* TI_TASK_H_ */

inc/ti/task.t.h

@@ -90,6 +90,8 @@ typedef enum
     TI_TASK_REPLACE_ROOT,                   /* 77  */
     TI_TASK_IMPORT,                         /* 78  */
     TI_TASK_ROOM_SET_NAME,                  /* 79  */
+    TI_TASK_WHITELIST_ADD,                  /* 80  */
+    TI_TASK_WHITELIST_DEL,                  /* 81  */
 } ti_task_enum;
 
 typedef struct ti_task_s ti_task_t;

inc/ti/user.t.h

@@ -25,6 +25,10 @@ struct ti_user_s
     ti_raw_t * name;
     char * encpass;         /* may be NULL if no password is set */
     vec_t * tokens;         /* ti_token_t */
+    vec_t * whitelists[2];  /* For the vectors:
+                             *   TI_WHITELIST_ROOMS
+                             *   TI_WHITELIST_PROCEDURES
+                             * Both may be NULL, contains ti_val_t */
 };
 
 #endif  /* TI_USER_T_H_ */

inc/ti/val.inline.h

@@ -317,7 +317,7 @@ static ti_val_type_t ti_val_type_props[22] = {
         .to_str = val__str_to_str,
         .to_arr_cb = val__to_arr_cb,
         .to_client_pk = val__str_to_client_pk,
-        .to_store_pk = (ti_val_to_store_pk_cb) ti_raw_str_to_pk,
+        .to_store_pk = (ti_val_to_store_pk_cb) ti_name_to_pk,
         .get_type_str = val__str_type_str,
         .allowed_as_vtask_arg = true,
     },

inc/ti/version.h

@@ -25,7 +25,7 @@
  *  "-rc0"
  *  ""
  */
-#define TI_VERSION_PRE_RELEASE "-alpha5"
+#define TI_VERSION_PRE_RELEASE "-rc0"
 
 #define TI_MAINTAINER \
     "Jeroen van der Heijden <[email protected]>"

inc/ti/whitelist.h

@@ -0,0 +1,26 @@
+/*
+ * ti/whitelist.h
+ */
+#ifndef TI_WHITELIST_H_
+#define TI_WHITELIST_H_
+
+#include <stdlib.h>
+#include <inttypes.h>
+#include <ti/name.t.h>
+#include <ti/user.t.h>
+#include <ex.h>
+
+/* Do NOT CHANGE the order as this is used and stored in tasks */
+enum
+{
+    TI_WHITELIST_ROOMS,             /* 0 */
+    TI_WHITELIST_PROCEDURES,        /* 1 */
+};
+
+int ti_whitelist_add(vec_t ** whitelist, ti_val_t * val, ex_t * e);
+int ti_whitelist_del(vec_t ** whitelist, ti_val_t * val, ex_t * e);
+int ti_whitelist_from_strn(const char * str, size_t n, ex_t * e);
+int ti_whitelist_test(vec_t * whitelist, ti_name_t * name, ex_t * e);
+int ti_whitelist_check(vec_t * whitelist, ti_name_t * name);
+
+#endif  /* TI_WHITELIST_H_ */

inc/util/mpack.h

@@ -23,6 +23,7 @@ typedef enum
     MPACK_EXT_ROOM,
     MPACK_EXT_TASK,
     MPACK_EXT_THING,
+    MPACK_EXT_NAME,
 } mpack_ext_t;
 
 typedef struct

itest/run_all_tests.py

@@ -37,6 +37,7 @@
 from test_types import TestTypes
 from test_user_access import TestUserAccess
 from test_variable import TestVariable
+from test_whitelist import TestWhitelist
 from test_wrap import TestWrap
 from test_ws import TestWS
 from test_wss import TestWSS
@@ -98,6 +99,7 @@ def no_mem_test(test_class):
     run_test(TestTypes())
     run_test(TestUserAccess())
     run_test(TestVariable())
+    run_test(TestWhitelist())
     run_test(TestWS())
     run_test(TestWSS())
     run_test(TestWrap())

itest/test_advanced.py

@@ -2613,6 +2613,34 @@ async def test_json_load_i(self, client):
             is_tuple(x.first());
         """))
 
+    async def test_names_loading(self, client):
+        await self.node0.shutdown()
+        await self.node0.run()
+
+        q = client.query
+        # Force a few new keys. (issue #394)
+        # Before 1.7.0 ThingsDB did not reload existing names as names, but
+        # they were loaded as strings instead. This is not an issue but it is
+        # preferred to load them back as names. This checks verifies that
+        # names are really restored as cached names, not strings.
+        await q("""//ti
+            .my_test_key1 = 1;
+            .my_test_key2 = 2;
+            .arr = .keys();
+            .del('my_test_key1');
+            .del('my_test_key2');
+        """)
+        n1 = await q('node_info().load().cached_names;', scope='/n/0')
+
+        await self.node0.shutdown()
+        await self.node0.run()
+        n2 = await q('node_info().load().cached_names;', scope='/n/0')
+
+        self.assertEqual(n1, n2)
+        res = await q('.arr;')
+        self.assertEqual(res[0], 'my_test_key1')
+        self.assertEqual(res[1], 'my_test_key2')
+
 
 if __name__ == '__main__':
     run_test(TestAdvanced())

itest/test_migrate.py

@@ -14,7 +14,7 @@
 
 class TestMigrate(TestBase):
 
-    title = 'Test simple run'
+    title = 'Test migration'
 
     @default_test_setup(num_nodes=3, seed=1, threshold_full_storage=10)
     async def run(self):

itest/test_user_access.py

@@ -117,7 +117,8 @@ async def run(self):
             'created_at': created_at,
             'name': 'admin',
             'tokens': [],
-            'user_id': 1
+            'user_id': 1,
+            'whitelists': {}
         })
 
         with self.assertRaisesRegex(ForbiddenError, error_msg):