feat: Add SELECT * EXCEPT capability

resources/inferenceql/query/base.bnf

@@ -76,11 +76,13 @@ distinct-clause ::= #'(?i)DISTINCT'
 
 select-clause ::= #'(?i)SELECT' (ws distinct-clause)? ws select-list
 
-select-list ::= star
+select-list ::= select-star-clause
               / selection (ws? ',' ws? selection)*
               / aggregation (ws? ',' ws? aggregation)*
 
+select-star-clause ::= star (ws? select-except-clause)?
 star ::= '*'
+select-except-clause ::= #'(?i)EXCEPT' ws? '(' ws? identifier-list ws? ')'
 
 selection ::= (scalar-expr | aggregation) (ws alias-clause)?
 
@@ -149,7 +151,6 @@ insert-expr ::= #'(?i)INSERT' ws #'(?i)INTO' ws relation-expr ws relation-expr
 
 relation-value ::= '(' ws? identifier-list ws? ')' ws #'(?i)VALUES' ws value-lists
 
-identifier-list ::= identifier (ws? ',' ws identifier)*
 value-list ::= '(' ws? value (ws? ',' ws? value)* ws? ')'
 value-lists ::= value-lists-full | value-lists-sparse
 value-lists-full ::= value-list (ws? ',' ws? value-list)*
@@ -241,6 +242,7 @@ bool             ::= #'true|false'
 float            ::= #'-?\d+\.\d+(E-?\d+)?'
 int              ::= #'-?\d+'
 nat              ::= #'\d+'
+identifier-list  ::= identifier (ws? ',' ws? identifier)*
 identifier       ::= simple-symbol | delimited-symbol
 simple-symbol    ::= #'(?u)(?!G__)\p{L}[\p{L}\p{N}_\-\?\.]*'
 delimited-symbol ::= <'"'> #'[^\"]+' <'"'>

src/inferenceql/query/base.cljc

@@ -22,14 +22,14 @@
   - parse - a parsing function"
   [s db parse]
   (let [node-or-failure (parse s)]
+    (tap> #:base.query{:node node-or-failure})
     (cond (insta/failure? node-or-failure)
           (throw (error/parse node-or-failure))
 
           (plan/relation-node? node-or-failure)
           (let [plan (plan/plan node-or-failure)
                 env (db/env @db)]
-            (tap> #:base.query{:node node-or-failure
-                               :plan plan
+            (tap> #:base.query{:plan plan
                                :env env})
             (plan/eval plan env {}))
 

src/inferenceql/query/parser/tree.cljc

@@ -40,7 +40,7 @@
            (string/blank? node))))
 
 (defn children
-  "Returns `node`'s children."
+  "Returns `node`'s children. Removes whitespace nodes."
   [node]
   (into []
         (clojure/remove whitespace?)

src/inferenceql/query/plan.cljc

@@ -49,25 +49,32 @@
   [node]
   (tree/match [node]
     [[:selection child]] (star? child)
+    [[:select-star-clause & _]] true
     [[:star & _]] true
     :else false))
 
 ;;; plan
 
 (defn lookup
+  "A plan to retrieve a relation/table from the environment by name."
   [id]
   {::type :inferenceql.query.plan.type/lookup
    ::env/name id})
 
 (defn select
+  "A plan to filters the tuples/rows that match the pred fn.
+
+  NB: Used for the WHERE clause; unrelated to the SELECT clause."
   [op sexpr]
   {::type :inferenceql.query.plan.type/select
    ::sexpr sexpr
    ::plan op})
 
 (defn extended-project
+  "An (extended) projection plan. Used for simple selections of existing
+  columns as well as derived columns."
   [op coll]
-  ;; `coll` is a sequence of (s-expression, attribute) pairs.
+  ;; `coll` is a sequence of (fn-to-eval-s-expression, attribute-name) pairs.
   (let [terms (mapv #(zipmap [::sexpr ::relation/attribute] %)
                     coll)]
     {::type :inferenceql.query.plan.type/extended-project
@@ -335,15 +342,40 @@
     [[:scalar-expr & _]] true
     :else false))
 
+(defn ^:private select-star-plan
+  "Returns a plan for SELECT * and SELECT * EXCEPT ... clauses."
+  [node op]
+  (tree/match [node]
+    [[:select-star-clause & children]]
+    (select-star-plan children op)                          ; trying recur breaks with CLJ-2808
+
+    ;; plain SELECT *
+    [[[:star _]]]
+    op
+
+    ;; SELECT * EXCEPT (col1, col2, ...)
+    [[[:star & _] [:select-except-clause _except _lparen [:identifier-list & id-list] _rparen]]]
+    (loop [exclusions []
+           [id & rest-ids] (filterv (tree/tag-pred :identifier) id-list)]
+      (if id
+        (recur (conj exclusions (literal/read id)) rest-ids)
+        {::type :inferenceql.query.plan.type/star-except
+         ::exclusions exclusions
+         ::plan op}))
+
+    :else
+    (throw (ex-info "Can't generate selection plan - unknown * pattern"
+                    {:star-node node
+                     :partial-plan op}))))
+
 (defn select-plan
   [select-node group-by-node op]
   (assert (= :select-clause (tree/tag select-node)))
   (assert (or (nil? group-by-node) (= :group-by-clause (tree/tag group-by-node))))
-  (let [selections (selections select-node)
+  (let [selections (vec (selections select-node))
         distinct-clause (tree/get-node select-node :distinct-clause)
-        plan (cond (and (= 1 (count selections))
-                        (star? (first selections)))
-                   op
+        plan (cond (star? (first selections))
+                   (select-star-plan (first selections) op)
 
                    (or group-by-node (some aggregation? selections))
                    (aggregation-plan select-node group-by-node op)
@@ -505,6 +537,12 @@
         pred #(scalar/eval sexpr env bindings %)]
     (relation/select rel pred)))
 
+(defmethod eval :inferenceql.query.plan.type/star-except
+  [plan env bindings]
+  (let [{::keys [exclusions plan]} plan
+        rel (eval plan env bindings)]
+    (relation/remove-attributes rel exclusions)))
+
 (defmethod eval :inferenceql.query.plan.type/extended-project
   [plan env bindings]
   (let [{::keys [terms plan]} plan

src/inferenceql/query/relation.cljc

@@ -74,6 +74,7 @@
               :attrs (mapv second coll))))
 
 (defn select
+  "Selects/filters the tuples/rows that match the pred fn"
   [rel pred]
   (with-meta (filter pred (tuples rel))
     (meta rel)))
@@ -112,6 +113,14 @@
                                attr))]
     (relation rel :attrs attributes)))
 
+(defn remove-attributes
+  "Remove attributes from a relation."
+  [rel attrs]
+  (let [attributes' (remove (set attrs)
+                            (attributes rel))]
+    (-> (map #(apply dissoc % attrs) rel)
+        (relation :attrs attributes'))))
+
 (defn group-by
   [rel f]
   (->> (tuples rel)

src/inferenceql/query/tuple.cljc

@@ -24,7 +24,7 @@
 (defn select-attrs
   "Retrives tuple `tup` with only the attributes in `attrs`."
   [tup attrs]
-  (let [names (into #{} (map name) attrs)]
+  (let [names (into #{} (map name) attrs)]                  ; FIXME: should this be clojure.core/name?
     (medley/filter-keys (comp names name)
                         tup)))
 

test/inferenceql/query/strict_test.cljc

@@ -113,7 +113,7 @@
 (defspec select-star-except
   (prop/for-all [[table ks] gen-table-col-subset]
     (let [kset (set ks)
-          col-list (->> ks #_ (map name) (string/join ", "))
+          col-list (->> ks (string/join ", "))
           results (q (str "SELECT * EXCEPT (" col-list ") FROM data") table)]
       (is (every? (every-pred #(empty? (select-keys % ks))
                               #(empty? (set/intersection kset (set (keys %)))))