feat: Add GENERATE * EXCEPT capability

resources/inferenceql/query/base.bnf

@@ -186,6 +186,10 @@ join-expr-group ::= '(' join-expr ')'
 (* generate-expr *)
 
 generate-expr ::= #'(?i)GENERATE' ws generate-list ws #'(?i)UNDER' ws model-expr
+<generate-list> ::= generate-star-clause
+                  / model-var-list
+generate-star-clause ::= star (ws? generate-except-clause)?
+generate-except-clause ::= #'(?i)EXCEPT' ws? '(' ws? model-var-list ws? ')'
 
 (* generative-join-expr *)
 

resources/inferenceql/query/permissive.bnf

@@ -43,9 +43,6 @@ probability-expr ::= #'(?i)PROBABILITY' ws #'(?i)OF'
                      ws permissive-event-list
                      ws #'(?i)UNDER' ws model-expr
 
-(* generate-expr *)
-
-<generate-list> ::= star / identifier-list
 
 (* mutual-information-expr *)
 
@@ -57,3 +54,8 @@ mutual-info-expr ::= #'(?i)MUTUAL' ws #'(?i)INFORMATION'
                      ws #'(?i)OF' ws distribution-event
                      ws #'(?i)WITH' ws distribution-event
                      ws #'(?i)UNDER' ws model-expr
+
+(* model var abstraction *)
+
+<model-var> ::= identifier
+<model-var-list> ::= identifier-list

resources/inferenceql/query/strict.bnf

@@ -32,10 +32,6 @@ density-expr ::= #'(?i)PROBABILITY' ws #'(?i)DENSITY' ws #'(?i)OF'
                  ws density-event
                  ws #'(?i)UNDER' ws model-expr
 
-(* generate-expr *)
-
-<generate-list> ::= star / variable-list
-
 (* mutual-information-expr *)
 
 mutual-info-expr ::= #'(?i)MUTUAL' ws #'(?i)INFORMATION'
@@ -47,3 +43,8 @@ approx-mutual-info-expr ::= #'(?i)APPROXIMATE' ws #'(?i)MUTUAL' ws #'(?i)INFORMA
                             ws #'(?i)OF' ws variable-list
                             ws #'(?i)WITH' ws variable-list
                             ws #'(?i)UNDER' ws model-expr
+
+(* model var list *)
+
+<model-var> ::= variable
+<model-var-list> ::= variable-list

src/inferenceql/query/literal.cljc

@@ -14,14 +14,15 @@
                       node)]
     [[:value child]] (read child)
 
-    [[:bool s]]             (edn/read-string s)
-    [[:float s]]            (edn/read-string s)
-    [[:int s]]              (edn/read-string s)
-    [[:nat s]]              (edn/read-string s)
-    [[:identifier child]]   (read child)
-    [[:simple-symbol s]]    (edn/read-string (str \" s \"))
-    [[:delimited-symbol s]] (edn/read-string (str \" s \"))
-    [[:string s]]           (edn/read-string (str \" s \"))
+    [[:bool s]]              (edn/read-string s)
+    [[:float s]]             (edn/read-string s)
+    [[:int s]]               (edn/read-string s)
+    [[:nat s]]               (edn/read-string s)
+    [[:identifier child]]    (read child)
+    [[:variable _var child]] (read child)
+    [[:simple-symbol s]]     (edn/read-string (str \" s \"))
+    [[:delimited-symbol s]]  (edn/read-string (str \" s \"))
+    [[:string s]]            (edn/read-string (str \" s \"))
 
     [[:null _]] nil
     [nil] nil

src/inferenceql/query/parser/tree.cljc

@@ -123,6 +123,7 @@
     (walk/postwalk remove-ws node)))
 
 (defmacro match
+  "Like core.match/match, but removes whitespace nodes before matching."
   [vars & clauses]
   (let [match (macrovich/case :clj 'clojure.core.match/match
                               :cljs 'cljs.core.match/match)]

src/inferenceql/query/permissive.cljc

@@ -101,6 +101,9 @@
                               nodes)]
       [:generate-expr generate ws variable-list ws under ws model])
 
+    [[:generate-except-clause except "(" [:identifier-list & nodes] ")"]]
+    [:generate-except-clause except ws "(" (identifier-list->variable-list (into [:identifier-list] nodes)) ")"]
+
     [[:density-event-list & _]]
     (and-node :density-event-and
               (map tree/only-child

src/inferenceql/query/plan.cljc

@@ -82,9 +82,13 @@
      ::plan op}))
 
 (defn generate
-  [variables sexpr]
+  "Generate tuples from a model.
+
+  `mode` can be :include or :exclude"
+  [variables sexpr mode]
   {::type :inferenceql.query.plan.type/generate
    ::sexpr sexpr
+   ::mode mode
    ::variables variables})
 
 (defn limit
@@ -223,22 +227,26 @@
   (let [id (literal/read node)]
     (lookup id)))
 
-(defn variable-node->symbol
+(defn variable-node->string
+  "Extracts the identifier name from a variable node."
   [node]
-  (-> node tree/only-child-node literal/read ))
+  (-> node tree/only-child-node literal/read))
 
 (defmethod plan-impl :generate-expr
   [node]
   (tree/match [node]
     [[:generate-expr _generate generate-list _under model-expr]]
-    (let [sexpr (scalar/plan model-expr)
-          variables (case (tree/tag generate-list)
-                      :star
-                      '*
+    (let [sexpr (scalar/plan model-expr)]
+      (tree/match [generate-list]
+        [[:generate-star-clause [:star & _]]]
+        (generate :* sexpr :include)
+
+        [[:generate-star-clause [:star & _]
+          [:generate-except-clause _except _lparen (:or [:variable-list & variables] [:identifier-list & variables]) _rparen]]]
+        (generate (map variable-node->string (filter tree/branch? variables)) sexpr :exclude)
 
-                      :variable-list
-                      (map variable-node->symbol (tree/child-nodes generate-list)))]
-      (generate variables sexpr))))
+        [[:variable-list & variables]]
+        (generate (map variable-node->string (filter tree/branch? variables)) sexpr :include)))))
 
 (defmethod plan-impl :where-clause
   [node op]
@@ -574,15 +582,23 @@
 
 (defmethod eval :inferenceql.query.plan.type/generate
   [plan env bindings]
-  (let [{::keys [sexpr variables]} plan
+  (let [{::keys [sexpr variables mode]} plan
         model (scalar/eval sexpr env bindings)
-        variables (->> (if (= '* variables)
-                         (gpm/variables model)
-                         variables)
-                       (map str))
-        samples (map #(update-keys % str)
-                     (repeatedly #(gpm/simulate model variables {})))]
-    (relation/relation samples :attrs variables)))
+        model-vars (gpm/variables model)
+        ;; handle models with non-str vars
+        str-vars->model-vars (zipmap (map name model-vars) model-vars)
+        ->model-var (fn [k]
+                      (or (str-vars->model-vars k)
+                          (throw (ex-info (str "Could not find model var: " (pr-str k))
+                                          {:requested-var-name k
+                                           :known-model-vars   model-vars}))))
+        sim-vars (match/match [mode variables]
+                   [:include :*] model-vars
+                   [:include _] (map ->model-var variables)
+                   [:exclude _] (remove (set (map ->model-var variables)) model-vars))
+        samples (map #(update-keys % name)
+                     (repeatedly #(gpm/simulate model sim-vars {})))]
+    (relation/relation samples :attrs (map name sim-vars))))
 
 (defmethod eval :inferenceql.query.plan.type/insert
   [plan env bindings]