debug gramars

meta-introspector · Feb 20, 2024 · 9b4890e · 9b4890e
1 parent 97449d8
commit 9b4890e
Show file tree

Hide file tree

Showing 2 changed files with 37 additions and 4 deletions.
diff --git a/common/grammar-parser.cpp b/common/grammar-parser.cpp
@@ -278,6 +278,7 @@ namespace grammar_parser {
             while (*pos) {
                 pos = parse_rule(state, pos);
             }
+	    print_grammar(stderr,state);
             return state;
         } catch (const std::exception & err) {
             fprintf(stderr, "%s: error parsing grammar: %s\n", __func__, err.what());
@@ -403,10 +404,10 @@ namespace grammar_parser {
                 symbol_id_names[kv.second] = kv.first;
             }
             for (size_t i = 0, end = state.rules.size(); i < end; i++) {
-                // fprintf(file, "%zu: ", i);
-                // print_rule_binary(file, state.rules[i]);
+                fprintf(file, "%zu: ", i);
+                print_rule_binary(file, state.rules[i]);
                 print_rule(file, uint32_t(i), state.rules[i], symbol_id_names);
-                // fprintf(file, "\n");
+                fprintf(file, "\n");
             }
         } catch (const std::exception & err) {
             fprintf(stderr, "\n%s: error printing grammar: %s\n", __func__, err.what());

diff --git a/llama.cpp b/llama.cpp
@@ -8995,6 +8995,7 @@ static void llama_grammar_advance_stack(
     switch (pos->type) {
         case LLAMA_GRETYPE_RULE_REF: {
             const size_t                  rule_id = static_cast<size_t>(pos->value);
+	    fprintf(stderr, "RULE SHIFT:%ld\n", rule_id);
             const llama_grammar_element * subpos  = rules[rule_id].data();
             do {
                 // init new stack without the top (pos)
@@ -9060,8 +9061,39 @@ static std::vector<std::vector<const llama_grammar_element *>> llama_grammar_acc
             }
             llama_grammar_advance_stack(rules, new_stack, new_stacks);
         }
+	else
+	  {
+	    const llama_grammar_element * pos = stack.back();	    
+	    switch (pos->type) {
+	    case LLAMA_GRETYPE_RULE_REF: {
+	      const size_t                  rule_id = static_cast<size_t>(pos->value);
+	      fprintf(stderr, "RULE FAILED:%ld char:%c\n", rule_id, chr);
+	    };
+	      break;
+	    case LLAMA_GRETYPE_ALT:
+		fprintf(stderr, "something 1FAILED:%d %u char:%c\n", pos->type, pos->value, chr);
+		break;
+	    case LLAMA_GRETYPE_CHAR:
+	      fprintf(stderr, "CHAR FAILED:want:'%c' get:'%c'\n",
+		      pos->value,
+		      chr);
+	      break;
+	    case LLAMA_GRETYPE_CHAR_NOT:
+	      fprintf(stderr, "something 3 FAILED:%d %u char:%c\n", pos->type, pos->value, chr);
+	      break;
+	    case LLAMA_GRETYPE_CHAR_RNG_UPPER:
+	      fprintf(stderr, "something 4 FAILED:%d %u char:%c\n", pos->type, pos->value, chr);
+	      break;
+	    case LLAMA_GRETYPE_CHAR_ALT:
+	      fprintf(stderr, "something 5 FAILED:%d %u char:%c\n", pos->type, pos->value, chr);
+	     break;
+	    case LLAMA_GRETYPE_END:
+	      fprintf(stderr, "something 6 FAILED:%d %u char:%c\n", pos->type, pos->value, chr);
+	     break;
+
+	    }// switch
+	  }
     }
-
     return new_stacks;
 }