Upgrade codemirror with new parser

frontend/components/CellInput.js

@@ -43,7 +43,7 @@ import {
 } from "../imports/CodemirrorPlutoSetup.js"
 
 import { markdown, html as htmlLang, javascript, sqlLang, python, julia_mixed } from "./CellInput/mixedParsers.js"
-import { julia_andrey } from "../imports/CodemirrorPlutoSetup.js"
+import { julia } from "../imports/CodemirrorPlutoSetup.js"
 import { pluto_autocomplete } from "./CellInput/pluto_autocomplete.js"
 import { NotebookpackagesFacet, pkgBubblePlugin } from "./CellInput/pkg_bubble_plugin.js"
 import { awesome_line_wrapping, get_start_tabs } from "./CellInput/awesome_line_wrapping.js"
@@ -119,7 +119,7 @@ const common_style_tags = [
 
 export const pluto_syntax_colors_julia = HighlightStyle.define(common_style_tags, {
     all: { color: `var(--cm-color-editor-text)` },
-    scope: julia_andrey().language,
+    scope: julia().language,
 })
 
 export const pluto_syntax_colors_javascript = HighlightStyle.define(common_style_tags, {
@@ -661,7 +661,7 @@ export const CellInput = ({
                           ]
                         : [
                               //
-                              julia_andrey(),
+                              julia(),
                           ]),
                     go_to_definition_plugin,
                     pluto_autocomplete({

frontend/components/CellInput/LiveDocsFromCursor.js

@@ -2,7 +2,7 @@ import { EditorState, syntaxTree } from "../../imports/CodemirrorPlutoSetup.js"
 import { ScopeStateField } from "./scopestate_statefield.js"
 
 let get_root_variable_from_expression = (cursor) => {
-    if (cursor.name === "SubscriptExpression") {
+    if (cursor.name === "IndexExpression") {
         cursor.firstChild()
         return get_root_variable_from_expression(cursor)
     }
@@ -20,12 +20,13 @@ let get_root_variable_from_expression = (cursor) => {
 let VALID_DOCS_TYPES = [
     "Identifier",
     "FieldExpression",
-    "SubscriptExpression",
+    "IndexExpression",
     "MacroFieldExpression",
     "MacroIdentifier",
     "Operator",
-    "Definition",
-    "ParameterizedIdentifier",
+    "TypeHead",
+    "Signature",
+    "ParametrizedExpression",
 ]
 let keywords_that_have_docs_and_are_cool = [
     "import",
@@ -49,12 +50,12 @@ let is_docs_searchable = (/** @type {import("../../imports/CodemirrorPlutoSetup.
     } else if (VALID_DOCS_TYPES.includes(cursor.name)) {
         if (cursor.firstChild()) {
             do {
-                // Numbers themselves can't be docs searched, but using numbers inside SubscriptExpression can be.
-                if (cursor.name === "Number") {
+                // Numbers themselves can't be docs searched, but using numbers inside IndexExpression can be.
+                if (cursor.name === "IntegerLiteral" || cursor.name === "FloatLiteral") {
                     continue
                 }
                 // This is for the VERY specific case like `Vector{Int}(1,2,3,4) which I want to yield `Vector{Int}`
-                if (cursor.name === "TypeArgumentList") {
+                if (cursor.name === "BraceExpression") {
                     continue
                 }
                 if (cursor.name === "FieldName" || cursor.name === "MacroName" || cursor.name === "MacroFieldName") {
@@ -129,7 +130,7 @@ export let get_selected_doc_from_state = (/** @type {EditorState} */ state, verb
                     // - Struct name is useless: you are looking at the definition
                     // - Properties are just named, not in the workspace or anything
                     // Only thing we do want, are types and the right hand side of `=`'s.
-                    if (parents.includes("AssignmentExpression") && parents.indexOf("AssignmentExpression") < index_of_struct_in_parents) {
+                    if (parents.includes("binding") && parents.indexOf("binding") < index_of_struct_in_parents) {
                         // We're inside a `... = ...` inside the struct
                     } else if (parents.includes("TypedExpression") && parents.indexOf("TypedExpression") < index_of_struct_in_parents) {
                         // We're inside a `x::X` inside the struct
@@ -229,7 +230,7 @@ export let get_selected_doc_from_state = (/** @type {EditorState} */ state, verb
                 }
 
                 // `a = 1` would yield `=`, `a += 1` would yield `+=`
-                if (cursor.name === "AssignmentExpression") {
+                if (cursor.name === "binding") {
                     let end_of_first = cursor.node.firstChild.to
                     let beginning_of_last = cursor.node.lastChild.from
                     return state.doc.sliceString(end_of_first, beginning_of_last).trim()
@@ -320,7 +321,7 @@ export let get_selected_doc_from_state = (/** @type {EditorState} */ state, verb
                     return undefined
                 }
                 // If we are expanding to an AssigmentExpression, we DONT want to show `=`
-                if (parent.name === "AssignmentExpression") {
+                if (parent.name === "binding") {
                     return undefined
                 }
             } finally {

frontend/components/CellInput/lezer_template.js

@@ -1,4 +1,4 @@
-import { julia_andrey, NodeProp, syntaxTree, Text } from "../../imports/CodemirrorPlutoSetup.js"
+import { julia, NodeProp, syntaxTree, Text } from "../../imports/CodemirrorPlutoSetup.js"
 import lodash from "../../imports/lodash.js"
 
 // @ts-ignore
@@ -9,7 +9,7 @@ import ManyKeysWeakMap from "https://esm.sh/[email protected]?pin=v113&tar
  * @returns {SyntaxNode}
  */
 export let julia_to_ast = (julia_code) => {
-    return /** @type {any} */ (julia_andrey().language.parser.parse(julia_code).topNode.firstChild)
+    return /** @type {any} */ (julia().language.parser.parse(julia_code).topNode.firstChild)
 }
 
 // When you get errors while creating the templates (stuff related to substitutions),
@@ -23,7 +23,7 @@ const TEMPLATE_CREATION_VERBOSE = false
  *
  * @type {"SPEED" | "VALIDITY"}
  */
-const PERFORMANCE_MODE = /** @type {any} */ ("SPEED")
+const PERFORMANCE_MODE = /** @type {any} */ ("VALIDITY")
 
 const IS_IN_VALIDATION_MODE = PERFORMANCE_MODE === "VALIDITY"
 
@@ -261,7 +261,7 @@ export let match_template = (haystack_cursor, template, matches, verbose = false
                         // Skip comments
                         // TODO This is, I think, one of the few julia-only things right now.....
                         // .... Any sane way to factor this out?
-                        while (haystack_cursor.name === "Comment" || haystack_cursor.name === "BlockComment") {
+                        while (haystack_cursor.name === "LineComment" || haystack_cursor.name === "BlockComment") {
                             if (!haystack_cursor.nextSibling()) break
                         }
 
@@ -386,7 +386,7 @@ let substitutions_to_template = (ast, substitutions) => {
     }
 }
 
-let node_to_explorable = (cursor) => {
+export let node_to_explorable = (cursor) => {
     if ("cursor" in cursor) {
         cursor = cursor.cursor()
     }
@@ -466,7 +466,11 @@ export let to_template = function* (julia_code_object) {
                     }
                 })
             if (unused_substitutions.length > 0) {
-                throw new Error(`Some substitutions not applied, this means it couldn't be matched to a AST position: ${JSON.stringify(unused_substitutions)}`)
+                throw new Error(
+                    `Some substitutions not applied, this means it couldn't be matched to a AST position.\n\nUnused substitutions: ${JSON.stringify(
+                        unused_substitutions
+                    )}\n`
+                )
             }
             return result
         } else if (typeof julia_code_object === "function") {
@@ -490,6 +494,7 @@ export let jl_dynamic = weak_memo((template, ...substitutions) => {
 
 /** @type {WeakMap<TemplateStringsArray, { input: Array<Templatable>, result: JuliaCodeObject }>} */
 let template_cache = new WeakMap()
+
 /**
  * @param {TemplateStringsArray} template
  * @param {Array<Templatable>} substitutions
@@ -560,6 +565,9 @@ export let template = weak_memo1((julia_code_object) => {
     })
 })
 
+/**
+ *
+ */
 export let as_string = weak_memo1((/** @type {JuliaCodeObject} */ julia_code_object) => {
     let template_generator = to_template(julia_code_object)
     let julia_to_parse = intermediate_value(template_generator.next())
@@ -837,7 +845,7 @@ export const t = /** @type {const} */ ({
         yield "69"
         return {
             pattern: function Number(haystack, matches, verbose = false) {
-                return haystack != null && narrow_name(haystack) === "Number"
+                return haystack != null && (narrow_name(haystack) === "IntegerLiteral" || narrow_name(haystack) === "FloatLiteral")
             },
         }
     },
@@ -846,9 +854,7 @@ export const t = /** @type {const} */ ({
         yield `"A113"`
         return {
             pattern: function String(haystack, matches, verbose = false) {
-                return (
-                    haystack != null && (narrow_name(haystack) === "StringWithoutInterpolation" || narrow_name(haystack) === "TripleStringWithoutInterpolation")
-                )
+                return haystack != null && (narrow_name(haystack) === "StringLiteral" || narrow_name(haystack) === "NsStringLiteral")
             },
         }
     },
@@ -1057,6 +1063,8 @@ export function iterate_with_cursor({ tree, enter, leave, from = 0, to = tree.le
 ///////////////////////////////////
 
 /**
+ * Return `iterater_result.value` if `iterater_result.done` is `false`, otherwise throw an error.
+ *
  * Not sure why typescript doesn't infer the `Generator<T>` when I ask !iterater_result.done...
  * Maybe it will bite me later 🤷‍♀️
  *

frontend/components/CellInput/mixedParsers.js

@@ -12,7 +12,7 @@ import {
     sql,
     javascript,
     python,
-    julia_andrey,
+    julia,
     parseCode,
 } from "../../imports/CodemirrorPlutoSetup.js"
 
@@ -80,38 +80,46 @@ const overlayHack = (overlay, input) => {
     })
 }
 
-const STRING_NODE_NAMES = new Set([
-    "TripleString",
-    "String",
-    "CommandString",
-    "TripleStringWithoutInterpolation",
-    "StringWithoutInterpolation",
-    "CommandStringWithoutInterpolation",
-])
+export const STRING_NODE_NAMES = new Set(["StringLiteral", "CommandLiteral", "NsStringLiteral", "NsCommandLiteral"])
 
 const juliaWrapper = parseMixed((node, input) => {
-    if (!STRING_NODE_NAMES.has(node.type.name)) {
+    // TODO: only get .node once
+    if (node.name !== "NsStringLiteral" && node.name !== "StringLiteral") {
         return null
     }
 
-    let is_tripple_string = node.name === "TripleString" || node.name === "TripleStringWithoutInterpolation"
+    const first_string_delim = node.node.getChild('"""') ?? node.node.getChild('"')
+    if (first_string_delim == null) return null
+    const last_string_delim = node.node.lastChild
+    if (last_string_delim == null) return null
 
-    const offset = is_tripple_string ? 3 : 1
-    const string_content_from = node.from + offset
-    const string_content_to = Math.min(node.to - offset, input.length)
+    const offset = first_string_delim.to - first_string_delim.from
+    console.log({ first_string_delim, last_string_delim, offset })
+    const string_content_from = first_string_delim.to
+    const string_content_to = Math.min(last_string_delim.from, input.length)
 
     if (string_content_from >= string_content_to) {
         return null
     }
 
-    // Looking for tag OR MacroIdentifier
-    const tagNode = node.node?.prevSibling || node.node?.parent?.prevSibling
-    if (tagNode == null || (tagNode.name !== "MacroIdentifier" && tagNode.name !== "Prefix")) {
-        // If you can't find a tag node, something is broken in the julia syntax,
-        // so parse it as Julia. Probably wrong interpolation!
-        return null
+    let tagNode
+    if (node.name === "NsStringLiteral") {
+        tagNode = node.node.firstChild
+        // if (tagNode) tag = input.read(tagNode.from, tagNode.to)
+    } else {
+        // must be a string, let's search for the parent `@htl`.
+        const start = node.node
+        const p1 = start.parent
+        if (p1 != null && p1.name === "Arguments") {
+            const p2 = p1.parent
+            if (p2 != null && p2.name === "MacrocallExpression") {
+                tagNode = p2.getChild("MacroIdentifier")
+            }
+        }
     }
 
+    if (tagNode == null) return null
+
     const is_macro = tagNode.name === "MacroIdentifier"
 
     const tag = input.read(tagNode.from, tagNode.to)
@@ -136,7 +144,7 @@ const juliaWrapper = parseMixed((node, input) => {
         let child = node.node.firstChild.cursor()
 
         do {
-            if (last_content_start !== child.from) {
+            if (last_content_start < child.from) {
                 overlay.push({ from: last_content_start, to: child.from })
             }
             last_content_start = child.to
@@ -166,10 +174,10 @@ const juliaWrapper = parseMixed((node, input) => {
 })
 
 const julia_mixed = (config) => {
-    const julia = julia_andrey(config)
+    const julia_simple = julia(config)
     // @ts-ignore
-    julia.language.parser = julia.language.parser.configure({ wrap: juliaWrapper })
-    return julia
+    julia_simple.language.parser = julia_simple.language.parser.configure({ wrap: juliaWrapper })
+    return julia_simple
 }
 
 export { julia_mixed, sqlLang, pythonLanguage, javascript, htmlLanguage, javascriptLanguage, python, markdown, html }

frontend/components/CellInput/pkg_bubble_plugin.js

@@ -39,7 +39,7 @@ function find_import_statements({ doc, tree, from, to }) {
                 quotelevel++
             }
             // `$(...)` when inside quote
-            if (cursor.name === "InterpolationExpression") {
+            if (cursor.name === "InterpExpression") {
                 quotelevel--
             }
             if (quotelevel !== 0) return
@@ -160,7 +160,7 @@ function find_import_statements({ doc, tree, from, to }) {
             if (cursor.name === "QuoteExpression" || cursor.name === "QuoteStatement") {
                 quotelevel--
             }
-            if (cursor.name === "InterpolationExpression") {
+            if (cursor.name === "InterpExpression") {
                 quotelevel++
             }
         },

frontend/components/CellInput/pluto_autocomplete.js

@@ -7,6 +7,7 @@ import { ScopeStateField } from "./scopestate_statefield.js"
 import { open_bottom_right_panel } from "../BottomRightPanel.js"
 import { ENABLE_CM_AUTOCOMPLETE_ON_TYPE } from "../CellInput.js"
 import { GlobalDefinitionsFacet } from "./go_to_definition_plugin.js"
+import { STRING_NODE_NAMES } from "./mixedParsers.js"
 
 let { autocompletion, completionKeymap, completionStatus, acceptCompletion, selectedCompletion } = autocomplete
 
@@ -123,7 +124,7 @@ const match_operator_symbol_complete = (/** @type {autocomplete.CompletionContex
 function match_string_complete(state, pos) {
     const tree = syntaxTree(state)
     const node = tree.resolve(pos)
-    if (node == null || (node.name !== "TripleString" && node.name !== "String")) {
+    if (node == null || !STRING_NODE_NAMES.has(node.name)) {
         return false
     }
     return true
@@ -165,7 +166,8 @@ const julia_code_completions_to_cm =
     async (/** @type {autocomplete.CompletionContext} */ ctx) => {
         if (match_latex_symbol_complete(ctx)) return null
         if (!ctx.explicit && writing_variable_name_or_keyword(ctx)) return null
-        if (!ctx.explicit && ctx.tokenBefore(["Number", "Comment", "String", "TripleString", "Symbol"]) != null) return null
+        if (!ctx.explicit && ctx.tokenBefore(["IntegerLiteral", "FloatLiteral", "LineComment", "BlockComment", "Symbol", ...STRING_NODE_NAMES]) != null)
+            return null
 
         let to_complete = /** @type {String} */ (ctx.state.sliceDoc(0, ctx.pos))
 
@@ -266,13 +268,12 @@ const julia_code_completions_to_cm =
 const complete_anyword = async (/** @type {autocomplete.CompletionContext} */ ctx) => {
     if (match_latex_symbol_complete(ctx)) return null
     if (!ctx.explicit && writing_variable_name_or_keyword(ctx)) return null
-    if (!ctx.explicit && ctx.tokenBefore(["Number", "Comment", "String", "TripleString", "Symbol"]) != null) return null
+    if (!ctx.explicit && ctx.tokenBefore(["IntegerLiteral", "FloatLiteral", "LineComment", "BlockComment", "Symbol", ...STRING_NODE_NAMES]) != null) return null
 
     const results_from_cm = await autocomplete.completeAnyWord(ctx)
     if (results_from_cm === null) return null
 
-    const last_token = ctx.tokenBefore(["Identifier", "Number"])
-    if (last_token == null || last_token.type?.name === "Number") return null
+    if (ctx.tokenBefore(["Identifier", "IntegerLiteral", "FloatLiteral"])) return null
 
     return {
         from: results_from_cm.from,
@@ -323,7 +324,8 @@ const global_variables_completion =
     async (/** @type {autocomplete.CompletionContext} */ ctx) => {
         if (match_latex_symbol_complete(ctx)) return null
         if (!ctx.explicit && writing_variable_name_or_keyword(ctx)) return null
-        if (!ctx.explicit && ctx.tokenBefore(["Number", "Comment", "String", "TripleString", "Symbol"]) != null) return null
+        if (!ctx.explicit && ctx.tokenBefore(["IntegerLiteral", "FloatLiteral", "LineComment", "BlockComment", "Symbol", ...STRING_NODE_NAMES]) != null)
+            return null
 
         // see `is_wc_cat_id_start` in Julia's source for a complete list
         const there_is_a_dot_before = ctx.matchBefore(/\.[\p{L}\p{Nl}\p{Sc}\d_!]*$/u)
@@ -454,7 +456,7 @@ const special_symbols_completion = (/** @type {() => Promise<SpecialSymbols?>} *
     return async (/** @type {autocomplete.CompletionContext} */ ctx) => {
         if (!match_latex_symbol_complete(ctx)) return null
         if (!ctx.explicit && writing_variable_name_or_keyword(ctx)) return null
-        if (!ctx.explicit && ctx.tokenBefore(["Number", "Comment"]) != null) return null
+        if (!ctx.explicit && ctx.tokenBefore(["IntegerLiteral", "FloatLiteral", "LineComment", "BlockComment"]) != null) return null
 
         const result = await get_special_symbols()
         return await autocomplete.completeFromList(result ?? [])(ctx)