diff --git a/frontend/components/CellInput/debug_syntax_plugin.js b/frontend/components/CellInput/debug_syntax_plugin.js index 83d4d6ca5d..387676e8ba 100644 --- a/frontend/components/CellInput/debug_syntax_plugin.js +++ b/frontend/components/CellInput/debug_syntax_plugin.js @@ -1,8 +1,8 @@ -import { EditorView, syntaxTree, syntaxTreeAvailable } from "../../imports/CodemirrorPlutoSetup.js" +import { EditorView, syntaxTree, syntaxTreeAvailable, Text } from "../../imports/CodemirrorPlutoSetup.js" import { iterate_with_cursor } from "./lezer_template.js" /** - * @param {any} doc + * @param {Text} doc * @param {ReturnType} tree */ let find_error_nodes = (doc, tree) => { diff --git a/frontend/components/CellInput/lezer_template.js b/frontend/components/CellInput/lezer_template.js index 64f4723d76..beeb255a73 100644 --- a/frontend/components/CellInput/lezer_template.js +++ b/frontend/components/CellInput/lezer_template.js @@ -1,1033 +1,3 @@ -import { julia, NodeProp, syntaxTree, Text } from "../../imports/CodemirrorPlutoSetup.js" -import lodash from "../../imports/lodash.js" - -// @ts-ignore -import ManyKeysWeakMap from "https://esm.sh/many-keys-weakmap@1.0.0?pin=v113&target=es2020" - -/** - * @param {string} julia_code - * @returns {SyntaxNode} - */ -export let julia_to_ast = (julia_code) => { - return /** @type {any} */ (julia().language.parser.parse(julia_code).topNode.firstChild) -} - -// When you get errors while creating the templates (stuff related to substitutions), -// turn this on, and you will get a lot of info you can debug with! -const TEMPLATE_CREATION_VERBOSE = false - -/** - * Settings this to `"VALIDITY"` will enable some (currently, one) slow validations. - * Might be useful to run set this to `"VALIDITY"` every so often to make sure there are no bugs. - * In production this should always to `"SPEED"` - * - * @type {"SPEED" | "VALIDITY"} - */ -const PERFORMANCE_MODE = /** @type {any} */ ("VALIDITY") - -const IS_IN_VALIDATION_MODE = PERFORMANCE_MODE === "VALIDITY" - -/** - * @template {Array} P - * @template {(...args: P) => any} T - * @param {T} fn - * @param {(...args: P) => any} cachekey_resolver - * @param {WeakMap} cache - * @returns {T} - */ -let memo = (fn, cachekey_resolver = /** @type {any} */ ((x) => x), cache = new Map()) => { - return /** @type {any} */ ( - (/** @type {P} */ ...args) => { - let cachekey = cachekey_resolver(...args) - let result = cache.get(cachekey) - if (result != null) { - return result - } else { - let result = fn(...args) - - if (result == undefined) { - throw new Error("Memoized function returned undefined") - } - cache.set(cachekey, result) - return result - } - } - ) -} - -/** - * @template {(...args: any) => any} T - * @param {T} fn - * @param {(...x: Parameters) => Array} cachekey_resolver - * @returns {T} - */ -let weak_memo = (fn, cachekey_resolver = (...x) => x) => memo(fn, cachekey_resolver, new ManyKeysWeakMap()) - -/** - * @template {(arg: any) => any} T - * @param {T} fn - * @returns {T} - */ -let weak_memo1 = (fn) => memo(fn, (x) => x, new WeakMap()) - -// Good luck figuring anything out from these types πŸ’• - -/** - * @typedef TreeCursor - * @type {import("../../imports/CodemirrorPlutoSetup.js").TreeCursor} - * - * @typedef SyntaxNode - * @type {TreeCursor["node"]} - */ - -/** - * @typedef LezerOffsetNode - * @type {{ - * name: string, - * from: number, - * to: number, - * node: SyntaxNode, - * }} - */ - -/** - * @typedef TemplateGenerator - * @type {Generator} - */ - -/** - * @typedef Substitution - * @type {() => TemplateGenerator} - */ - -/** - * @typedef Templatable - * @type {JuliaCodeObject | Substitution} - */ - -/** - * @typedef MatchResult - * @type {any} - */ - -/** - * @typedef Matcher - * @type {{ - * match: (haystack_cursor: TreeCursor | SyntaxNode, verbose?: boolean) => void | { [key: string]: MatchResult } - * }} - */ - -/** @param {TreeCursor} cursor */ -export let child_cursors = function* (cursor) { - if (cursor.firstChild()) { - try { - do { - yield cursor - } while (cursor.nextSibling()) - } finally { - cursor.parent() - } - } -} - -/** @param {SyntaxNode} node */ -export let child_nodes = function* (node) { - if (node.firstChild) { - /** @type {SyntaxNode?} */ - let child = node.firstChild - do { - yield child - } while ((child = child.nextSibling)) - } -} - -/** - * @typedef AstTemplate - * @type {{ - * name?: string, - * from?: number, - * to?: number, - * node: SyntaxNode, - * children: Array, - * } | { - * pattern: ( - * haystack: (TreeCursor | null), - * matches: { [key: string]: any }, - * verbose?: boolean - * ) => boolean, - * }} - */ - -/** - * @param {TreeCursor | null} haystack_cursor - * @param {AstTemplate} template - * @param {{ [name: string]: any }} matches - * @param {boolean} verbose - */ -export let match_template = (haystack_cursor, template, matches, verbose = false) => { - if (verbose) { - console.group("Current haystack:", !haystack_cursor ? null : haystack_cursor.node.name) - - console.groupCollapsed("Details") - try { - console.log(`template:`, template) - console.log(`haystack_cursor:`, !haystack_cursor ? null : haystack_cursor.node.toString()) - - if ("node" in template) { - console.log(`template.node:`, template.node) - console.log(`template.node.toString():`, template.node.toString()) - } else if ("pattern" in template) { - console.log(`template.pattern:`, template.pattern) - } - } finally { - console.groupEnd() - } - } - - try { - if ("pattern" in template) { - let pattern = template.pattern - if (typeof pattern !== "function") { - throw new Error(`Unknown pattern "${pattern}"`) - } - - let matches_before_matching = {} - if (verbose) { - matches_before_matching = { ...matches } - console.groupCollapsed(`Matching against pattern: ${template.pattern.name}()`) - } - - let did_match = null - try { - did_match = pattern(haystack_cursor, matches, verbose) - } finally { - if (verbose) { - console.groupEnd() - } - } - if (verbose) { - if (did_match) { - console.log(`βœ… because the pattern was happy! All hail the pattern!`) - if (!lodash.isEqual(matches, matches_before_matching)) { - let new_matches = lodash.omit(matches, Object.keys(matches_before_matching)) - console.log(` WE EVEN GOT NEW MATCHES SAY WHAAAAAAT:`, new_matches) - } - } else { - console.log(`❌ because... well, you should ask the pattern that!`) - } - } - - return did_match - } else if ("node" in template) { - let { node, children } = template - - verbose && console.log(`Matching against node: ${template.node.name}`) - - if (!haystack_cursor) { - if (node.name === "end") { - verbose && console.log(`βœ… No node left to match, but it was the end anyway`) - return true - } - verbose && console.log(`❌ because no cursor left to match against`) - return false - } - - if (haystack_cursor.type.isError) { - // Not sure about this yet but ehhhh - verbose && console.log(`βœ… because ⚠`) - return true - } - - if (haystack_cursor.name !== node.name) { - verbose && console.log(`❌ because name mismatch "${haystack_cursor.name}" !== "${node.name}"`) - return false - } - - if (haystack_cursor.firstChild()) { - try { - let is_last_from_haystack = false - for (let template_child of children) { - if (is_last_from_haystack) { - verbose && console.log(`Haystack is empty, but there are more children in template... lets see`) - let child_does_match = match_template(null, template_child, matches, verbose) - if (!child_does_match) { - verbose && console.log(`❌ template child did not accept null for an answer`, template_child, haystack_cursor.toString()) - return false - } - verbose && console.log(`πŸ‘ŒπŸ½ This template child was fine with null`) - continue - } - - // Skip comments - // TODO This is, I think, one of the few julia-only things right now..... - // .... Any sane way to factor this out? - while (haystack_cursor.name === "LineComment" || haystack_cursor.name === "BlockComment") { - if (!haystack_cursor.nextSibling()) break - } - - let child_does_match = match_template(haystack_cursor, template_child, matches, verbose) - - if (!child_does_match) { - verbose && console.log(`❌ because a child mismatch`, template_child, haystack_cursor.toString()) - return false - } - - // This is where we actually move the haystack_cursor (in sync with the `template.children`) - // to give the nested match templates more freedom in move the cursor around between siblings. - is_last_from_haystack = !haystack_cursor.nextSibling() - } - - if (verbose && !is_last_from_haystack) { - let spare_children = [] - do { - spare_children.push(haystack_cursor.node) - } while (haystack_cursor.nextSibling()) - for (let child of spare_children) { - haystack_cursor.prevSibling() - } - - // prettier-ignore - console.warn("We did match all the children of the template, but there are more in the haystack... Might want to actually not match this?", spare_children, template) - } - - verbose && console.log(`βœ… because all children match`) - return true - } finally { - haystack_cursor.parent() - } - } else { - if (template.children.length !== 0) { - verbose && console.log(`Haystack node is empty, but template has children... lets see`) - - for (let child of template.children) { - if (!match_template(null, child, matches, verbose)) { - verbose && console.log(`❌ because child template wasn't okay with having no children`, child) - return false - } - } - verbose && console.log(`βœ… All template children we're fine with having no children to check on`) - return true - } else { - verbose && console.log(`βœ… Template also has no children, yayyy`) - return true - } - } - } else { - console.log(`template:`, template) - throw new Error("waaaah") - } - } finally { - if (verbose) { - console.groupEnd() - } - } -} - -export class JuliaCodeObject { - /** - * @param {TemplateStringsArray} template - * @param {any[]} substitutions - * */ - constructor(template, substitutions) { - let flattened_template = [] - let flattened_substitutions = [] - - flattened_template.push(template[0]) - for (let [string_part, substitution] of lodash.zip(template.slice(1), substitutions)) { - if (substitution instanceof JuliaCodeObject) { - flattened_template[flattened_template.length - 1] += substitution.template[0] - for (let [sub_string_part, sub_substitution] of lodash.zip(substitution.template.slice(1), substitution.substitutions)) { - flattened_substitutions.push(sub_substitution) - flattened_template.push(sub_string_part) - } - flattened_template[flattened_template.length - 1] += string_part - } else { - flattened_substitutions.push(substitution) - flattened_template.push(string_part) - } - } - - this.template = flattened_template - this.substitutions = flattened_substitutions - } -} - -/** - * @param {SyntaxNode} ast - * @param {Array<{ generator: TemplateGenerator, from: number, to: number, used?: boolean }>} substitutions - * @returns {AstTemplate} - */ -let substitutions_to_template = (ast, substitutions) => { - for (let substitution of substitutions) { - if (ast.from === substitution.from && ast.to === substitution.to) { - // Hacky and weird, but it is only for validation - substitution.used = true - let result = substitution.generator.next({ - name: ast.name, - from: ast.from, - to: ast.to, - node: ast, - }) - - if (result.done) { - return result.value - } else { - throw new Error("Template generator not done providing ast node") - } - } - } - - return { - name: ast.name, - from: ast.from, - to: ast.to, - children: Array.from(child_nodes(ast)).map((node) => substitutions_to_template(node, substitutions)), - node: ast, - } -} - -export let node_to_explorable = (cursor) => { - if ("cursor" in cursor) { - cursor = cursor.cursor() - } - - let children = [] - if (cursor.firstChild()) { - try { - do { - children.push(node_to_explorable(cursor)) - } while (cursor.nextSibling()) - } finally { - cursor.parent() - } - } - return { - name: cursor.name, - from: cursor.from, - to: cursor.to, - children, - } -} - -/** - * @param {Templatable} julia_code_object - * @returns {TemplateGenerator} - */ -export let to_template = function* (julia_code_object) { - TEMPLATE_CREATION_VERBOSE && console.group(`to_template(`, typeof julia_code_object === "function" ? julia_code_object.name + "()" : julia_code_object, `)`) - try { - if (julia_code_object instanceof JuliaCodeObject) { - let julia_code_to_parse = "" - - let subsitions = [] - for (let [string_part, substitution] of lodash.zip(julia_code_object.template, julia_code_object.substitutions)) { - julia_code_to_parse += string_part - - if (substitution) { - let substitution_generator = to_template(substitution) - - let substitution_code = intermediate_value(substitution_generator.next()) - - subsitions.push({ - from: julia_code_to_parse.length, - to: julia_code_to_parse.length + substitution_code.length, - generator: substitution_generator, - }) - julia_code_to_parse += substitution_code - } - } - - let template_node = yield julia_code_to_parse - - let substitution_with_proper_position = subsitions.map((substitution) => { - return { - from: substitution.from + template_node.from, - to: substitution.to + template_node.from, - generator: substitution.generator, - used: false, - } - }) - - if (TEMPLATE_CREATION_VERBOSE) { - console.log(`julia_code_to_parse:`, julia_code_to_parse) - console.log(`template_node:`, node_to_explorable(template_node.node.cursor())) - console.log(`subsitions:`, subsitions) - console.log(`substitution_with_proper_position:`, substitution_with_proper_position) - } - - let result = substitutions_to_template(template_node.node, substitution_with_proper_position) - let unused_substitutions = substitution_with_proper_position - .filter((substitution) => !substitution.used) - .map((x) => { - return { - text: julia_code_to_parse.slice(x.from, x.to), - from: x.from, - to: x.to, - } - }) - if (unused_substitutions.length > 0) { - throw new Error( - `Some substitutions not applied, this means it couldn't be matched to a AST position.\n\nUnused substitutions: ${JSON.stringify( - unused_substitutions - )}\n` - ) - } - return result - } else if (typeof julia_code_object === "function") { - return yield* julia_code_object() - } else { - console.log(`julia_code_object:`, julia_code_object) - throw new Error("Unknown substition type") - } - } finally { - TEMPLATE_CREATION_VERBOSE && console.groupEnd() - } -} - -/** - * @param {TemplateStringsArray} template - * @param {any[]} substitutions - * */ -export let jl_dynamic = weak_memo((template, ...substitutions) => { - return new JuliaCodeObject(template, substitutions) -}) - -/** @type {WeakMap, result: JuliaCodeObject }>} */ -let template_cache = new WeakMap() - -/** - * @param {TemplateStringsArray} template - * @param {Array} substitutions - */ -export let jl = (template, ...substitutions) => { - let cached = template_cache.get(template) - if (cached != null) { - let { input, result } = cached - if (IS_IN_VALIDATION_MODE) { - if (!lodash.isEqual(substitutions, input)) { - console.trace("Substitutions changed on `jl` template string.. change to `jl_dynamic` if you need this.") - } - } - return result - } else { - // Uncomment this if you want to check if the cache is working - // console.log("Creating template for", template, substitutions) - let result = new JuliaCodeObject(template, substitutions) - template_cache.set(template, { - input: substitutions, - result: result, - }) - return result - } -} - -/** - * Turns a ``` jl`` ``` (or substitution) into a template with a `.match(cursor)` method. - * - * @type {(code: Templatable) => Matcher} - */ -export let template = weak_memo1((julia_code_object) => { - let template_generator = to_template(julia_code_object) - let julia_to_parse = intermediate_value(template_generator.next()) - let template_ast = julia_to_ast(julia_to_parse) - - let template_description = return_value( - template_generator.next({ - from: 0, - to: julia_to_parse.length, - name: template_ast.name, - node: /** @type {any} */ (template_ast), - }) - ) - - return /** @type {Matcher} */ ({ - /** - * @param {TreeCursor | SyntaxNode} haystack_cursor - * @param {boolean} verbose? - **/ - template_description, - match(haystack_cursor, verbose = false) { - // Performance gain for not converting to `TreeCursor` possibly πŸ€·β€β™€οΈ - if ("node" in template_description && template_description.node.name !== haystack_cursor.name) return - if (haystack_cursor.type.isError) return null - - if ("cursor" in haystack_cursor) haystack_cursor = haystack_cursor.cursor() - - let matches = /** @type {{ [key: string]: MatchResult }} */ ({}) - - verbose && console.groupCollapsed(`Starting a match at ${haystack_cursor.name}`) - try { - return match_template(haystack_cursor, template_description, matches, verbose) ? matches : null - } finally { - verbose && console.groupEnd() - } - }, - }) -}) - -/** - * - */ -export let as_string = weak_memo1((/** @type {JuliaCodeObject} */ julia_code_object) => { - let template_generator = to_template(julia_code_object) - let julia_to_parse = intermediate_value(template_generator.next()) - // @ts-ignore - template_generator.return() - return julia_to_parse -}) - -export let as_node = weak_memo1((/** @type {JuliaCodeObject} */ julia_code_object) => { - return julia_to_ast(as_string(julia_code_object)) -}) - -export let as_doc = weak_memo1((/** @type {JuliaCodeObject} */ julia_code_object) => { - return Text.of([as_string(julia_code_object)]) -}) - -/** - * @template {(name: string, other_arg: Object) => any} T - * @param {T} func - * @returns {T} - **/ -let memo_first_argument_weakmemo_second = (func) => { - let fake_weakmap_no_arg = {} - let per_name_memo = memo((name) => { - return weak_memo1((arg) => { - if (arg === fake_weakmap_no_arg) return func(name, undefined) - return func(name, arg) - }) - }) - - return /** @type {T} */ ( - (name, arg = fake_weakmap_no_arg) => { - let sub_memo = per_name_memo(name) - return sub_memo(arg) - } - ) -} - -/** @type {Substitution} */ -function* any() { - yield "expression" - return { - pattern: function expression(cursor, matches, verbose = false) { - if (!cursor) { - verbose && console.log("❌ I want anything!! YOU GIVE ME NULL???") - return false - } - if (cursor.type.is("keyword")) { - verbose && console.log("❌ Keywords are not allowed!") - return false - } - return true - }, - } -} - -/** - * @param {TemplateGenerator} template_generator - * @return {TemplateGenerator} - * */ -function* narrow_template(template_generator) { - let ast = yield intermediate_value(template_generator.next()) - - if (ast.node.firstChild && ast.node.from === ast.node.firstChild.from && ast.node.to === ast.node.firstChild.to) { - console.log("Narrowing!!!!", ast.node, ast.node.firstChild) - return { - node: ast.node, - from: ast.from, - to: ast.to, - children: [ - return_value( - template_generator.next({ - ...ast, - node: ast.node.firstChild, - }) - ), - ], - } - } else { - return return_value(template_generator.next(ast)) - } -} - -export const t = /** @type {const} */ ({ - any: any, - /** - * Match no, one, or multiple! Like `*` in regex. - * It stores it's matches as `{ [name]: Array<{ node: SyntaxNode, matches: MatchResult }> }` - * - * If name isn't provided it will not store any of the matches.. useful if you really really don't care about something. - * - * @type {(name?: string, what?: Templatable) => Substitution} - */ - many: memo_first_argument_weakmemo_second((name, of_what = any) => { - return function* many() { - let template_generator = to_template(of_what) - let ast = yield intermediate_value(template_generator.next()) - - // Ugly but it works - let narrowed_node = null - let sub_template = null - if (ast.node.firstChild && ast.node.from === ast.node.firstChild.from && ast.node.to === ast.node.firstChild.to) { - narrowed_node = ast.node - sub_template = return_value( - template_generator.next({ - ...ast, - node: ast.node.firstChild, - }) - ) - } else { - sub_template = return_value(template_generator.next(ast)) - } - - // let sub_template = yield* narrow_template(to_template(of_what)) - - return { - narrowed_node, - sub_template, - pattern: function many(cursor, matches, verbose = false) { - if (!cursor) { - verbose && console.log("βœ… Nothing to see here... I'm fine with that - many") - return true - } - - if (narrowed_node) { - if (cursor.name !== narrowed_node.name) { - verbose && console.log("❌ Tried to go in, but she wasn't my type - many") - cursor.prevSibling() - return true - } - cursor.firstChild() - } - - try { - let matches_nodes = [] - while (true) { - // So, big oof here, but I think we shouldn't match error nodes in many - if (cursor.type.isError) { - cursor.prevSibling() - verbose && console.log("βœ‹ I don't do errors - many") - return true // Still we did finish, lets just hope someone else cares about the error - } - - let local_match = {} - let did_match = match_template(cursor, sub_template, local_match, verbose) - if (!did_match) { - // Move back on child, as that is the child that DIDN'T match - // And we want to give the next template a change to maybe match it - cursor.prevSibling() - break - } - matches_nodes.push({ node: cursor.node, match: local_match }) - - if (!cursor.nextSibling()) break - } - - if (name != null) { - matches[name] = matches_nodes - } - return true - } finally { - if (narrowed_node) { - cursor.parent() - } - } - }, - } - } - }), - - /** - * Match either a single node or none. Like `?` in regex. - * @type {(what: Templatable) => Substitution} - */ - maybe: weak_memo1((what) => { - return function* maybe() { - let sub_template = yield* to_template(what) - return { - sub_template, - pattern: function maybe(cursor, matches, verbose = false) { - if (!cursor) return true - if (cursor.type.isError) return true - - let did_match = match_template(cursor, sub_template, matches, verbose) - - if (did_match === false) { - // Roll back position because we didn't match - cursor.prevSibling() - } - return true - }, - } - } - }), - - /** - * This is an escape hatch. - * Ideally I'd want to ask for multiple different templates to match, but I can't easily get that to work yet. - * So for now, you can ask templates to just match "Anything kinda like this", - * and then do further matching on the result manually. - * - * More technically, this says "match anything that will appear in my position in the AST". - * It does not care about the type. Don't use this recklessly! - * - * @type {(what: Templatable) => Substitution} - * */ - anything_that_fits: weak_memo1((what) => { - return function* anything_that_fits() { - // We send the template code upwards, but we fully ignore the output - yield* to_template(what) - return { - pattern: function anything_that_fits(cursor, matches, verbose = false) { - return true - }, - } - } - }), - /** - * This is an escape hatch, like {@linkcode anything_that_fits}, - * but it does also check for node type at least. - * - * @type {(what: Templatable) => Substitution} - * */ - something_with_the_same_type_as: weak_memo1((what) => { - return function* something_with_the_same_type_as() { - let template_generator = to_template(what) - let julia_to_parse = intermediate_value(template_generator.next()) - - let ast = yield julia_to_parse - - // @ts-ignore - template_generator.return() - - return { - pattern: function something_with_the_same_type_as(haystack, matches, verbose = false) { - return haystack != null && ast.name === haystack.name - }, - } - } - }), - - /** - * This "higher-order" template pattern is for adding their nodes to the matches. - * Without a pattern (e.g. `t.as("foo")`) it will default to `t.any` - * - * @type {(name: string, what?: Templatable) => Substitution} - */ - as: memo_first_argument_weakmemo_second((name, what = any) => { - return function* as() { - let sub_template = yield* to_template(what) - return { - sub_template, - pattern: function as(haystack, matches, verbose = false) { - let did_match = match_template(haystack, sub_template, matches, verbose) - if (did_match === true) { - matches[name] = haystack?.["node"] - } - return haystack != null && did_match - }, - } - } - }), - - /** @type {Substitution} */ - Identifier: function* Identifier() { - yield "identifier" - return { - pattern: function Identifier(haystack, matches, verbose = false) { - return haystack != null && narrow_name(haystack) === "Identifier" - }, - } - }, - /** @type {Substitution} */ - Number: function* Number() { - yield "69" - return { - pattern: function Number(haystack, matches, verbose = false) { - return haystack != null && (narrow_name(haystack) === "IntegerLiteral" || narrow_name(haystack) === "FloatLiteral") - }, - } - }, - /** @type {Substitution} */ - String: function* String() { - yield `"A113"` - return { - pattern: function String(haystack, matches, verbose = false) { - return haystack != null && (narrow_name(haystack) === "StringLiteral" || narrow_name(haystack) === "NsStringLiteral") - }, - } - }, -}) - -/** - * Basically exists for {@linkcode create_specific_template_maker} - * - * @type {(template: Templatable, meta_template: Matcher) => Matcher} - */ -export let take_little_piece_of_template = weak_memo((template, meta_template) => { - let template_generator = to_template(template) - let julia_to_parse = intermediate_value(template_generator.next()) - - // Parse the AST from the template, but we don't send it back to the template_generator yet! - let template_ast = julia_to_ast(julia_to_parse) - - let match = null - // Match our created template code to the meta-template, which will yield us the part of the - // AST that falls inside the "content" in the meta-template. - if ((match = meta_template.match(template_ast))) { - let { content } = /** @type {{ content: SyntaxNode }} */ (match) - - let possible_parents = [] - while (content.firstChild && content.firstChild.from == content.from && content.firstChild.to == content.to) { - possible_parents.push(content.type) - content = content.firstChild - } - - if (content == null) { - console.log(`match:`, match) - throw new Error("No content match?") - } - - // Now we send just the `content` back to the template generator, which will happily accept it... - // (We do send the original from:to though, as these are the from:to's that are also in the template AST still) - let template_description = return_value( - template_generator.next({ - name: content.name, - node: content, - // Need to provide the original from:to range - from: template_ast.from, - to: template_ast.to, - }) - ) - - // And for some reason this works? - // Still feels like it shouldn't... it feels like I conjured some dark magic and I will be swiming in tartarus soon... - - return /** @type {Matcher} */ ({ - possible_parents, - template_description, - /** - * @param {TreeCursor | SyntaxNode} haystack_cursor - * @param {boolean} verbose? - * */ - match(haystack_cursor, verbose = false) { - if (haystack_cursor.type.isError) { - verbose && console.log(`❌ Short circuiting because haystack(${haystack_cursor.name}) is an error`) - return false - } - if ("cursor" in haystack_cursor) haystack_cursor = haystack_cursor.cursor() - - // Should possible parents be all-or-nothing? - // So either it matches all the possible parents, or it matches none? - let depth = 0 - for (let possible_parent of possible_parents) { - if (haystack_cursor.type === possible_parent) { - let parent_from = haystack_cursor.from - let parent_to = haystack_cursor.to - // Going in - if (haystack_cursor.firstChild()) { - if (haystack_cursor.from === parent_from && haystack_cursor.to === parent_to) { - verbose && console.log(`βœ… Matched parent, going one level deeper (${possible_parent})`) - depth++ - } else { - haystack_cursor.parent() - verbose && - console.log( - `❌ Was matching possible parent (${possible_parent}), but it wasn't filling?! That's weird.... ${haystack_cursor.toString()}` - ) - for (let i = 0; i < depth; i++) { - haystack_cursor.parent() - } - return false - } - } - } else { - break - } - } - - // prettier-ignore - verbose && console.groupCollapsed(`Starting a specific at match haystack(${haystack_cursor.name}) vs. template(${"node" in template_description ? template_description.name : template_description.pattern.name})`) - - try { - let matches = {} - return match_template(haystack_cursor, template_description, matches, verbose) ? matches : null - } finally { - // ARE FOR LOOPS REALLY THE BEST I CAN DO HERE?? - for (let i = 0; i < depth; i++) { - haystack_cursor.parent() - } - - verbose && console.groupEnd() - } - }, - }) - } else { - console.log(`meta_template:`, meta_template) - console.log(`template:`, template) - throw new Error("Template passed into `take_little_piece_of_template` doesn't match meta_template") - } -}) - -/** - * Sometimes nodes are nested at the exact same position: - * `struct X end`, here `X` could be both a `Definition(Identifier)` or just the `Identifier`. - * This function will get you the deepest node, so in the above example, it would be `Identifier`. - * If the node has multiple children, or the child is offset, it will return the original node. - * - * @param {SyntaxNode} node - * @returns {SyntaxNode} - **/ -export let narrow = (node) => { - if (node.firstChild && node.firstChild.from === node.from && node.firstChild.to === node.to) { - return narrow(node.firstChild) - } else { - return node - } -} - -/** - * Effecient, cursor-based, version of `narrow(node)`, - * for if all you care about is the name. - * - * Which will be most of the time.. - * - * @param {TreeCursor} cursor - * @return {string} - */ -export let narrow_name = (cursor) => { - let from = cursor.from - let to = cursor.to - if (cursor.firstChild()) { - try { - if (cursor.from === from && cursor.to === to) { - return narrow_name(cursor) - } - } finally { - cursor.parent() - } - } - return cursor.name -} - -/** - * This allows for selecting the unselectable! - * By default templates need to match the topnode of their AST, but sometimes we want to match something on a special position. - * - * ```create_specific_template_maker(x => jl_dynamic`import X: ${x}`)``` will match specifiers that could occur specifically on the `${x}` position. - * - * NOTE: Inside `create_specific_template_maker` you'll have to use `jl_dynamic` not going to explain why. - * - * @param {(subtemplate: Templatable) => Templatable} fn - */ -export let create_specific_template_maker = (fn) => { - return (argument) => { - let meta_template = template(fn(t.as("content", argument))) - return take_little_piece_of_template(fn(argument), meta_template) - } -} - /** * Like Lezers `iterate`, but instead of `{ from, to, getNode() }` * this will give `enter()` and `leave()` the `cursor` (which can be effeciently matches with lezer template) @@ -1057,41 +27,3 @@ export function iterate_with_cursor({ tree, enter, leave, from = 0, to = tree.le } } } - -/////////////////////////////////// -// FULL ON UTILITY FUNCTIONS -/////////////////////////////////// - -/** - * Return `iterater_result.value` if `iterater_result.done` is `false`, otherwise throw an error. - * - * Not sure why typescript doesn't infer the `Generator` when I ask !iterater_result.done... - * Maybe it will bite me later πŸ€·β€β™€οΈ - * - * @template T - * @param {IteratorResult} iterater_result - * @returns {T} iterater_result - */ -let intermediate_value = (iterater_result) => { - if (iterater_result.done) { - throw new Error("Expected `yield`-d value, but got `return`") - } else { - return /** @type {any} */ (iterater_result.value) - } -} - -/** - * Not sure why typescript doesn't infer the `Generator<_, T>` when I ask !iterater_result.done... - * Maybe it will bite me later πŸ€·β€β™€οΈ - * - * @template T - * @param {IteratorResult} iterater_result - * @returns {T} iterater_result - */ -let return_value = (iterater_result) => { - if (iterater_result.done) { - return /** @type {any} */ (iterater_result.value) - } else { - throw new Error("Expected `yield`-d value, but got `return`") - } -} diff --git a/frontend/components/CellInput/pkg_bubble_plugin.js b/frontend/components/CellInput/pkg_bubble_plugin.js index f38a5e9dee..63d48087ee 100644 --- a/frontend/components/CellInput/pkg_bubble_plugin.js +++ b/frontend/components/CellInput/pkg_bubble_plugin.js @@ -1,9 +1,9 @@ import _ from "../../imports/lodash.js" -import { EditorView, syntaxTree, Decoration, ViewUpdate, ViewPlugin, Facet } from "../../imports/CodemirrorPlutoSetup.js" +import { EditorView, syntaxTree, Decoration, ViewUpdate, ViewPlugin, Facet, EditorState } from "../../imports/CodemirrorPlutoSetup.js" import { PkgStatusMark, PkgActivateMark } from "../PkgStatusMark.js" import { html } from "../../imports/Preact.js" import { ReactWidget } from "./ReactWidget.js" -import { create_specific_template_maker, iterate_with_cursor, jl, jl_dynamic, narrow, t, template } from "./lezer_template.js" +import { iterate_with_cursor } from "./lezer_template.js" /** * @typedef PkgstatusmarkWidgetProps @@ -23,145 +23,78 @@ export const pkg_disablers = [ "@quickactivate", ] -function find_import_statements({ doc, tree, from, to }) { - // This quotelevel stuff is waaaay overengineered and precise... - // but I love making stuff like this SO LET ME OKAY - let quotelevel = 0 +/** + * @param {object} a + * @param {EditorState} a.state + * @param {Number} a.from + * @param {Number} a.to + */ +function find_import_statements({ state, from, to }) { + const doc = state.doc + const tree = syntaxTree(state) + let things_to_return = [] + let currently_using_or_import = "import" + let currently_selected_import = false + iterate_with_cursor({ tree, from, to, - enter: (cursor) => { - // `quote ... end` or `:(...)` - if (cursor.name === "QuoteExpression" || cursor.name === "QuoteStatement") { - quotelevel++ - } - // `$(...)` when inside quote - if (cursor.name === "InterpExpression") { - quotelevel-- - } - if (quotelevel !== 0) return + enter: (node) => { + let go_to_parent_afterwards = null - // Check for Pkg.activate() and friends - if (cursor.name === "CallExpression" || cursor.name === "MacroExpression") { - let node = cursor.node - let callee = node.firstChild - let callee_name = doc.sliceString(callee.from, callee.to) + if (node.name === "QuoteExpression" || node.name === "FunctionDefinition") return false - if (pkg_disablers.includes(callee_name)) { - things_to_return.push({ - type: "package_disabler", - name: callee_name, - from: cursor.to, - to: cursor.to, - }) - } + if (node.name === "import") currently_using_or_import = "import" + if (node.name === "using") currently_using_or_import = "using" - return - } + // console.group("exploring", node.name, doc.sliceString(node.from, node.to), node) - let import_specifier_template = create_specific_template_maker((x) => jl_dynamic`import A, ${x}`) - // Because the templates can't really do recursive stuff, we need JavaScriptℒ️! - let unwrap_scoped_import = (specifier) => { - let match = null - if ((match = import_specifier_template(jl`${t.as("package")}.${t.any}`).match(specifier))) { - return unwrap_scoped_import(match.package) - } else if ((match = import_specifier_template(jl`.${t.maybe(t.any)}`).match(specifier))) { - // Still trash! - return null - } else if ((match = import_specifier_template(jl`${t.Identifier}`).match(specifier))) { - return specifier - } else { - console.warn("Unknown nested import specifier: " + specifier.toString()) + if (node.name === "CallExpression" || node.name === "MacrocallExpression") { + let callee = node.node.firstChild + if (callee) { + let callee_name = doc.sliceString(callee.from, callee.to) + + if (pkg_disablers.includes(callee_name)) { + things_to_return.push({ + type: "package_disabler", + name: callee_name, + from: node.from, + to: node.to, + }) + } } + return false } - let match = null - if ( - // These templates might look funky... and they are! - // But they are necessary to force the matching to match as specific as possible. - // With just `import ${t.many("specifiers")}` it will match `import A, B, C`, but - // it will do so by giving back [`A, B, C`] as one big specifier! - (match = template(jl`import ${t.as("specifier")}: ${t.many()}`).match(cursor)) ?? - (match = template(jl`import ${t.as("specifier")}, ${t.many("specifiers")}`).match(cursor)) ?? - (match = template(jl`using ${t.as("specifier")}: ${t.many()}`).match(cursor)) ?? - (match = template(jl`using ${t.as("specifier")}, ${t.many("specifiers")}`).match(cursor)) - ) { - let { specifier, specifiers = [] } = match + if (node.name === "ImportStatement") { + currently_selected_import = false + } + if (node.name === "SelectedImport") { + currently_selected_import = true + node.firstChild() + go_to_parent_afterwards = true + } - if (specifier) { - specifiers = [{ node: specifier }, ...specifiers] + if (node.name === "ImportPath") { + const item = { + type: "package", + name: doc.sliceString(node.from, node.to), + from: node.from, + to: node.to, } - for (let { node: specifier } of specifiers) { - specifier = narrow(specifier) - - let match = null - if ((match = import_specifier_template(jl`${t.as("package")} as ${t.maybe(t.any)}`).match(specifier))) { - let node = unwrap_scoped_import(match.package) - if (node) { - things_to_return.push({ - type: "package", - name: doc.sliceString(node.from, node.to), - from: node.to, - to: node.to, - }) - } - } else if ((match = import_specifier_template(jl`${t.as("package")}.${t.any}`).match(specifier))) { - let node = unwrap_scoped_import(match.package) - if (node) { - things_to_return.push({ - type: "package", - name: doc.sliceString(node.from, node.to), - from: node.to, - to: node.to, - }) - } - } else if ((match = import_specifier_template(jl`.${t.as("scoped")}`).match(specifier))) { - // Trash! - } else if ((match = import_specifier_template(jl`${t.as("package")}`).match(specifier))) { - let node = unwrap_scoped_import(match.package) - if (node) { - things_to_return.push({ - type: "package", - name: doc.sliceString(node.from, node.to), - from: node.to, - to: node.to, - }) - } - } else { - console.warn("Unknown import specifier: " + specifier.toString()) - } - } + things_to_return.push(item) - match = null - if ((match = template(jl`using ${t.as("specifier")}, ${t.many("specifiers")}`).match(cursor))) { - let { specifier } = match - if (specifier) { - if (doc.sliceString(specifier.to, specifier.to + 1) === "\n" || doc.sliceString(specifier.to, specifier.to + 1) === "") { - things_to_return.push({ - type: "implicit_using", - name: doc.sliceString(specifier.from, specifier.to), - from: specifier.to, - to: specifier.to, - }) - } - } - } + // This is just for show... might delete it later + if (currently_using_or_import === "using" && !currently_selected_import) things_to_return.push({ ...item, type: "implicit_using" }) + } + if (go_to_parent_afterwards) { + node.parent() return false - } else if (cursor.name === "ImportStatement") { - throw new Error("What") - } - }, - leave: (cursor) => { - if (cursor.name === "QuoteExpression" || cursor.name === "QuoteStatement") { - quotelevel-- - } - if (cursor.name === "InterpExpression") { - quotelevel++ } }, }) @@ -179,8 +112,7 @@ function pkg_decorations(view, { pluto_actions, notebook_id, nbpkg }) { let widgets = view.visibleRanges .flatMap(({ from, to }) => { let things_to_mark = find_import_statements({ - doc: view.state.doc, - tree: syntaxTree(view.state), + state: view.state, from: from, to: to, }) @@ -240,7 +172,7 @@ function pkg_decorations(view, { pluto_actions, notebook_id, nbpkg }) { } /** - * @type {Facet} + * @type {Facet} */ export const NotebookpackagesFacet = Facet.define({ combine: (values) => values[0], @@ -277,6 +209,7 @@ export const pkgBubblePlugin = ({ pluto_actions, notebook_id_ref }) => } }, { + // @ts-ignore decorations: (v) => v.decorations, } ) diff --git a/frontend/components/CellInput/pluto_autocomplete.js b/frontend/components/CellInput/pluto_autocomplete.js index f44239d00e..12718ff4c9 100644 --- a/frontend/components/CellInput/pluto_autocomplete.js +++ b/frontend/components/CellInput/pluto_autocomplete.js @@ -312,7 +312,9 @@ const writing_variable_name_or_keyword = (/** @type {autocomplete.CompletionCont let inside_do_argument_expression = ctx.matchBefore(/do [\(\), \p{L}\p{Nl}\p{Sc}\d_!]*$/u) let node = syntaxTree(ctx.state).resolve(ctx.pos, -1) + // TODO: BareTupleExpression let node2 = node?.parent?.name === "BareTupleExpression" ? node?.parent : node + // TODO: AssignmentExpression let inside_assigment_lhs = node?.name === "Identifier" && node2?.parent?.name === "AssignmentExpression" && node2?.nextSibling != null return just_finished_a_keyword || after_keyword || inside_do_argument_expression || inside_assigment_lhs diff --git a/frontend/components/CellInput/scopestate_statefield.js b/frontend/components/CellInput/scopestate_statefield.js index 027d9035b7..19572bc53e 100644 --- a/frontend/components/CellInput/scopestate_statefield.js +++ b/frontend/components/CellInput/scopestate_statefield.js @@ -1,6 +1,7 @@ -import { syntaxTree, StateField } from "../../imports/CodemirrorPlutoSetup.js" +import { syntaxTree, StateField, NodeWeakMap, Text } from "../../imports/CodemirrorPlutoSetup.js" import _ from "../../imports/lodash.js" -import { child_cursors, child_nodes, create_specific_template_maker, jl, jl_dynamic, narrow, t, template } from "./lezer_template.js" + +const VERBOSE = false /** * @typedef TreeCursor @@ -29,1051 +30,331 @@ import { child_cursors, child_nodes, create_specific_template_maker, jl, jl_dyna * @property {Array<{ definition: Range, validity: Range, name: string }>} locals */ -/** - * @param {ScopeState} a - * @param {ScopeState} b - * @returns {ScopeState} - */ -let merge_scope_state = (a, b) => { - if (a === b) return a +const r = (cursor) => ({ from: cursor.from, to: cursor.to }) - let usages = [...a.usages, ...b.usages] - let definitions = new Map(a.definitions) - for (let [key, value] of b.definitions) { - definitions.set(key, value) +const find_local_definition = (locals, name, cursor) => { + for (let lo of locals) { + if (lo.name === name && cursor.from >= lo.validity.from && cursor.to <= lo.validity.to) { + return lo + } } - let locals = [...a.locals, ...b.locals] - return { usages, definitions, locals } } -/** @param {TreeCursor} cursor */ -let search_for_interpolations = function* (cursor) { - for (let child of child_cursors(cursor)) { - if (child.name === "InterpExpression") { - yield cursor - } else if (child.name === "QuoteExpression" || child.name === "QuoteStatement") { - for (let child_child of search_for_interpolations(child)) { - yield* search_for_interpolations(child_child) - } - } else { - yield* search_for_interpolations(child) +const HardScopeNames = new Set([ + "WhileStatement", + "ForStatement", + "TryStatement", + "LetStatement", + "FunctionDefinition", + "MacroDefinition", + "DoClause", + "Generator", +]) + +const does_this_create_scope = (/** @type {TreeCursor} */ cursor) => { + if (HardScopeNames.has(cursor.name)) return true + + if (cursor.name === "Assignment") { + const reset = cursor.firstChild() + try { + // f(x) = x + // @ts-ignore + if (cursor.name === "CallExpression") return true + } finally { + if (reset) cursor.parent() } } -} -/** @param {TreeCursor} cursor */ -let go_through_quoted_expression_looking_for_interpolations = function* (cursor) { - if (cursor.name !== "QuoteExpression" && cursor.name !== "QuoteStatement") throw new Error("Expected QuotedExpression or QuoteStatement") - yield* search_for_interpolations(cursor) + + return false } /** - * So this was a late addition, and it creates a bit crazy syntax... - * but I love it for that syntax! It really makes the patterns pop out, - * which it really needs, because the patterns are the most important part of this code.. - * @param {(subsitution: import("./lezer_template.js").Templatable) => import("./lezer_template.js").Matcher} template_fn + * Look into the left-hand side of an Assigment expression and find all ranges where variables are defined. + * E.g. `a, (b,c) = something` will return ranges for a, b, c. + * @param {TreeCursor} root_cursor + * @returns {Range[]} */ -let make_beautiful_matcher = (template_fn) => { - return function match(cursor, verbose = false) { - if (cursor == null) { - /** @type {(...args: Parameters) => any} */ - return (x, ...args) => { - return template_fn(jl(x, ...args)) - } +const explore_assignment_lhs = (root_cursor) => { + const a = cursor_not_moved_checker(root_cursor) + let found = [] + root_cursor.iterate((cursor) => { + if (cursor.name === "Identifier" || cursor.name === "MacroIdentifier" || cursor.name === "Operator") { + found.push(r(cursor)) } - - /** @type {(...args: Parameters) => any} */ - return function jl_and_match(x, ...args) { - return template_fn(jl(x, ...args)).match(cursor, verbose) + if (cursor.name === "IndexExpression" || cursor.name === "FieldExpression") { + // not defining a variable but modifying an object + return false } - } + }) + a() + return found } /** - * @param {Parameters[0]} template_creator + * Remember the position where this is called, and return a function that will move into parents until we are are back at that position. + * + * You can use this before exploring children of a cursor, and then go back when you are done. */ -let make_beautiful_specific_matcher = (template_creator) => { - let template_fn = create_specific_template_maker(template_creator) - return function match(cursor, verbose = false) { - if (cursor == null) { - /** @type {(...args: Parameters) => any} */ - return (x, ...args) => { - return template_fn(jl(x, ...args)) - } - } - - /** @type {(...args: Parameters) => any} */ - return function jl_and_match(x, ...args) { - return template_fn(jl(x, ...args)).match(cursor, verbose) +const back_to_parent_resetter = (/** @type {TreeCursor} */ cursor) => { + const map = new NodeWeakMap() + map.cursorSet(cursor, "here") + return () => { + while (map.cursorGet(cursor) !== "here") { + if (!cursor.parent()) throw new Error("Could not find my back to the original parent!") } } } -let match_for_binding = make_beautiful_specific_matcher((x) => jl_dynamic`[i for i in i ${x}]`) -let match_assignee = make_beautiful_specific_matcher((x) => jl_dynamic`${x} = nothing`) -let match_function_definition_argument = make_beautiful_specific_matcher((x) => jl_dynamic`function f(${x}) end`) -let match_function_call_argument = make_beautiful_specific_matcher((x) => jl_dynamic`f(${x})`) -let match_function_call_named_argument = make_beautiful_specific_matcher((x) => jl_dynamic`f(; ${x})`) +const cursor_not_moved_checker = (cursor) => { + const map = new NodeWeakMap() + map.cursorSet(cursor, "yay") -/** - * @param {TreeCursor | SyntaxNode} cursor - * @param {any} doc - * @param {ScopeState} scopestate - * @param {boolean} [verbose] - * @returns {ScopeState} - */ -let explorer_function_definition_argument = (cursor, doc, scopestate, verbose = false) => { - let match = null + const debug = (cursor) => `${cursor.name}(${cursor.from},${cursor.to})` - if ((match = match_function_call_argument(cursor)`; ${t.many("named_args")}`)) { - // "Parameters", the `y, z` in `function f(x; y, z) end` - let { named_args = [] } = match - for (let { node: named_arg } of named_args) { - scopestate = explorer_function_definition_argument(named_arg, doc, scopestate, verbose) - } - return scopestate - } else if ((match = match_function_definition_argument(cursor)`${t.Identifier}`)) { - return scopestate_add_definition(scopestate, doc, cursor) - } else if ((match = match_function_definition_argument(cursor)`${t.as("subject")}...`)) { - // `function f(x...)` => ["x"] - return explore_pattern(match.subject, doc, scopestate, null, verbose) - } else if ((match = match_function_definition_argument(cursor)`${t.as("name")} = ${t.as("value")}`)) { - // `function f(x = 10)` => ["x"] - let { name, value } = match - scopestate = explore_pattern(name, doc, scopestate, value.to, verbose) - scopestate = explore_variable_usage(value.cursor(), doc, scopestate, verbose) - return scopestate - } else if ( - (match = match_function_definition_argument(cursor)`${t.as("name")}::${t.as("type")}`) ?? - (match = match_function_definition_argument(cursor)`${t.as("name")}:`) ?? - // (match = match_function_definition_argument(cursor)`${t.as("name")}::`) ?? - (match = match_function_definition_argument(cursor)`::${t.as("type")}`) - ) { - let { name, type } = match - if (name) scopestate = explore_pattern(name, doc, scopestate, type.to, verbose) - if (type) scopestate = explore_variable_usage(type.cursor(), doc, scopestate, verbose) - return scopestate - } else { - // Fall back to "just explore pattern"... - // There is more overlap between function arguments and patterns than I use now, I think - scopestate = explore_pattern(cursor, doc, scopestate) + const debug_before = debug(cursor) - verbose && console.warn("UNKNOWN FUNCTION DEFINITION ARGUMENT:", cursor.toString()) - return scopestate + return () => { + if (map.cursorGet(cursor) !== "yay") { + throw new Error(`Cursor changed position when forbidden! Before: ${debug_before}, after: ${debug(cursor)}`) + } } } -/** - * @param {TreeCursor | SyntaxNode} node - * @param {any} doc - * @param {ScopeState} scopestate - * @param {number?} valid_from - * @param {boolean} [verbose] - * @returns {ScopeState} - */ -let explore_pattern = (node, doc, scopestate, valid_from = null, verbose = false) => { - let match = null - - verbose && console.group("Explorering pattern:", node.toString()) - try { - if ((match = match_assignee(node)`${t.Identifier}`)) { - verbose && console.log("It's an identifier, adding it to the scope") - return scopestate_add_definition(scopestate, doc, node, valid_from) - } else if ((match = match_assignee(node)`${t.as("object")}::${t.as("type")}`)) { - let { object, type } = match - scopestate = explore_variable_usage(type.cursor(), doc, scopestate, verbose) - scopestate = scopestate_add_definition(scopestate, doc, object) - return scopestate - } else if ((match = match_assignee(node)`${t.as("subject")}...`)) { - // `x... = [1,2,3]` => ["x"] - return explore_pattern(match.subject, doc, scopestate, valid_from, verbose) - } else if ((match = match_function_definition_argument(node)`${t.as("name")} = ${t.as("value")}`)) { - let { name, value } = match - scopestate = explore_pattern(name, doc, scopestate, value.from, verbose) - scopestate = explore_variable_usage(value.cursor(), doc, scopestate, verbose) - return scopestate - } else if ((match = match_assignee(node)`(; ${t.many("named_tuples")})`)) { - // `(; x, y) = z` => ["x", "y"] - let { named_tuples } = match - for (let name of named_tuples) { - scopestate = explore_pattern(name.node.cursor(), doc, scopestate, valid_from, verbose) - } - return scopestate - } else if ( - (match = match_assignee(node)`${t.as("first")}, ${t.many("rest")}`) ?? - (match = match_assignee(node)`(${t.as("first")}, ${t.many("rest")})`) - ) { - // console.warn("Tuple assignment... but the bad one") - for (let { node: name } of [{ node: match.first }, ...(match.rest ?? [])]) { - scopestate = explore_pattern(name.cursor(), doc, scopestate, valid_from, verbose) - } - return scopestate - } else if ((match = match_julia(node)`${t.as("prefix")}${t.as("string", t.String)}`)) { - // This one is also a bit enigmatic, but `t.String` renders in the template as `"..."`, - // so the template with match things that look like `prefix"..."` - let { prefix, string } = match - let prefix_string = doc.sliceString(prefix.from, prefix.to) - - if (prefix_string === "var") { - let name = doc.sliceString(string.from + 1, string.to - 1) - if (name.length !== 0) { - scopestate.definitions.set(name, { - from: node.from, - to: node.to, - valid_from: node.to, - }) - } - } else { - scopestate = explore_variable_usage("cursor" in node ? node.cursor() : node, doc, scopestate, verbose) - } - return scopestate - } else if ((match = match_assignee(node)`${t.as("object")}[${t.as("property")}]`)) { - let { object, property } = match - scopestate = explore_variable_usage(object.cursor(), doc, scopestate, verbose) - if (property) scopestate = explore_variable_usage(property.cursor(), doc, scopestate, verbose) - return scopestate - } else if ((match = match_assignee(node)`${t.as("object")}.${t.as("property")}`)) { - let { object, property } = match - scopestate = explore_variable_usage(object.cursor(), doc, scopestate, verbose) - return scopestate - } else { - verbose && console.warn("UNKNOWN PATTERN:", node.toString(), doc.sliceString(node.from, node.to)) - return scopestate +const i_am_nth_child = (cursor) => { + const map = new NodeWeakMap() + map.cursorSet(cursor, "here") + if (!cursor.parent()) throw new Error("Cannot be toplevel") + cursor.firstChild() + let i = 0 + while (map.cursorGet(cursor) !== "here") { + i++ + if (!cursor.nextSibling()) { + throw new Error("Could not find my way back") } - } finally { - verbose && console.groupEnd() } + return i } /** - * Explores the definition part of a struct or such. - * Takes care of defining that actual name, defining type parameters, - * and using all the types used. - * - * It returns an inner and an outer scope state. - * The inner scope state is for code "inside" the struct, which has access to the implicitly created types. - * Outer scope is only the actual defined name, so will always exist of just one definition and no usages. - * This distinction is so the created types don't escape outside the struct. - * Usages all go in the inner scope. - * * @param {TreeCursor} cursor - * @param {any} doc - * @param {ScopeState} scopestate - * @param {boolean} [verbose] - * @returns {{ inner: ScopeState, outer: ScopeState }} + * @returns {Range[]} */ -let explore_definition = function (cursor, doc, scopestate, verbose = false) { - let match = null +const explore_funcdef_arguments = (cursor, { enter, leave }) => { + let found = [] - if (cursor.name === "Definition" && cursor.firstChild()) { - try { - return explore_definition(cursor, doc, scopestate) - } finally { - cursor.parent() - } - } else if (cursor.name === "Identifier") { - return { - inner: scopestate_add_definition(scopestate, doc, cursor), - outer: scopestate_add_definition(fresh_scope(), doc, cursor), - } - } else if ((match = match_julia(cursor)`${t.as("subject")}{ ${t.many("parameters")} }`)) { - // A{B} - let { subject, parameters } = match - let outer = fresh_scope() - if (subject) { - let subject_explored = explore_definition(subject.cursor(), doc, scopestate) - outer = subject_explored.outer - scopestate = subject_explored.inner - } - for (let { node: parameter } of parameters) { - // Yes, when there is a type parameter in the definition itself (so not after `::`), - // it implies a new type parameter being implicitly instanciated. - let { inner: parameter_inner } = explore_definition(parameter.cursor(), doc, scopestate) - scopestate = parameter_inner + const position_validation = cursor_not_moved_checker(cursor) + const position_resetter = back_to_parent_resetter(cursor) + + if (!cursor.firstChild()) throw new Error(`Expected to go into function definition argument expression, stuck at ${cursor.name}`) + // should be in the TupleExpression now + + // @ts-ignore + VERBOSE && console.assert(cursor.name === "TupleExpression" || cursor, name === "Arguments", cursor.name) + + cursor.firstChild() + do { + if (cursor.name === "KeywordArguments") { + cursor.firstChild() // go into kwarg arguments } - return { inner: scopestate, outer: outer } - } else if ((match = match_julia(cursor)`${t.as("subject")} <: ${t.maybe(t.as("type"))}`)) { - let { subject, type } = match - let outer = fresh_scope() - if (subject) ({ outer, inner: scopestate } = explore_definition(subject.cursor(), doc, scopestate)) - if (type) scopestate = explore_variable_usage(type.cursor(), doc, scopestate) - return { inner: scopestate, outer: outer } - } else { - verbose && console.warn(`Unknown thing in definition: "${doc.sliceString(cursor.from, cursor.to)}", "${cursor.toString()}"`) - return { inner: scopestate, outer: fresh_scope() } - } -} -let match_julia = make_beautiful_matcher(template) + if (cursor.name === "Identifier" || cursor.name === "Operator") { + found.push(r(cursor)) + } else if (cursor.name === "KwArg") { + let went_in = cursor.firstChild() + found.push(r(cursor)) + // cursor.nextSibling() + // find stuff used here + // cursor.iterate(enter, leave) -/** - * @param {TreeCursor} cursor - * @param {any} doc - * @param {ScopeState} scopestate - * @param {boolean} [verbose] - * @returns {ScopeState} - */ -let explore_macro_identifier = (cursor, doc, scopestate, verbose = false) => { - let match = null + if (went_in) cursor.parent() + } + } while (cursor.nextSibling()) - let match_macro_identifier = make_beautiful_specific_matcher((x) => jl_dynamic`${x} x y z`) + position_resetter() + position_validation() - if ((match = match_macro_identifier(cursor)`${t.as("macro", jl`@${t.any}`)}`)) { - let { macro } = match - let name = doc.sliceString(macro.from, macro.to) - scopestate.usages.push({ - usage: macro, - definition: scopestate.definitions.get(name) ?? null, - name: name, - }) - return scopestate - } else if ((match = match_macro_identifier(cursor)`${t.as("object")}.@${t.as("macro")}`)) { - let { object } = match - let name = doc.sliceString(object.from, object.to) - scopestate.usages.push({ - usage: object, - definition: scopestate.definitions.get(name) ?? null, - name: name, - }) - return scopestate - } else if ((match = match_macro_identifier(cursor)`@${t.as("object")}.${t.as("macro")}`)) { - let { object } = match - let name = doc.sliceString(object.from, object.to) - scopestate.usages.push({ - usage: object, - definition: scopestate.definitions.get(name) ?? null, - name: name, - }) - return scopestate - } else { - verbose && console.warn("Mwep mweeeep", cursor.toString()) - return scopestate - } + VERBOSE && console.log({ found }) + return found } /** + * @param {TreeCursor | SyntaxNode} tree + * @param {Text} doc + * @param {any} _scopestate + * @param {boolean} [verbose] * @returns {ScopeState} */ -let fresh_scope = () => { - return { - usages: [], - definitions: new Map(), - locals: [], +export let explore_variable_usage = (tree, doc, _scopestate, verbose = VERBOSE) => { + if ("cursor" in tree) { + console.trace("`explore_variable_usage()` called with a SyntaxNode, not a TreeCursor") + tree = tree.cursor() } -} -/** - * Currently this clones a scope state, except for the usages. - * The reason is skips the usages is a premature optimisation. - * We don't need them in the inner scope, but we just as well could leave them on - * (as they won't do any harm either way) - * - * @param {ScopeState} scopestate - * @returns {ScopeState} - */ -let lower_scope = (scopestate) => { - return { + const scopestate = { usages: [], - definitions: new Map(scopestate.definitions), + definitions: new Map(), locals: [], } -} -/** - * For use in combination with `lower_scope`. - * This will take an inner scope and merge only the usages into the outer scope. - * So we see the usages of the inner scope, but the definitions don't exist in the outer scope. - * - * @param {ScopeState} nested_scope - * @param {ScopeState} scopestate - * @param {number} [nested_scope_validity] - * @returns {ScopeState} - */ -let raise_scope = (nested_scope, scopestate, nested_scope_validity = undefined) => { - return { - usages: [...scopestate.usages, ...nested_scope.usages], - definitions: scopestate.definitions, - locals: [ - // TODO: Disabled because of performance problems, see https://github.com/fonsp/Pluto.jl/pull/1925 - // ...(nested_scope_validity === null - // ? [] - // : Array.from(nested_scope.definitions) - // .filter(([name, _]) => !scopestate.definitions.has(name)) - // .map(([name, definition]) => ({ - // name, - // definition, - // validity: { - // from: definition.valid_from, - // to: nested_scope_validity, - // }, - // }))), - // ...nested_scope.locals, - // ...scopestate.locals, - ], - } -} + let local_scope_stack = /** @type {Range[]} */ ([]) -/** - * @param {ScopeState} scopestate - * @param {any} doc - * @param {SyntaxNode | TreeCursor} node - * @param {number?} valid_from - */ -let scopestate_add_definition = (scopestate, doc, node, valid_from = null) => { - valid_from = valid_from === null ? node.to : valid_from - scopestate.definitions.set(doc.sliceString(node.from, node.to), { - from: node.from, - to: node.to, - valid_from, - }) - return scopestate -} + const definitions = /** @type {Map} */ new Map() + const locals = /** @type {Array<{ definition: Range, validity: Range, name: string }>} */ ([]) + const usages = /** @type {Array<{ usage: Range, definition: Range | null, name: string }>} */ ([]) -/** - * @param {TreeCursor | SyntaxNode} cursor - * @param {any} doc - * @param {ScopeState} scopestate - * @param {boolean} [verbose] - * @returns {ScopeState} - */ -export let explore_variable_usage = ( - cursor, - doc, - scopestate = { - usages: [], - definitions: new Map(), - locals: [], - }, - verbose = false -) => { - if ("cursor" in cursor) { - // console.trace("`explore_variable_usage()` called with a SyntaxNode, not a TreeCursor") - cursor = cursor.cursor() - } + const return_false_immediately = new NodeWeakMap() - let start_node = null - if (verbose) { - console.group(`Explorer: ${cursor.name}`) + let enter, leave - console.groupCollapsed("Details") - try { - console.log(`Full tree: ${cursor.toString()}`) - console.log("Full text:", doc.sliceString(cursor.from, cursor.to)) - console.log(`scopestate:`, scopestate) - } finally { - console.groupEnd() + enter = (/** @type {TreeCursor} */ cursor) => { + if (verbose) { + console.group(`Explorer: ${cursor.name}`) + + console.groupCollapsed("Details") + try { + console.log(`Full tree: ${cursor.toString()}`) + console.log("Full text:", doc.sliceString(cursor.from, cursor.to)) + console.log(`scopestate:`, scopestate) + } finally { + console.groupEnd() + } } - start_node = cursor.node - } - try { - let match = null - // Doing these checks in front seems to speed things up a bit. if ( - cursor.type.is("keyword") || - cursor.name === "Program" || - cursor.name === "BoolLiteral" || - cursor.name === "CharLiteral" || - cursor.name === "String" || - cursor.name === "IntegerLiteral" || - cursor.name === "FloatLiteral" || - cursor.name === "Comment" || - cursor.name === "BinaryExpression" || - cursor.name === "Operator" || - cursor.name === "MacroArgumentList" || - cursor.name === "ReturnStatement" || - cursor.name === "OpenTuple" || - cursor.name === "ParenExpression" || - cursor.name === "Type" || - cursor.name === "InterpExpression" || - cursor.name === "SpreadExpression" || - cursor.name === "CompoundExpression" || - cursor.name === "ParameterizedIdentifier" || - cursor.name === "BraceExpression" || - cursor.name === "TernaryExpression" || - cursor.name === "Coefficient" || - cursor.name === "TripleString" || - cursor.name === "RangeExpression" || - cursor.name === "IndexExpression" || - cursor.name === "UnaryExpression" || - cursor.name === "ConstStatement" || - cursor.name === "GlobalStatement" || - cursor.name === "ContinueStatement" || - cursor.name === "MatrixExpression" || - cursor.name === "MatrixRow" || - cursor.name === "VectorExpression" + return_false_immediately.cursorGet(cursor) || + cursor.name === "ModuleDefinition" || + cursor.name === "QuoteStatement" || + cursor.name === "QuoteExpression" || + cursor.name === "MacroIdentifier" || + cursor.name === "ImportStatement" ) { - for (let subcursor of child_cursors(cursor)) { - scopestate = explore_variable_usage(subcursor, doc, scopestate, verbose) - } - return scopestate + if (verbose) console.groupEnd() + return false + } + + const register_variable = (range) => { + const name = doc.sliceString(range.from, range.to) + + if (local_scope_stack.length === 0) + definitions.set(name, { + ...range, + valid_from: range.from, + }) + else locals.push({ name, validity: _.last(local_scope_stack), definition: range }) + } + + if (does_this_create_scope(cursor)) { + local_scope_stack.push(r(cursor)) } - if (cursor.name === "Identifier" || cursor.name === "MacroIdentifier") { - let name = doc.sliceString(cursor.from, cursor.to) - scopestate.usages.push({ + if (cursor.name === "Identifier" || cursor.name === "MacroIdentifier" || cursor.name === "Operator") { + const name = doc.sliceString(cursor.from, cursor.to) + usages.push({ name: name, usage: { from: cursor.from, to: cursor.to, }, - definition: scopestate.definitions.get(name) ?? null, - }) - return scopestate - } else if ((match = match_julia(cursor)`:${t.any}`)) { - // Nothing, ha! - return scopestate - } else if ((match = match_julia(cursor)`${t.Number}`)) { - // Nothing, ha! - return scopestate - } else if ((match = match_julia(cursor)`${t.String}`)) { - // Nothing, ha! - return scopestate - } else if ((match = match_julia(cursor)`${t.as("object")}.${t.as("property")}`)) { - let { object, property } = match - if (object) scopestate = explore_variable_usage(object.cursor(), doc, scopestate, verbose) - return scopestate - } else if ((match = match_julia(cursor)`${t.as("assignee")} = ${t.maybe(t.as("value"))}`)) { - let { assignee, value } = match - if (value) scopestate = explore_variable_usage(value.cursor(), doc, scopestate, verbose) - if (assignee) scopestate = explore_pattern(assignee.cursor(), doc, scopestate, value?.to ?? null, verbose) - return scopestate - } else if ( - (match = match_julia(cursor)` - ${t.as("macro", t.anything_that_fits(jl`@macro`))}(${t.many("args")}) ${t.maybe(jl`do ${t.maybe(t.many("do_args"))} - ${t.many("do_expressions")} - end`)}} - `) - ) { - let { macro, args = [], do_args, do_expressions } = match - if (macro) explore_macro_identifier(macro.cursor(), doc, scopestate, verbose) - - for (let { node: arg } of args) { - if ((match = match_function_call_argument(arg)`${t.as("name")} = ${t.as("value")}`)) { - let { name, value } = match - if (value) scopestate = explore_variable_usage(value.cursor(), doc, scopestate, verbose) - } else { - scopestate = explore_variable_usage(arg.cursor(), doc, scopestate, verbose) - } - } - - if (do_args && do_expressions) { - // Cheating because lezer-julia isn't up to this task yet - // TODO julia-lezer is up to the task now!! - let inner_scope = lower_scope(scopestate) - - // Don't ask me why, but currently `do (x, y)` is parsed as `DoClauseArguments(ArgumentList(x, y))` - // while an actual argumentslist, `do x, y` is parsed as `DoClauseArguments(BareTupleExpression(x, y))` - let do_args_actually = do_args.firstChild - if (do_args_actually?.name === "Identifier") { - inner_scope = scopestate_add_definition(inner_scope, doc, do_args_actually) - } else if (do_args_actually?.name === "ArgumentList") { - for (let child of child_nodes(do_args_actually)) { - inner_scope = explorer_function_definition_argument(child, doc, inner_scope) - } - } else if (do_args_actually?.name === "BareTupleExpression") { - for (let child of child_nodes(do_args_actually)) { - inner_scope = explorer_function_definition_argument(child, doc, inner_scope) - } - } else { - verbose && console.warn("Unrecognized do args", do_args_actually.toString()) - } - - for (let { node: expression } of do_expressions) { - inner_scope = explore_variable_usage(expression.cursor(), doc, inner_scope, verbose) - } - return raise_scope(inner_scope, scopestate, cursor.to) - } - - return scopestate - } else if ((match = match_julia(cursor)`${t.as("macro", t.anything_that_fits(jl`@macro`))} ${t.many("args")}`)) { - let { macro, args = [] } = match - if (macro) explore_macro_identifier(macro.cursor(), doc, scopestate, verbose) - - for (let { node: arg } of args) { - scopestate = explore_variable_usage(arg.cursor(), doc, scopestate, verbose) - } - return scopestate - } else if ( - (match = match_julia(cursor)` - struct ${t.as("defined_as")} - ${t.many("expressions")} - end - `) ?? - (match = match_julia(cursor)` - mutable struct ${t.as("defined_as")} - ${t.many("expressions")} - end - `) - ) { - let { defined_as, expressions = [] } = match - defined_as = narrow(defined_as) - - let inner_scope = lower_scope(scopestate) - let outer_scope = fresh_scope() - - if (defined_as) ({ inner: inner_scope, outer: outer_scope } = explore_definition(defined_as.cursor(), doc, inner_scope)) - - // Struct body - for (let { node: expression } of expressions) { - if (cursor.name === "Identifier") { - // Nothing, this is just the name inside the struct blegh get it out of here - } else if ((match = match_julia(expression)`${t.as("subject")}::${t.as("type")}`)) { - // We're in X::Y, and Y is a reference - let { subject, type } = match - inner_scope = explore_variable_usage(type.cursor(), doc, inner_scope, verbose) - } else if ((match = match_julia(expression)`${t.as("assignee")} = ${t.as("value")}`)) { - let { assignee, value } = match - - // Yeah... we do the same `a::G` check again - if ((match = match_julia(assignee)`${t.as("subject")}::${t.as("type")}`)) { - let { subject, type } = match - inner_scope = explore_variable_usage(type.cursor(), doc, inner_scope, verbose) - } - inner_scope = explore_variable_usage(value.cursor(), doc, inner_scope, verbose) - } - } - - scopestate = raise_scope(inner_scope, scopestate, cursor.to) - scopestate = merge_scope_state(scopestate, outer_scope) - return scopestate - } else if ((match = match_julia(cursor)`abstract type ${t.as("name")} end`)) { - let { name } = match - if (name) { - let { outer } = explore_definition(name.cursor(), doc, scopestate) - scopestate = merge_scope_state(scopestate, outer) - } - return scopestate - } else if ((match = match_julia(cursor)`quote ${t.many("body")} end`) ?? (match = match_julia(cursor)`:(${t.many("body")})`)) { - // We don't use the match because I made `go_through_quoted_expression_looking_for_interpolations` - // to take a cursor at the quoted expression itself - for (let interpolation_cursor of go_through_quoted_expression_looking_for_interpolations(cursor)) { - scopestate = explore_variable_usage(interpolation_cursor, doc, scopestate, verbose) - } - return scopestate - } else if ( - (match = match_julia(cursor)` - module ${t.as("name")} - ${t.many("expressions")} - end - `) - ) { - let { name, expressions = [] } = match - - if (name) scopestate = scopestate_add_definition(scopestate, doc, name) - - let module_scope = fresh_scope() - for (let { node: expression } of expressions) { - module_scope = explore_variable_usage(expression.cursor(), doc, module_scope) - } - // We still merge the module scopestate with the global scopestate, but only the usages that don't escape. - // (Later we can have also shadowed definitions for the dimming of unused variables) - scopestate = merge_scope_state(scopestate, { - usages: Array.from(module_scope.usages).filter((x) => x.definition != null), - definitions: new Map(), - locals: [], + definition: find_local_definition(locals, name, cursor) ?? null, }) - - for (let { node: expression } of expressions) { - scopestate = explore_variable_usage(expression.cursor(), doc, scopestate) - } - return scopestate - } else if ((match = match_julia(cursor)`${t.as("prefix")}${t.as("string", t.String)}`)) { - // This one is also a bit enigmatic, but `t.String` renders in the template as `"..."`, - // so the template with match things that look like `prefix"..."` - let { prefix, string } = match - let prefix_string = doc.sliceString(prefix.from, prefix.to) - - if (prefix_string === "var") { - let name = doc.sliceString(string.from + 1, string.to - 1) - if (name.length !== 0) { - scopestate.usages.push({ - name: name, - usage: { - from: cursor.from, - to: cursor.to, - }, - definition: scopestate.definitions.get(name) ?? null, - }) - } - return scopestate - } else { - let name = `@${prefix_string}_str` - scopestate.usages.push({ - name: name, - usage: { - from: prefix.from, - to: prefix.to, - }, - definition: scopestate.definitions.get(name) ?? null, - }) - } - return scopestate - } else if ((match = match_julia(cursor)`${t.Number}${t.as("unit")}`)) { - // This isn't that useful, just wanted to test (and show off) the template - return explore_variable_usage(match.unit.cursor(), doc, scopestate, verbose) - } else if ( - (match = match_julia(cursor)`import ${t.any}: ${t.many("specifiers")}`) ?? - (match = match_julia(cursor)`using ${t.any}: ${t.many("specifiers")}`) - ) { - let { specifiers = [] } = match - let match_selected_import_specifier = make_beautiful_specific_matcher((x) => jl_dynamic`import X: ${x}`) - - for (let { node: specifier } of specifiers) { - if ((match = match_selected_import_specifier(specifier)`${t.as("name")} as ${t.as("alias")}`)) { - let { alias } = match - scopestate = scopestate_add_definition(scopestate, doc, alias) - } else if ((match = match_selected_import_specifier(specifier)`${t.as("name", t.Identifier)}`)) { - let { name } = match - scopestate = scopestate_add_definition(scopestate, doc, name) - } else if ((match = match_selected_import_specifier(specifier)`@${t.any}`)) { - scopestate = scopestate_add_definition(scopestate, doc, specifier) - } else { - verbose && console.warn("Hmmmm, I don't know what to do with this selected import specifier:", specifier.toString()) - } - } - return scopestate - } else if ((match = match_julia(cursor)`import ${t.many("specifiers")}`)) { - let { specifiers = [] } = match - - let match_import_specifier = make_beautiful_specific_matcher((x) => jl_dynamic`import ${x}`) - - for (let { node: specifier } of specifiers) { - if ((match = match_import_specifier(specifier)`${t.any} as ${t.as("alias")}`)) { - let { alias } = match - scopestate = scopestate_add_definition(scopestate, doc, alias) - } else if ((match = match_import_specifier(specifier)`${t.as("package")}.${t.as("name", t.Identifier)}`)) { - scopestate = scopestate_add_definition(scopestate, doc, match.name) - } else if ((match = match_import_specifier(specifier)`.${t.as("scoped")}`)) { - let scopedmatch = null - while ((scopedmatch = match_import_specifier(match.scoped)`.${t.as("scoped")}`)) { - match = scopedmatch - } - scopestate = scopestate_add_definition(scopestate, doc, match.scoped) - } else if ((match = match_import_specifier(specifier)`${t.as("name", t.Identifier)}`)) { - scopestate = scopestate_add_definition(scopestate, doc, match.name) - } else { - verbose && console.warn("Hmmm, I don't know what to do with this import specifier:", specifier) - } - } - return scopestate - } else if ((match = match_julia(cursor)`using ${t.many()}`)) { - // Can't care less - return scopestate - } else if ( - (match = match_julia(cursor)` - for ${t.many("bindings", t.something_with_the_same_type_as(jl`x in y`))}; - ${t.many("expressions")} - end - `) - ) { - let for_loop_binding_template = create_specific_template_maker((arg) => jl_dynamic`for ${arg}; x end`) - let for_loop_binding_match_julia = - (cursor) => - (...args) => { - // @ts-ignore - return for_loop_binding_template(jl(...args)).match(cursor) + } else if (cursor.name === "Assignment" || cursor.name === "KwArg" || cursor.name === "ForBinding" || cursor.name === "CatchClause") { + if (cursor.firstChild()) { + // @ts-ignore + if (cursor.name === "catch") cursor.nextSibling() + // CallExpression means function definition `f(x) = x`, this is handled elsewhere + // @ts-ignore + if (cursor.name !== "CallExpression") { + explore_assignment_lhs(cursor).forEach(register_variable) + // mark this one as finished + return_false_immediately.cursorSet(cursor, true) } - - let { bindings, expressions } = match - let inner_scope = lower_scope(scopestate) - - for (let { node: binding } of bindings) { - let match = null - if ( - (match = for_loop_binding_match_julia(binding)`${t.as("name")} in ${t.as("range")}`) ?? - (match = for_loop_binding_match_julia(binding)`${t.as("name")} ∈ ${t.as("range")}`) ?? - (match = for_loop_binding_match_julia(binding)`${t.as("name")} = ${t.as("range")}`) - ) { - let { name, range } = match - if (range) inner_scope = explore_variable_usage(range.cursor(), doc, inner_scope, verbose) - if (name) inner_scope = explore_pattern(name, doc, inner_scope, range?.to ?? null, verbose) - } else { - verbose && console.warn("Unrecognized for loop binding", binding.toString()) + cursor.parent() + } + } else if (cursor.name === "Parameters") { + explore_assignment_lhs(cursor).forEach(register_variable) + if (verbose) console.groupEnd() + return false + } else if (cursor.name === "Field") { + if (verbose) console.groupEnd() + return false + } else if (cursor.name === "CallExpression") { + if (cursor.matchContext(["FunctionDefinition", "Signature"]) || (cursor.matchContext(["Assignment"]) && i_am_nth_child(cursor) === 0)) { + const pos_resetter = back_to_parent_resetter(cursor) + + cursor.firstChild() // CallExpression now + cursor.firstChild() + // @ts-ignore + if (cursor.name === "Identifier" || cursor.name === "Operator") { + if (verbose) console.log("found function name", doc.sliceString(cursor.from, cursor.to)) + + const last_scoper = local_scope_stack.pop() + register_variable(r(cursor)) + if (last_scoper) local_scope_stack.push(last_scoper) + + cursor.nextSibling() } - } - - for (let { node: expression } of expressions) { - inner_scope = explore_variable_usage(expression.cursor(), doc, inner_scope, verbose) - } - - return raise_scope(inner_scope, scopestate, cursor.to) - } else if ( - (match = match_julia(cursor)` - ${t.as("callee")}(${t.many("args")}) ${t.maybe(jl`do ${t.maybe(t.many("do_args"))} - ${t.many("do_expressions")} - end`)} - `) ?? - (match = match_julia(cursor)` - ${t.as("callee")}.(${t.many("args")}) - `) - ) { - let { callee, args = [], do_args = [], do_expressions = [] } = match - - scopestate = explore_variable_usage(callee.cursor(), doc, scopestate, verbose) - - for (let { node: arg } of args) { - let match = null - if ((match = match_function_call_argument(arg)`; ${t.many("named_args")}`)) { - // "Parameters", the part in `f(x; y, z)` after the `;` - let { named_args = [] } = match - for (let { node: named_arg } of named_args) { - let match = null - if ((match = match_function_call_named_argument(named_arg)`${t.as("name")} = ${t.as("value")}`)) { - let { name, value } = match - scopestate = explore_variable_usage(value.cursor(), doc, scopestate, verbose) - } else { - scopestate = explore_variable_usage(named_arg.cursor(), doc, scopestate, verbose) - } - } - } else if ((match = match_function_call_argument(arg)`${t.as("name")} = ${t.as("value")}`)) { - let { name, value } = match - if (value) scopestate = explore_variable_usage(value.cursor(), doc, scopestate, verbose) - } else if ((match = match_function_call_argument(arg)`${t.as("result")} ${t.many("clauses", t.anything_that_fits(jl`for x = y`))}`)) { - let { result, clauses } = match - let nested_scope = lower_scope(scopestate) - // Because of the `t.anything_that_fits`, we can now match different `for x ? y`'s and `if x`s manually. - // There might be a way to express this in the template, but this keeps templates a lot simpler yet powerful. - for (let { node: for_binding } of clauses) { - let match = null - if ( - (match = match_for_binding(for_binding)`for ${t.as("variable")} = ${t.maybe(t.as("value"))}`) ?? - (match = match_for_binding(for_binding)`for ${t.as("variable")} in ${t.maybe(t.as("value"))}`) ?? - (match = match_for_binding(for_binding)`for ${t.as("variable")} ∈ ${t.maybe(t.as("value"))}`) ?? - (match = match_for_binding(for_binding)`for ${t.as("variable")}`) - ) { - let { variable, value } = match - - if (value) nested_scope = explore_variable_usage(value.cursor(), doc, nested_scope, verbose) - if (variable) nested_scope = explore_pattern(variable, doc, nested_scope) - } else if ((match = match_for_binding(for_binding)`if ${t.maybe(t.as("if"))}`)) { - let { if: node } = match - if (node) nested_scope = explore_variable_usage(node.cursor(), doc, nested_scope, verbose) - } else { - verbose && console.log("Hmmm, can't parse for binding", for_binding) - } - } + if (verbose) console.log("expl funcdef ", doc.sliceString(cursor.from, cursor.to)) + explore_funcdef_arguments(cursor, { enter, leave }).forEach(register_variable) + if (verbose) console.log("expl funcdef ", doc.sliceString(cursor.from, cursor.to)) - nested_scope = explore_variable_usage(result.cursor(), doc, nested_scope, verbose) + pos_resetter() - return raise_scope(nested_scope, scopestate, cursor.to) - } else { - scopestate = explore_variable_usage(arg.cursor(), doc, scopestate, verbose) - } - } + if (verbose) console.log("end of FunctionDefinition, currently at ", cursor.node) - let inner_scope = lower_scope(scopestate) - - for (let { node: arg } of do_args) { - inner_scope = explorer_function_definition_argument(arg, doc, inner_scope) - } - for (let { node: expression } of do_expressions) { - inner_scope = explore_variable_usage(expression.cursor(), doc, inner_scope, verbose) + if (verbose) console.groupEnd() + return false } - return raise_scope(inner_scope, scopestate, cursor.to) - } else if ((match = match_julia(cursor)`(${t.many("tuple_elements")},)`)) { - // TODO.. maybe? `(x, g = y)` is a "ParenthesizedExpression", but lezer parses it as a tuple... - // For now I fix it here hackily by checking if there is only NamedFields + } else if (cursor.name === "Generator") { + // This is: (f(x) for x in xs) or [f(x) for x in xs] + const savior = back_to_parent_resetter(cursor) - let { tuple_elements = [] } = match + // We do a Generator in two steps: + // First we explore all the ForBindings (where locals get defined), and then we go into the first child (where those locals are used). - let match_tuple_element = make_beautiful_specific_matcher((arg) => jl_dynamic`(${arg},)`) - - let is_named_field = tuple_elements.map(({ node }) => match_tuple_element(cursor)`${t.Identifier} = ${t.any}` != null) - - if (is_named_field.every((x) => x === true) || is_named_field.every((x) => x === false)) { - // Valid tuple, either named or unnamed - for (let { node: element } of tuple_elements) { - let match = null - if ((match = match_tuple_element(cursor)`${t.as("name")} = ${t.as("value")}`)) { - let { name, value } = match - if (value) scopestate = explore_variable_usage(value.cursor(), doc, scopestate, verbose) - } else { - scopestate = explore_variable_usage(element.cursor(), doc, scopestate, verbose) - } - } - } else { - // Sneaky! Actually an expression list 😏 - for (let { node: element } of tuple_elements) { - let match = null - if ((match = match_tuple_element(cursor)`${t.as("name")} = ${t.as("value")}`)) { - // 🚨 actually an assignment 🚨 - let { name, value } = match - if (value) scopestate = explore_variable_usage(value.cursor(), doc, scopestate, verbose) - if (name) scopestate = scopestate_add_definition(scopestate, doc, name, value?.to ?? null) - } else { - scopestate = explore_variable_usage(element.cursor(), doc, scopestate, verbose) - } + // 1. The for bindings `x in xs` + if (cursor.firstChild()) { + // Note that we skip the first child here, which is what we want! That's the iterated expression that we leave for the end. + while (cursor.nextSibling()) { + cursor.iterate(enter, leave) } + savior() } - return scopestate - } else if ( - (match = match_julia(cursor)`(${t.many("args")}) -> ${t.many("body")}`) ?? - (match = match_julia(cursor)`${t.as("arg")} -> ${t.many("body")}`) ?? - (match = match_julia(cursor)`${t.as("name")}(${t.many("args")})::${t.as("return_type")} = ${t.many("body")}`) ?? - (match = match_julia(cursor)`${t.as("name")}(${t.many("args")}) = ${t.many("body")}`) ?? - (match = match_julia(cursor)`${t.as("name")}(${t.many("args")}) = ${t.many("body", t.anything_that_fits(jl`x, y`))}`) ?? - (match = match_julia(cursor)` - function ${t.as("name")}(${t.many("args")})::${t.as("return_type")} where ${t.as("type_param")} - ${t.many("body")} - end - `) ?? - (match = match_julia(cursor)` - function ${t.as("name")}(${t.many("args")}) where ${t.as("type_param")} - ${t.many("body")} - end - `) ?? - (match = match_julia(cursor)` - function ${t.as("name")}(${t.many("args")})::${t.as("return_type")} - ${t.many("body")} - end - `) ?? - (match = match_julia(cursor)` - function ${t.as("name")}(${t.many("args")}) - ${t.many("body")} - end - `) ?? - (match = match_julia(cursor)` - function ${t.as("name", t.Identifier)} end - `) ?? - // Putting macro definitions here too because they are very similar - (match = match_julia(cursor)`macro ${t.as("macro_name")} end`) ?? - (match = match_julia(cursor)` - macro ${t.as("macro_name")}(${t.many("args")}) - ${t.many("body")} - end - `) - ) { - let { name, macro_name, arg, args = [], return_type, type_param, body = [] } = match - - if (arg) { - args.push({ node: arg }) - } - - if (name) { - scopestate = scopestate_add_definition(scopestate, doc, name) - } else if (macro_name) { - scopestate.definitions.set(`@${doc.sliceString(macro_name.from, macro_name.to)}`, { - from: macro_name.from, - to: macro_name.to, - valid_from: macro_name.to, - }) - } - - let inner_scope = lower_scope(scopestate) - if (type_param) { - let match_where_types = make_beautiful_specific_matcher((x) => jl_dynamic`function X() where ${x} end`) - let match_where_type = make_beautiful_specific_matcher((x) => jl_dynamic`function X() where {${x}} end`) - - let type_params = [{ node: type_param }] - let multiple_types_match = match_where_types(type_param)`{${t.many("type_params")}}` - if (multiple_types_match) { - type_params = multiple_types_match.type_params - } - - for (let { node: type_param } of type_params) { - let match = null - if ((match = match_where_type(type_param)`${t.as("defined", t.Identifier)} <: ${t.as("parent_type")}`)) { - let { defined, parent_type } = match - inner_scope = explore_variable_usage(parent_type, doc, inner_scope, verbose) - inner_scope = scopestate_add_definition(inner_scope, doc, defined) - } else if ((match = match_where_type(type_param)`${t.as("defined", t.Identifier)}`)) { - let { defined } = match - inner_scope = scopestate_add_definition(inner_scope, doc, defined) - } else { - verbose && console.warn(`Can't handle type param:`, type_param) - } - } + // 2. The iterated expression `f(x)` + if (cursor.firstChild()) { + cursor.iterate(enter, leave) + savior() } - if (return_type) { - inner_scope = explore_variable_usage(narrow(return_type).cursor(), doc, inner_scope, verbose) - } - for (let { node: arg } of args) { - inner_scope = explorer_function_definition_argument(arg.cursor(), doc, inner_scope, verbose) - } - for (let { node: expression } of body) { - inner_scope = explore_variable_usage(expression.cursor(), doc, inner_scope, verbose) - } - return raise_scope(inner_scope, scopestate, cursor.to) - } else if ( - (match = match_julia(cursor)` - let ${t.many("assignments", jl`${t.as("assignee")} = ${t.as("value")}`)} - ${t.many("body", t.any)} - end - `) - ) { - let { assignments = [], body = [] } = match - let innerscope = lower_scope(scopestate) - for (let { - match: { assignee, value }, - } of assignments) { - // Explorer lefthandside in inner scope - if (assignee) innerscope = explore_pattern(assignee, doc, innerscope, value?.to ?? null, verbose) - // Explorer righthandside in the outer scope - if (value) scopestate = explore_variable_usage(value.cursor(), doc, scopestate, verbose) - } - // Explorer body in innerscope - for (let { node: line } of body) { - innerscope = explore_variable_usage(line.cursor(), doc, innerscope, verbose) - } - return raise_scope(innerscope, scopestate, cursor.to) - } else if ( - // A bit hard to see from the template, but these are array (and generator) comprehensions - // e.g. [x for x in y] - (match = match_julia(cursor)`[ - ${t.as("result")} - ${t.many("clauses", t.anything_that_fits(jl`for x = y`))} - ]`) ?? - // Are there syntax differences between Array or Generator expressions? - // For now I treat them the same... - // (Also this is one line because lezer doesn't parse multiline generator expressions yet) - (match = match_julia(cursor)`(${t.as("result")} ${t.many("clauses", t.anything_that_fits(jl`for x = y`))})`) - ) { - let { result, clauses } = match + // k thx byeee + leave(cursor) + return false + } + } - let nested_scope = lower_scope(scopestate) + leave = (/** @type {TreeCursor} */ cursor) => { + if (verbose) { + console.groupEnd() + } - // Because of the `t.anything_that_fits`, we can now match different `for x in/∈/= y`-s and `if x`-s manually. - // There might be a way to express this in the template, but this keeps templates a lot simpler yet powerful. - for (let { node: for_binding } of clauses) { - let match = null - if ( - (match = match_for_binding(for_binding)`for ${t.as("variable")} = ${t.maybe(t.as("value"))}`) ?? - (match = match_for_binding(for_binding)`for ${t.as("variable")} in ${t.maybe(t.as("value"))}`) ?? - (match = match_for_binding(for_binding)`for ${t.as("variable")} ∈ ${t.maybe(t.as("value"))}`) ?? - (match = match_for_binding(for_binding)`for ${t.as("variable")}`) - ) { - let { variable, value } = match + if (does_this_create_scope(cursor)) { + local_scope_stack.pop() + } + } - if (value) nested_scope = explore_variable_usage(value.cursor(), doc, nested_scope, verbose) - if (variable) nested_scope = explore_pattern(variable, doc, nested_scope) - } else if ((match = match_for_binding(for_binding)`if ${t.maybe(t.as("if"))}`)) { - let { if: node } = match - if (node) nested_scope = explore_variable_usage(node.cursor(), doc, nested_scope, verbose) - } else { - verbose && console.warn("Hmmm, can't parse for binding", for_binding) - } - } + const debugged_enter = (cursor) => { + const a = cursor_not_moved_checker(cursor) + const result = enter(cursor) + a() + return result + } - nested_scope = explore_variable_usage(result.cursor(), doc, nested_scope, verbose) + tree.iterate(verbose ? debugged_enter : enter, leave) - return raise_scope(nested_scope, scopestate, cursor.to) - } else { - if (verbose) { - console.groupCollapsed(`Cycling through all children of`, cursor.name) - console.log(`text:`, doc.sliceString(cursor.from, cursor.to)) - console.groupEnd() - } + if (local_scope_stack.length > 0) throw new Error(`Some scopes were not leaved... ${JSON.stringify(local_scope_stack)}`) - // In most cases we "just" go through all the children separately - for (let subcursor of child_cursors(cursor)) { - scopestate = explore_variable_usage(subcursor, doc, scopestate, verbose) - } - return scopestate - } - } finally { - verbose && console.groupEnd() - } + const output = { usages, definitions, locals } + if (verbose) console.log(output) + return output } /** @@ -1099,7 +380,7 @@ export let ScopeStateField = StateField.define({ try { if (syntaxTree(tr.state) != syntaxTree(tr.startState)) { let cursor = syntaxTree(tr.state).cursor() - let scopestate = explore_variable_usage(cursor, tr.state.doc) + let scopestate = explore_variable_usage(cursor, tr.state.doc, null) return scopestate } else { return value diff --git a/frontend/components/CellInput/tests/.gitignore b/frontend/components/CellInput/tests/.gitignore deleted file mode 100644 index ed9f9cc128..0000000000 --- a/frontend/components/CellInput/tests/.gitignore +++ /dev/null @@ -1 +0,0 @@ -coverage \ No newline at end of file diff --git a/frontend/components/CellInput/tests/README.md b/frontend/components/CellInput/tests/README.md deleted file mode 100644 index c8dd93eef5..0000000000 --- a/frontend/components/CellInput/tests/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# Deno unit tests - -Unit tests for the scopestate explorer (and maybe later more who knows) - -```shell -deno test --import-map=./import_map.json --allow-env -``` - -You can also do coverage, which looks fun but not sure how to interpret it: - -```shell -deno test --import-map=./import_map.json --allow-env --coverage=coverage -deno coverage coverage --include=../scopestate_statefield.js -``` \ No newline at end of file diff --git a/frontend/components/CellInput/tests/import_map.json b/frontend/components/CellInput/tests/import_map.json deleted file mode 100644 index 57a87cf9b4..0000000000 --- a/frontend/components/CellInput/tests/import_map.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "imports": { - "../../../imports/lodash.js": "https://deno.land/x/lodash@4.17.15-es/lodash.js" - } -} diff --git a/frontend/components/CellInput/tests/scopestate.test.js b/frontend/components/CellInput/tests/scopestate.test.js deleted file mode 100644 index 5dc05e668c..0000000000 --- a/frontend/components/CellInput/tests/scopestate.test.js +++ /dev/null @@ -1,151 +0,0 @@ -import { jl } from "../lezer_template.js" -import { test_implicit } from "./scopestate_helpers.js" -// @ts-ignore -import chalk from "https://deno.land/x/chalk_deno@v4.1.1-deno/source/index.js" - -let broken = (fn) => { - try { - console.log() - fn() - console.log(chalk.green("BROKEN TEST PASSED???")) - } catch (error) { - console.log(chalk.blue("Broken test failed as expected")) - } -} - -Deno.test("Function call", () => { - test_implicit(jl`global_call(global_argument1, global_argument2...)`) -}) - -Deno.test("Simple definition", () => { - test_implicit(jl`defined = 10`) -}) - -Deno.test("Tuple destructuring", () => { - test_implicit(jl`(defined_1, defined_2, defined_3...) = global_call()`) -}) - -// Need to fix this later -Deno.test("BROKEN Named destructuring", () => { - broken(() => { - test_implicit(jl`(; irrelevant_1=defined_1, irrelevant_2=defined_2) = global_call()`) - }) -}) - -Deno.test("Array comprehension", () => { - test_implicit(jl`(local_x + global_const for local_x in global_xs)`) - test_implicit(jl`( - local_x + local_y + global_const - for local_x in global_xs - for local_y in [local_x, local_x * 2] - )`) - test_implicit(jl`(local_x for local_x in global_xs if local_x > global_const)`) -}) - -Deno.test("BROKEN Array comprehension with comma but VERY WEIRD", () => { - // So... `for x in xs, y in ys` does resolve the `_ in _`'s from right to left... - // Except..... it then specifically "unbinds" the defined variables..... - // So in this case, even though `irrelevant` might be globally defined, - // this will ALWAYS say `irrelevant is not defined`, because it is "overshadowed" by the `for irrelevant in global_ys`. - broken(() => { - test_implicit(jl`(local_x + global_const for local_x in irrelevant, for irrelevant in global_ys)`) - }) -}) - -Deno.test("BROKEN Array comprehension with comma", () => { - broken(() => { - test_implicit(jl`(local_x for global_array in global_array_array, for local_x in global_array)`) - }) -}) - -Deno.test("Function definition", () => { - test_implicit(jl` - function defined_function(local_argument1, local_argument2...; local_argument3, local_argument4...) - local_argument1, local_argument2, local_argument3, local_argument4 - global_var1, global_var2 - end - `) -}) - -Deno.test("Function definition", () => { - test_implicit(jl` - function defined_function(local_argument1 = global_default) - local_argument1 - end - `) -}) - -Deno.test("Function definition where", () => { - test_implicit(jl` - function defined_function(local_argument1) where {local_type} - local_argument1, local_type - end - `) -}) - -Deno.test("Function definition returntype", () => { - test_implicit(jl`begin - function defined_function(local_argument1)::global_type - local_argument1 - end - end`) -}) - -Deno.test("Function expression", () => { - test_implicit(jl`defined_fn = (local_argument1, local_argument2) -> (local_argument1, local_argument2, global_var1, global_var2)`) -}) - -Deno.test("Let block", () => { - test_implicit(jl`defined_outside = let local_let = global_let - local_inside = local_let - local_inside * global_inside - end`) -}) - -Deno.test("Imports", () => { - test_implicit(jl`import defined_module`) - test_implicit(jl`import X: defined_specific1, defined_specific2`) - test_implicit(jl`import X: defined_specific3, irrelevant as defined_alias`) - test_implicit(jl`import X.defined_specific4`) - test_implicit(jl`import X.irrelevant as defined_alias2`) -}) - -Deno.test("Typed struct", () => { - test_implicit(jl`begin - struct defined_struct{local_type} <: global_type{local_type, global_type2} - g - y::global_type3 - z = global_var - x::local_type = global_var2 - end - end`) -}) - -Deno.test("Quotes", () => { - test_implicit(jl`quote - irrelevant_1 = irrelevant_2 - irrelevant_3 = $(global_var) - end`) -}) - -Deno.test("Nested Quotes", () => { - test_implicit(jl`quote - :($irrelevant) - :($$global_var) - end - end`) -}) - -Deno.test("Macros", () => { - test_implicit(jl`global_used.@irrelevant`) - test_implicit(jl`@global_but_not_macro.irrelevant`) - test_implicit(jl`@macro_global`) -}) - -Deno.test("Lonely bare tuple", () => { - test_implicit(jl`defined, = (global_var,)`) -}) - -Deno.test("Very, very lonely arguments", () => { - test_implicit(jl`global_var(;)`) -}) diff --git a/frontend/components/CellInput/tests/scopestate_helpers.js b/frontend/components/CellInput/tests/scopestate_helpers.js deleted file mode 100644 index 45054c083a..0000000000 --- a/frontend/components/CellInput/tests/scopestate_helpers.js +++ /dev/null @@ -1,75 +0,0 @@ -import { assertEquals as untyped_assertEquals } from "https://deno.land/std@0.123.0/testing/asserts.ts" -import { jl, as_node, as_doc, JuliaCodeObject, as_string } from "../lezer_template.js" -import { explore_variable_usage } from "../scopestate_statefield.js" - -/** - * @template T - * @param {T} a - * @param {T} b - **/ -export let assertEquals = (a, b) => untyped_assertEquals(a, b) - -/** - * @param {import("../scopestate_statefield.js").ScopeState} scopestate - */ -let simplify_scopestate = (scopestate) => { - let { definitions, usages } = scopestate - return { - defined: new Set(Array.from(definitions.keys())), - local_used: new Set(usages.filter((x) => x.definition != null).map((x) => x.name)), - global_used: new Set(usages.filter((x) => x.definition == null).map((x) => x.name)), - } -} - -/** - * @param {import("../lezer_template.js").JuliaCodeObject} input - * @param {{ - * defined?: Array, - * local_used?: Array, - * global_used?: Array, - * }} expected - */ -export let test_scopestate = (input, expected) => { - let scopestate = { - defined: [], - local_used: [], - global_used: [], - } - assertEquals(simplify_scopestate(explore_variable_usage(as_node(input).cursor(), as_doc(input), undefined, false)), { - defined: new Set(expected.defined), - local_used: new Set(expected.local_used), - global_used: new Set(expected.global_used), - }) -} - -/** - * Compute the scopestate for the code. Variable names are used to create the expected scope: a variable called `global_something_2` is expected to be a global usage, etc. - * - * @param {JuliaCodeObject} code - */ -export function test_implicit(code) { - let expected_scopestate = { defined: [], local_used: [], global_used: [] } - - console.log(as_string(code)) - - for (let variable of as_string(code).matchAll(/(macro_)?(global|local|defined)(_[a-z0-9_]+)?/g)) { - let [variable_name, is_macro, usage_type, number] = variable - - if (is_macro != null) { - variable_name = `@${variable_name}` - } - - let index = variable.index - if (usage_type === "global") { - expected_scopestate.global_used.push(variable_name) - } else if (usage_type === "local") { - expected_scopestate.local_used.push(variable_name) - } else if (usage_type === "defined") { - expected_scopestate.defined.push(variable_name) - } - } - - console.log(expected_scopestate) - - return test_scopestate(code, expected_scopestate) -} diff --git a/frontend/imports/CodemirrorPlutoSetup.d.ts b/frontend/imports/CodemirrorPlutoSetup.d.ts index 4de86cb4d3..d86a63b1cb 100644 --- a/frontend/imports/CodemirrorPlutoSetup.d.ts +++ b/frontend/imports/CodemirrorPlutoSetup.d.ts @@ -4533,6 +4533,33 @@ declare class TreeCursor implements SyntaxNodeRef { */ matchContext(context: readonly string[]): boolean; } +/** +Provides a way to associate values with pieces of trees. As long +as that part of the tree is reused, the associated values can be +retrieved from an updated tree. +*/ +declare class NodeWeakMap { + private map; + private setBuffer; + private getBuffer; + /** + Set the value for this syntax node. + */ + set(node: SyntaxNode, value: T): void; + /** + Retrieve value for this syntax node, if it exists in the map. + */ + get(node: SyntaxNode): T | undefined; + /** + Set the value for the node that a cursor currently points to. + */ + cursorSet(cursor: TreeCursor, value: T): void; + /** + Retrieve the value for the node that a cursor currently points + to. + */ + cursorGet(cursor: TreeCursor): T | undefined; +} /** Objects returned by the function passed to @@ -7208,4 +7235,4 @@ Python language support. */ declare function python(): LanguageSupport; -export { Annotation, ChangeSet, Compartment, Decoration, Diagnostic, EditorSelection, EditorState, EditorView, Facet, HighlightStyle, MatchDecorator, NodeProp, PostgreSQL, SelectionRange, StateEffect, StateField, Text, Tooltip, Transaction, TreeCursor, ViewPlugin, ViewUpdate, WidgetType, index_d as autocomplete, bracketMatching, closeBrackets, closeBracketsKeymap, collab, combineConfig, completionKeymap, css, cssLanguage, defaultHighlightStyle, defaultKeymap, drawSelection, foldGutter, foldKeymap, getClientID, getSyncedVersion, highlightActiveLine, highlightSelectionMatches, highlightSpecialChars, history, historyKeymap, html, htmlLanguage, indentLess, indentMore, indentOnInput, indentUnit, javascript, javascriptLanguage, julia, keymap, lineNumbers, linter, markdown, markdownLanguage, moveLineDown, moveLineUp, parseCode, parseMixed, placeholder, python, pythonLanguage, receiveUpdates, rectangularSelection, searchKeymap, selectNextOccurrence, sendableUpdates, setDiagnostics, showTooltip, sql, syntaxHighlighting, syntaxTree, syntaxTreeAvailable, tags, tooltips }; +export { Annotation, ChangeSet, Compartment, Decoration, Diagnostic, EditorSelection, EditorState, EditorView, Facet, HighlightStyle, MatchDecorator, NodeProp, NodeWeakMap, PostgreSQL, SelectionRange, StateEffect, StateField, Text, Tooltip, Transaction, Tree, TreeCursor, ViewPlugin, ViewUpdate, WidgetType, index_d as autocomplete, bracketMatching, closeBrackets, closeBracketsKeymap, collab, combineConfig, completionKeymap, css, cssLanguage, defaultHighlightStyle, defaultKeymap, drawSelection, foldGutter, foldKeymap, getClientID, getSyncedVersion, highlightActiveLine, highlightSelectionMatches, highlightSpecialChars, history, historyKeymap, html, htmlLanguage, indentLess, indentMore, indentOnInput, indentUnit, javascript, javascriptLanguage, julia, keymap, lineNumbers, linter, markdown, markdownLanguage, moveLineDown, moveLineUp, parseCode, parseMixed, placeholder, python, pythonLanguage, receiveUpdates, rectangularSelection, searchKeymap, selectNextOccurrence, sendableUpdates, setDiagnostics, showTooltip, sql, syntaxHighlighting, syntaxTree, syntaxTreeAvailable, tags, tooltips }; diff --git a/frontend/imports/CodemirrorPlutoSetup.js b/frontend/imports/CodemirrorPlutoSetup.js index 55e25a5766..f1c5bba63c 100644 --- a/frontend/imports/CodemirrorPlutoSetup.js +++ b/frontend/imports/CodemirrorPlutoSetup.js @@ -44,6 +44,7 @@ export { indentUnit, combineConfig, NodeProp, + NodeWeakMap, autocomplete, html, htmlLanguage, @@ -64,4 +65,4 @@ export { linter, setDiagnostics, //@ts-ignore -} from "https://cdn.jsdelivr.net/gh/JuliaPluto/codemirror-pluto-setup@f01836f/dist/index.es.min.js" +} from "https://cdn.jsdelivr.net/gh/JuliaPluto/codemirror-pluto-setup@d17bc01/dist/index.es.min.js" diff --git a/sample/cm6 crash test.jl b/sample/cm6 crash test.jl index 3c3fecad28..e6f36782d6 100644 --- a/sample/cm6 crash test.jl +++ b/sample/cm6 crash test.jl @@ -52,6 +52,9 @@ end x = a : b:c end +# ╔═║ f248e96a-4050-4888-940b-f38158c102fe + + # ╔═║ daba5486-8d5e-4fce-959b-251e821e5dea # https://github.com/fonsp/Pluto.jl/issues/2639 let x = 1 end @@ -355,6 +358,7 @@ end # ╠═bf834c19-3d0d-4989-9ba3-ef7cb77f9a00 # ╠═3da33f9d-1240-4522-9463-8772b0c2539a # ╠═c88fe37a-c2e6-46f1-b92b-98737437a741 +# ╠═f248e96a-4050-4888-940b-f38158c102fe # ╠═daba5486-8d5e-4fce-959b-251e821e5dea # ╠═287dd3c7-33e6-482d-9639-d502fcff9234 # ╠═6db1e583-54f2-4d8f-9181-a7913345c7fd diff --git a/src/analysis/Parse.jl b/src/analysis/Parse.jl index 5385b0cecf..11dbb3a085 100644 --- a/src/analysis/Parse.jl +++ b/src/analysis/Parse.jl @@ -119,6 +119,24 @@ end preprocess_expr(val::Any) = val +""" +Does this `String` contain a single expression? If this function returns `false`, then Pluto will show a "multiple expressions in one cell" error in the editor. + +!!! compat "Pluto 0.20.5" + This function is new in Pluto 0.20.5. + +""" +function is_single_expression(s::String) + n = Pluto.Notebook([Pluto.Cell(s)]) + e = parse_custom(n, n.cells[1]) + bad = Meta.isexpr(e, :toplevel, 2) && Meta.isexpr(e.args[2], :call, 2) && e.args[2].args[1] == :(PlutoRunner.throw_syntax_error) && e.args[2].args[2] isa String && startswith(e.args[2].args[2], "extra token after end of expression") + + + return !bad +end + + + function updated_topology(old_topology::NotebookTopology{Cell}, notebook::Notebook, updated_cells) get_code_str(cell::Cell) = cell.code get_code_expr(cell::Cell) = parse_custom(notebook, cell) diff --git a/src/webserver/Dynamic.jl b/src/webserver/Dynamic.jl index 5960c0ff74..b347e1629a 100644 --- a/src/webserver/Dynamic.jl +++ b/src/webserver/Dynamic.jl @@ -427,7 +427,6 @@ function _set_cells_to_queued_in_local_state(client, notebook, cells) if haskey(results, cell.cell_id) old = results[cell.cell_id]["queued"] results[cell.cell_id]["queued"] = true - @debug "Setting val!" cell.cell_id old end end end diff --git a/src/webserver/SessionActions.jl b/src/webserver/SessionActions.jl index 0135c9dbcb..52733f50e6 100644 --- a/src/webserver/SessionActions.jl +++ b/src/webserver/SessionActions.jl @@ -258,7 +258,9 @@ function move(session::ServerSession, notebook::Notebook, newpath::String) else move_notebook!(notebook, newpath; disable_writing_notebook_files=session.options.server.disable_writing_notebook_files) putplutoupdates!(session, clientupdate_notebook_list(session.notebooks)) - WorkspaceManager.cd_workspace((session, notebook), newpath) + let workspace = WorkspaceManager.get_workspace((session, notebook); allow_creation=false) + isnothing(workspace) || WorkspaceManager.cd_workspace(workspace, newpath) + end end end diff --git a/test/misc API.jl b/test/misc API.jl new file mode 100644 index 0000000000..5d9bc9ccb1 --- /dev/null +++ b/test/misc API.jl @@ -0,0 +1,43 @@ +@testset "Misc API" begin + + + @testset "is_single_expression" begin + + @test Pluto.is_single_expression("") + @test Pluto.is_single_expression("a") + @test Pluto.is_single_expression("a + 1") + @test Pluto.is_single_expression("a; a + 1") + @test !Pluto.is_single_expression(""" + a = 1 + a + 1 + """) + + @test Pluto.is_single_expression(""" + "yooo" + function f(x) + X + C \\ c + end + """) + + + @test Pluto.is_single_expression(""" + # asdf + + "yooo" + function f(x) + X + C \\ c + end; # aasasdf + """) + + + + @test Pluto.is_single_expression(""" + a a a a a // / // / 123 1 21 1313 + """) + + + + end +end \ No newline at end of file diff --git a/test/runtests.jl b/test/runtests.jl index 4efab46c88..c088d38e33 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -46,6 +46,7 @@ verify_no_running_processes() @timeit_include("DependencyCache.jl") @timeit_include("Throttled.jl") @timeit_include("cell_disabling.jl") +@timeit_include("misc API.jl") verify_no_running_processes()