Compute lines and cols at tokenization time

src/dev_aid/lsp.rs

@@ -6,7 +6,7 @@ use lsp_server::{Response, Message, Connection};
 
 use lsp_types::notification::Notification;
 
-use crate::{parser::perform_full_semantic_parse, dev_aid::syntax_highlighting::create_token_ide_info, ast::{IdentifierType, Span}, errors::{ErrorCollector, CompileError, ErrorLevel}, linker::{FileUUIDMarker, Linker, FileUUID, FileData}, arena_alloc::ArenaVector};
+use crate::{arena_alloc::ArenaVector, ast::{IdentifierType, Span}, dev_aid::syntax_highlighting::create_token_ide_info, errors::{ErrorCollector, CompileError, ErrorLevel}, linker::{FileUUIDMarker, Linker, FileUUID, FileData}, parser::perform_full_semantic_parse, tokenizer::{CharLine, TokenizeResult}};
 
 use super::syntax_highlighting::{IDETokenType, IDEIdentifierType, IDEToken};
 
@@ -69,13 +69,13 @@ pub fn lsp_main(port : u16) -> Result<(), Box<dyn Error + Sync + Send>> {
     // Run the server and wait for the two threads to end (typically by trigger LSP Exit event).
     let server_capabilities = serde_json::to_value(&ServerCapabilities {
         definition_provider: Some(OneOf::Left(true)),
-        /*document_highlight_provider: Some(OneOf::Right(
+        document_highlight_provider: Some(OneOf::Right(
             DocumentHighlightOptions{
                 work_done_progress_options: WorkDoneProgressOptions{
-                    work_done_progress: Some(true)
+                    work_done_progress: Some(false)
                 }
             }
-        )),*/
+        )),
         semantic_tokens_provider: Some(SemanticTokensServerCapabilities::SemanticTokensOptions(SemanticTokensOptions{
             work_done_progress_options: WorkDoneProgressOptions {
                 work_done_progress: Some(false)
@@ -170,8 +170,7 @@ impl SemanticTokensDeltaAccumulator {
         }
 
         let delta_col = position.character - self.prev.character;
-        self.prev.character = position.character;
-        self.prev.line = position.line;
+        self.prev = position;
 
         self.semantic_tokens.push(SemanticToken{
             delta_line: delta_line,
@@ -183,100 +182,62 @@ impl SemanticTokensDeltaAccumulator {
     }
 }
 
-fn do_syntax_highlight(file_data : &FileData, linker : &Linker) -> (SemanticTokensResult, Vec<std::ops::Range<Position>>) {
-    let file_text = &file_data.file_text;
+fn do_syntax_highlight(file_data : &FileData, linker : &Linker) -> SemanticTokensResult {
     let ide_tokens = create_token_ide_info(&file_data, linker);
 
     let mut semantic_tokens_acc = SemanticTokensDeltaAccumulator{prev : Position {line : 0, character : 0}, semantic_tokens : Vec::new()};
     semantic_tokens_acc.semantic_tokens.reserve(file_data.tokens.len());
-    let mut positions : Vec<std::ops::Range<Position>> = Vec::new();
-    positions.reserve(file_data.tokens.len());
 
-    let mut cur_whitespace_start = 0;
-    let mut cur_position = Position{line : 0, character : 0};
     for (tok_idx, ide_tok) in ide_tokens.iter().enumerate() {
         let typ = get_semantic_token_type_from_ide_token(ide_tok);
         let mod_bits = get_modifiers_for_token(ide_tok);
 
-        let tok_range = file_data.tokens.get_token_range(tok_idx);
-        let whitespace_text = &file_text[cur_whitespace_start..tok_range.start];
-        cur_whitespace_start = tok_range.end;
-        let token_text = &file_text[tok_range];
-
-        // skip through whitespace
-        for c in whitespace_text.chars() {
-            if c == '\n' {
-                cur_position.line += 1;
-                cur_position.character = 0;
-            } else {
-                cur_position.character += 1;
-            }
-        }
-        let real_token_start_position = cur_position;
-        let mut part_start_position = cur_position;
-        for c in token_text.chars() {
-            if c == '\n' {
-                semantic_tokens_acc.push(part_start_position, cur_position.character - part_start_position.character, typ, mod_bits);
-                cur_position.line += 1;
-                cur_position.character = 0;
-                part_start_position = cur_position;
-            } else {
-                cur_position.character += 1;
-            }
-        }
-        semantic_tokens_acc.push(part_start_position, cur_position.character - part_start_position.character, typ, mod_bits);
-        positions.push(real_token_start_position..cur_position);
-    }
 
-    let eof_start = cur_position.clone();
-    for c in file_text[cur_whitespace_start..].chars() {
-        if c == '\n' {
-            cur_position.line += 1;
-            cur_position.character = 0;
-        } else {
-            cur_position.character += 1;
-        }
+        let tok_range = file_data.tokens.get_token_linechar_range(tok_idx);
+        let start_pos = Position{line : tok_range.start.line as u32, character : tok_range.start.character as u32};
+        let end_pos = Position{line : tok_range.end.line as u32, character : tok_range.end.character as u32};
+        semantic_tokens_acc.push(start_pos, end_pos.character - start_pos.character, typ, mod_bits)
     }
-    positions.push(eof_start..cur_position);
 
-    (SemanticTokensResult::Tokens(lsp_types::SemanticTokens {
+    SemanticTokensResult::Tokens(lsp_types::SemanticTokens {
         result_id: None,
         data: semantic_tokens_acc.semantic_tokens
-    }), positions)
+    })
 }
 
 use lsp_types::Diagnostic;
 
-fn cvt_span_to_lsp_range(ch_sp : Span, token_positions : &[std::ops::Range<Position>]) -> lsp_types::Range {
+fn cvt_span_to_lsp_range(ch_sp : Span, tokens : &TokenizeResult) -> lsp_types::Range {
+    let rng = tokens.get_span_linechar_range(ch_sp);
     Range {
-        start: token_positions[ch_sp.0].start,
-        end: token_positions[ch_sp.1].end
+        start: Position{character : rng.start.character as u32, line : rng.start.line as u32},
+        end: Position{character : rng.end.character as u32, line : rng.end.line as u32}
     }
 }
 
 // Requires that token_positions.len() == tokens.len() + 1 to include EOF token
-fn convert_diagnostic(err : CompileError, token_positions : &[std::ops::Range<Position>], uris : &ArenaVector<Url, FileUUIDMarker>) -> Diagnostic {
-    let error_pos = cvt_span_to_lsp_range(err.position, token_positions);
+fn convert_diagnostic(err : CompileError, tokens : &TokenizeResult, uris : &ArenaVector<Url, FileUUIDMarker>) -> Diagnostic {
+    let error_pos = cvt_span_to_lsp_range(err.position, tokens);
 
     let severity = match err.level {
         ErrorLevel::Error => DiagnosticSeverity::ERROR,
         ErrorLevel::Warning => DiagnosticSeverity::WARNING,
     };
     let mut related_info = Vec::new();
     for info in err.infos {
-        let info_pos = cvt_span_to_lsp_range(info.position, token_positions);
+        let info_pos = cvt_span_to_lsp_range(info.position, tokens);
         let location = Location{uri : uris[info.file].clone(), range : info_pos};
         related_info.push(DiagnosticRelatedInformation { location, message: info.info });
     }
     Diagnostic::new(error_pos, Some(severity), None, None, err.reason, Some(related_info), None)
 }
 
 // Requires that token_positions.len() == tokens.len() + 1 to include EOF token
-fn send_errors_warnings(connection: &Connection, errors : ErrorCollector, token_positions : &[std::ops::Range<Position>], uris : &ArenaVector<Url, FileUUIDMarker>) -> Result<(), Box<dyn Error + Sync + Send>> {
+fn send_errors_warnings(connection: &Connection, errors : ErrorCollector, token_boundaries : &TokenizeResult, uris : &ArenaVector<Url, FileUUIDMarker>) -> Result<(), Box<dyn Error + Sync + Send>> {
     let mut diag_vec : Vec<Diagnostic> = Vec::new();
     let (err_vec, file) = errors.get();
     for err in err_vec {
-        diag_vec.push(convert_diagnostic(err, token_positions, uris));
+        diag_vec.push(convert_diagnostic(err, token_boundaries, uris));
     }
 
     let params = &PublishDiagnosticsParams{
@@ -316,6 +277,14 @@ fn main_loop(
                         let params : GotoDefinitionParams = serde_json::from_value(req.params).expect("JSON Encoding Error while parsing params");
                         println!("got gotoDefinition request: {params:?}");
 
+                        let pos = &params.text_document_position_params.position;
+                        let text_document = &params.text_document_position_params.text_document;
+
+                        let uuid = file_cache.ensure_contains_file(&text_document.uri);
+
+
+                        let file_data = &file_cache.linker.files[uuid];
+
                         let result = Some(GotoDefinitionResponse::Array(Vec::new()));
                         let result = serde_json::to_value(&result).unwrap();
                         let resp = Response { id: req.id, result: Some(result), error: None };
@@ -330,7 +299,7 @@ fn main_loop(
 
                         let file_data = &file_cache.linker.files[uuid];
 
-                        let (syntax_highlight, token_positions) = do_syntax_highlight(file_data, &file_cache.linker);
+                        let syntax_highlight = do_syntax_highlight(file_data, &file_cache.linker);
 
                         let result = serde_json::to_value(&syntax_highlight).unwrap();
                         connection.sender.send(Message::Response(Response{
@@ -340,11 +309,13 @@ fn main_loop(
                         // println!("Flattening...");
                         file_cache.linker.recompile_all();
 
-                        let mut errors = file_cache.linker.files[uuid].parsing_errors.clone();
+                        let file_data = &file_cache.linker.files[uuid]; // Have to grab it again because previous line mutates
+
+                        let mut errors = file_data.parsing_errors.clone();
                         file_cache.linker.get_all_errors_in_file(uuid, &mut errors);
 
                         // println!("Errors: {:?}", &errors);
-                        send_errors_warnings(&connection, errors, &token_positions, &file_cache.uris)?;
+                        send_errors_warnings(&connection, errors, &file_data.tokens, &file_cache.uris)?;
                     },
                     // TODO ...
                     req => {

src/dev_aid/mod.rs

@@ -3,4 +3,3 @@ pub mod syntax_highlighting;
 
 #[cfg(feature = "lsp")]
 pub mod lsp;
-

src/flattening.rs

@@ -3,7 +3,7 @@ use std::{ops::Deref, iter::zip};
 use crate::{
     ast::{AssignableExpression, AssignableExpressionModifiers, CodeBlock, DeclID, DeclIDMarker, Expression, IdentifierType, InterfacePorts, LocalOrGlobal, Module, Operator, Span, SpanAssignableExpression, SpanExpression, SpanTypeExpression, Statement, TypeExpression},
     linker::{Linker, FileUUID, GlobalResolver, ResolvedGlobals, NamedConstant, ConstantUUID, ModuleUUID, NameElem, NamedType, TypeUUIDMarker},
-    errors::{ErrorCollector, error_info, ErrorInfo}, arena_alloc::{UUID, UUIDMarker, FlatAlloc, UUIDRange, ArenaAllocator}, typing::{get_binary_operator_types, typecheck, typecheck_is_array_indexer, typecheck_unary_operator, ResolvedTypeExpr, Type, BOOL_TYPE, INT_TYPE}, value::Value
+    errors::{ErrorCollector, error_info, ErrorInfo}, arena_alloc::{UUID, UUIDMarker, FlatAlloc, UUIDRange, ArenaAllocator}, typing::{get_binary_operator_types, typecheck, typecheck_is_array_indexer, typecheck_unary_operator, WrittenType, Type, BOOL_TYPE, INT_TYPE}, value::Value
 };
 
 #[derive(Debug,Clone,Copy,PartialEq,Eq,Hash)]
@@ -81,7 +81,7 @@ pub struct WireInstance {
 
 #[derive(Debug)]
 pub struct Declaration {
-    pub typ_expr : ResolvedTypeExpr,
+    pub typ_expr : WrittenType,
     pub typ : Type,
     pub is_declared_in_this_module : bool,
     pub name_token : usize,
@@ -185,20 +185,20 @@ struct FlatteningContext<'inst, 'l, 'm> {
 }
 
 impl<'inst, 'l, 'm> FlatteningContext<'inst, 'l, 'm> {
-    fn map_to_type(&mut self, type_expr : &SpanTypeExpression) -> ResolvedTypeExpr {
+    fn map_to_type(&mut self, type_expr : &SpanTypeExpression) -> WrittenType {
         match &type_expr.0 {
             TypeExpression::Named => {
                 if let Some(typ_id) = &self.linker.resolve_type(type_expr.1, &self.errors) {
-                    ResolvedTypeExpr::Named(type_expr.1, *typ_id)
+                    WrittenType::Named(type_expr.1, *typ_id)
                 } else {
-                    ResolvedTypeExpr::Error(type_expr.1)
+                    WrittenType::Error(type_expr.1)
                 }
             }
             TypeExpression::Array(b) => {
                 let (array_type_expr, array_size_expr) = b.deref();
                 let array_element_type = self.map_to_type(&array_type_expr);
                 let array_size_wire_id = self.flatten_expr(array_size_expr);
-                ResolvedTypeExpr::Array(type_expr.1, Box::new((array_element_type, array_size_wire_id)))
+                WrittenType::Array(type_expr.1, Box::new((array_element_type, array_size_wire_id)))
             }
         }
     }
@@ -212,14 +212,14 @@ impl<'inst, 'l, 'm> FlatteningContext<'inst, 'l, 'm> {
                     return self.alloc_module_interface(decl.name.clone(), md, id, decl.typ.1)
                 }
                 Some(NameElem::Type(id)) => {
-                    ResolvedTypeExpr::Named(decl.typ.1, id)
+                    WrittenType::Named(decl.typ.1, id)
                 }
                 Some(global_module_or_type) => {
                     let accepted = if ALLOW_MODULES {"Type or Module"} else {"Type"};
                     self.linker.make_bad_error_location_error(global_module_or_type, accepted, decl.typ.1, &self.errors);
-                    ResolvedTypeExpr::Error(decl.typ.1)
+                    WrittenType::Error(decl.typ.1)
                 }
-                None => ResolvedTypeExpr::Error(decl.typ.1)
+                None => WrittenType::Error(decl.typ.1)
             }
         } else {
             self.map_to_type(&decl.typ)

src/linker.rs

@@ -1,6 +1,6 @@
 use std::{collections::{HashMap, HashSet}, rc::Rc, cell::RefCell};
 
-use crate::{arena_alloc::{ArenaAllocator, UUID, UUIDMarker}, ast::{Module, LinkInfo, Span}, errors::{ErrorCollector, error_info}, flattening::{FlatID, FlattenedModule, Instruction}, instantiation::InstantiatedModule, parser::{FullParseResult, TokenTreeNode}, tokenizer::TokenizeResult, typing::Type, util::{const_str_position, const_str_position_in_tuples}, value::Value};
+use crate::{arena_alloc::{ArenaAllocator, UUID, UUIDMarker}, ast::{Module, LinkInfo, Span}, errors::{ErrorCollector, error_info}, flattening::{ConnectionWrite, FlatID, FlattenedModule, Instruction, WireInstance}, instantiation::InstantiatedModule, parser::{FullParseResult, TokenTreeNode}, tokenizer::TokenizeResult, typing::{WrittenType, Type}, util::{const_str_position, const_str_position_in_tuples}, value::Value};
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 pub struct ModuleUUIDMarker;
@@ -372,6 +372,81 @@ impl Linker {
 
         md.instantiations.instantiate(&md.link_info.name, &md.flattened, self)
     }
+
+    pub fn get_info_about_source_location<'linker>(&'linker self, token_idx : usize, file : FileUUID) -> Option<(LocationInfo<'linker>, Span)> {
+        let mut location_builder = LocationInfoBuilder::new(token_idx);
+
+        for global in &self.files[file].associated_values {
+            match *global {
+                NameElem::Module(md_id) => {
+                    let md = &self.modules[md_id];
+                    if md.link_info.span.contains_token(token_idx) {
+                        for (_id, inst) in &md.flattened.instructions {
+                            match inst {
+                                Instruction::SubModule(sm) => {
+                                    location_builder.update(sm.module_name_span, LocationInfo::Global(NameElem::Module(sm.module_uuid)));
+                                }
+                                Instruction::Declaration(decl) => {
+                                    if let Some(typ) = decl.typ_expr.get_deepest_selected(token_idx) {
+                                        location_builder.update(typ.get_span(), LocationInfo::Type(typ));
+                                    }
+                                }
+                                Instruction::Wire(wire) => {
+                                    location_builder.update(wire.span, LocationInfo::Wire(md, wire));
+                                }
+                                Instruction::Write(write) => {
+                                    location_builder.update(Span::new_single_token(write.to.span.0), LocationInfo::WriteWire(md, &write.to));
+                                }
+                                Instruction::IfStatement(_) | Instruction::ForStatement(_) => {}
+                            };
+                        }
+                        break;
+                    }
+                }
+                NameElem::Type(_) => {
+                    todo!()
+                }
+                NameElem::Constant(_) => {
+                    todo!()
+                }
+            }
+        }
+        if let Some(instr) = location_builder.best_instruction {
+            Some((instr, location_builder.best_span))
+        } else {
+            None
+        }
+    }
+}
+
+pub enum LocationInfo<'linker> {
+    WriteWire(&'linker Module, &'linker ConnectionWrite),
+    Wire(&'linker Module, &'linker WireInstance),
+    Type(&'linker WrittenType),
+    Global(NameElem)
+}
+
+struct LocationInfoBuilder<'linker> {
+    best_instruction : Option<LocationInfo<'linker>>,
+    best_span : Span,
+    token_idx : usize
+}
+
+impl<'linker> LocationInfoBuilder<'linker> {
+    fn new(token_idx : usize) -> Self {
+        Self{
+            best_instruction : None,
+            best_span : Span(0, usize::MAX),
+            token_idx
+        }
+    }
+    fn update(&mut self, span : Span, info : LocationInfo<'linker>) {
+        if span.contains_token(self.token_idx) && span.size() <= self.best_span.size() {
+            assert!(span.size() < self.best_span.size());
+            self.best_span = span;
+            self.best_instruction = Some(info);
+        }
+    }
 }
 
 #[derive(Debug)]