feat(langserver): correct highlightning to better highlight python ex…

…presseions, escape sequences, environment variable default value and embedded argument regexes

intellij-client/src/main/kotlin/dev/robotcode/robotcode4ij/highlighting/RobotTextMateHighlightingLexer.kt → intellij-client/src/main/kotlin/dev/robotcode/robotcode4ij/highlighting/RobotCodeLexer.kt

@@ -1,6 +1,8 @@
 package dev.robotcode.robotcode4ij.highlighting
 
+import com.intellij.lexer.LexerBase
 import com.intellij.openapi.util.registry.Registry
+import com.intellij.psi.TokenType
 import com.intellij.psi.tree.IElementType
 import dev.robotcode.robotcode4ij.TextMateBundleHolder
 import dev.robotcode.robotcode4ij.psi.ARGUMENT
@@ -10,6 +12,8 @@ import dev.robotcode.robotcode4ij.psi.CONTINUATION
 import dev.robotcode.robotcode4ij.psi.CONTROL_FLOW
 import dev.robotcode.robotcode4ij.psi.ENVIRONMENT_VARIABLE_BEGIN
 import dev.robotcode.robotcode4ij.psi.ENVIRONMENT_VARIABLE_END
+import dev.robotcode.robotcode4ij.psi.EXPRESSION_VARIABLE_BEGIN
+import dev.robotcode.robotcode4ij.psi.EXPRESSION_VARIABLE_END
 import dev.robotcode.robotcode4ij.psi.HEADER
 import dev.robotcode.robotcode4ij.psi.KEYWORD_CALL
 import dev.robotcode.robotcode4ij.psi.KEYWORD_NAME
@@ -20,22 +24,25 @@ import dev.robotcode.robotcode4ij.psi.TESTCASE_NAME
 import dev.robotcode.robotcode4ij.psi.VARIABLE
 import dev.robotcode.robotcode4ij.psi.VARIABLE_BEGIN
 import dev.robotcode.robotcode4ij.psi.VARIABLE_END
-import org.jetbrains.plugins.textmate.language.syntax.lexer.TextMateElementType
-import org.jetbrains.plugins.textmate.language.syntax.lexer.TextMateHighlightingLexer
+import org.jetbrains.plugins.textmate.language.syntax.lexer.TextMateLexer
+import org.jetbrains.plugins.textmate.language.syntax.lexer.TextMateScope
+import java.util.*
+import kotlin.math.min
 
-class RobotTextMateHighlightingLexer : TextMateHighlightingLexer(
-    TextMateBundleHolder.descriptor, Registry.get("textmate.line.highlighting.limit").asInteger()
-) {
+class RobotCodeTextMateHighlightingLexer : LexerBase() {
     companion object {
         val mapping by lazy {
             mapOf(
                 "comment.line.robotframework" to COMMENT_LINE,
                 "comment.line.rest.robotframework" to COMMENT_LINE,
                 "comment.block.robotframework" to COMMENT_BLOCK,
+
                 "punctuation.definition.variable.begin.robotframework" to VARIABLE_BEGIN,
                 "punctuation.definition.variable.end.robotframework" to VARIABLE_END,
                 "punctuation.definition.envvar.begin.robotframework" to ENVIRONMENT_VARIABLE_BEGIN,
                 "punctuation.definition.envvar.end.robotframework" to ENVIRONMENT_VARIABLE_END,
+                "punctuation.definition.expression.begin.robotframework" to EXPRESSION_VARIABLE_BEGIN,
+                "punctuation.definition.expression.end.robotframework" to EXPRESSION_VARIABLE_END,
 
                 "entity.name.function.testcase.name.robotframework" to TESTCASE_NAME,
                 "entity.name.function.keyword.name.robotframework" to KEYWORD_NAME,
@@ -63,16 +70,94 @@ class RobotTextMateHighlightingLexer : TextMateHighlightingLexer(
                 "string.unquoted.argument.robotframework" to ARGUMENT,
 
                 "keyword.operator.continue.robotframework" to CONTINUATION,
+
+                "punctuation.definition.variable.python.begin.robotframework" to VARIABLE_BEGIN,
             )
         }
     }
 
+
+    private val myLexer =
+        TextMateLexer(
+            TextMateBundleHolder.descriptor, Registry.get("textmate.line.highlighting.limit").asInteger(),
+            true
+        )
+    private var currentLineTokens = LinkedList<TextMateLexer.Token?>()
+    private lateinit var buffer: CharSequence
+    private var endOffset = 0
+    private var currentOffset = 0
+    private var tokenType: IElementType? = null
+    private var tokenStart = 0
+    private var tokenEnd = 0
+    private var restartable = false
+
+    override fun start(buffer: CharSequence, startOffset: Int, endOffset: Int, initialState: Int) {
+        this.buffer = buffer
+        this.endOffset = endOffset
+        this.currentOffset = startOffset
+        this.endOffset = endOffset
+        this.currentLineTokens.clear()
+        this.restartable = initialState == 0
+        myLexer.init(buffer, startOffset)
+        this.advance()
+    }
+
+    override fun getState(): Int {
+        return if (restartable) 0 else 1
+    }
+
     override fun getTokenType(): IElementType? {
-        val result = super.getTokenType() ?: return null
-        if (result is TextMateElementType) {
-            return mapping[result.scope.scopeName] ?: RobotTextMateElementType(result)
+        return tokenType
+    }
+
+    override fun getTokenStart(): Int {
+        return tokenStart
+    }
+
+    override fun getTokenEnd(): Int {
+        return tokenEnd
+    }
+
+    override fun advance() {
+        if (this.currentOffset >= this.endOffset) {
+            this.updateState(null as TextMateLexer.Token?, this.endOffset)
+        } else {
+            if (currentLineTokens.isEmpty()) {
+                myLexer.advanceLine(this.currentLineTokens)
+            }
+
+            this.updateState(
+                currentLineTokens.poll(),
+                myLexer.currentOffset
+            )
         }
-        return result
+    }
+
+    private fun updateState(token: TextMateLexer.Token?, fallbackOffset: Int) {
+        if (token != null) {
+            this.tokenType =
+                (if (token.scope === TextMateScope.WHITESPACE) TokenType.WHITE_SPACE else mapping[token.scope.scopeName]
+                    ?: RobotTextMateElementType.create(token.scope))
+
+            tokenStart = token.startOffset
+            tokenEnd = min(token.endOffset.toDouble(), endOffset.toDouble()).toInt()
+            currentOffset = token.endOffset
+            restartable = token.restartable
+        } else {
+            tokenType = null
+            tokenStart = fallbackOffset
+            tokenEnd = fallbackOffset
+            currentOffset = fallbackOffset
+            restartable = true
+        }
+    }
+
+    override fun getBufferSequence(): CharSequence {
+        return buffer
+    }
+
+    override fun getBufferEnd(): Int {
+        return endOffset
     }
 }
 

intellij-client/src/main/kotlin/dev/robotcode/robotcode4ij/highlighting/RobotCodeSyntaxHighlighter.kt

@@ -0,0 +1,109 @@
+package dev.robotcode.robotcode4ij.highlighting
+
+import com.intellij.lexer.Lexer
+import com.intellij.openapi.editor.colors.TextAttributesKey
+import com.intellij.openapi.fileTypes.PlainSyntaxHighlighter
+import com.intellij.openapi.fileTypes.SyntaxHighlighterBase
+import com.intellij.psi.tree.IElementType
+import com.intellij.util.containers.ContainerUtil
+import dev.robotcode.robotcode4ij.psi.ARGUMENT
+import dev.robotcode.robotcode4ij.psi.COMMENT_BLOCK
+import dev.robotcode.robotcode4ij.psi.COMMENT_LINE
+import dev.robotcode.robotcode4ij.psi.CONTINUATION
+import dev.robotcode.robotcode4ij.psi.CONTROL_FLOW
+import dev.robotcode.robotcode4ij.psi.ENVIRONMENT_VARIABLE_BEGIN
+import dev.robotcode.robotcode4ij.psi.ENVIRONMENT_VARIABLE_END
+import dev.robotcode.robotcode4ij.psi.HEADER
+import dev.robotcode.robotcode4ij.psi.KEYWORD_CALL
+import dev.robotcode.robotcode4ij.psi.KEYWORD_NAME
+import dev.robotcode.robotcode4ij.psi.OPERATOR
+import dev.robotcode.robotcode4ij.psi.RobotTextMateElementType
+import dev.robotcode.robotcode4ij.psi.SETTING
+import dev.robotcode.robotcode4ij.psi.TESTCASE_NAME
+import dev.robotcode.robotcode4ij.psi.VARIABLE
+import dev.robotcode.robotcode4ij.psi.VARIABLE_BEGIN
+import dev.robotcode.robotcode4ij.psi.VARIABLE_END
+import org.jetbrains.plugins.textmate.TextMateService
+import org.jetbrains.plugins.textmate.language.TextMateScopeComparator
+import org.jetbrains.plugins.textmate.language.syntax.highlighting.TextMateTheme
+import org.jetbrains.plugins.textmate.language.syntax.lexer.TextMateScope
+import java.util.function.Function
+
+
+class RobotCodeHighlighter : SyntaxHighlighterBase() {
+    companion object {
+        val elementTypeMap = mapOf(
+            COMMENT_LINE to arrayOf(RobotColors.LINE_COMMENT),
+            COMMENT_BLOCK to arrayOf(RobotColors.BLOCK_COMMENT),
+            VARIABLE_BEGIN to arrayOf(RobotColors.VARIABLE_BEGIN),
+            VARIABLE_END to arrayOf(RobotColors.VARIABLE_END),
+            ENVIRONMENT_VARIABLE_BEGIN to arrayOf(RobotColors.VARIABLE_BEGIN),
+            ENVIRONMENT_VARIABLE_END to arrayOf(RobotColors.VARIABLE_END),
+            TESTCASE_NAME to arrayOf(RobotColors.TESTCASE_NAME),
+            KEYWORD_NAME to arrayOf(RobotColors.KEYWORD_NAME),
+            HEADER to arrayOf(RobotColors.HEADER),
+            SETTING to arrayOf(RobotColors.SETTING),
+            KEYWORD_CALL to arrayOf(RobotColors.KEYWORD_CALL),
+            CONTROL_FLOW to arrayOf(RobotColors.CONTROL_FLOW),
+            VARIABLE to arrayOf(RobotColors.VARIABLE),
+            OPERATOR to arrayOf(RobotColors.OPERATOR),
+            ARGUMENT to arrayOf(RobotColors.ARGUMENT),
+            CONTINUATION to arrayOf(RobotColors.CONTINUATION),
+        )
+
+        val PLAIN_SYNTAX_HIGHLIGHTER: PlainSyntaxHighlighter = PlainSyntaxHighlighter()
+    }
+
+    private val myLexer = RobotTextMateHighlightingLexer()
+
+    override fun getHighlightingLexer(): Lexer {
+        return myLexer
+    }
+
+    override fun getTokenHighlights(tokenType: IElementType?): Array<TextAttributesKey> {
+        val result = elementTypeMap[tokenType]
+        if (result != null) return result
+
+        if (tokenType !is RobotTextMateElementType) return PLAIN_SYNTAX_HIGHLIGHTER.getTokenHighlights(tokenType)
+
+        val service = TextMateService.getInstance()
+        val customHighlightingColors = service.customHighlightingColors
+
+        val highlightingRules = ContainerUtil.union(customHighlightingColors.keys, TextMateTheme.INSTANCE.rules)
+
+        val textMateScope = trimEmbeddedScope(tokenType)
+        val selectors: List<CharSequence> = ContainerUtil.reverse(
+            TextMateScopeComparator(textMateScope, Function.identity())
+                .sortAndFilter(highlightingRules)
+        )
+        val result1 = ContainerUtil.map2Array(
+            selectors,
+            TextAttributesKey::class.java
+        ) { rule: CharSequence ->
+            val customTextAttributes = customHighlightingColors[rule]
+            customTextAttributes?.getTextAttributesKey(TextMateTheme.INSTANCE)
+                ?: TextMateTheme.INSTANCE.getTextAttributesKey(rule)
+        }
+
+        return result1
+    }
+
+    private fun trimEmbeddedScope(tokenType: RobotTextMateElementType): TextMateScope {
+        var current: TextMateScope? = tokenType.scope
+        val trail: MutableList<CharSequence?> = ArrayList()
+        while (current != null) {
+            val scopeName = current.scopeName
+            if (scopeName != null && scopeName.contains(".embedded.")) {
+                var result = TextMateScope.EMPTY
+                for (i in trail.indices.reversed()) {
+                    result = result.add(trail[i])
+                }
+                return result
+            }
+            trail.add(scopeName)
+            current = current.parent
+        }
+        return tokenType.scope
+    }
+}
+