Skip to content

Commit

Permalink
fix double-backslash escapes
Browse files Browse the repository at this point in the history
  • Loading branch information
simvux committed Oct 26, 2024
1 parent 4f15420 commit de6bbde
Showing 1 changed file with 24 additions and 1 deletion.
25 changes: 24 additions & 1 deletion lumina-parser/src/lexer.rs
Original file line number Diff line number Diff line change
@@ -1,14 +1,30 @@
use logos::{Lexer as LogosLexer, Logos, SpannedIter};
use lumina_util::Span;

fn find_str_end<'src>(lex: &mut LogosLexer<'src, Token>) {
let mut i = 0;
let bytes = lex.remainder().as_bytes();

while let Some(&c) = bytes.get(i) {
match c {
b'\\' if bytes.get(i + 1) == Some(&b'"') => i += 2,
b'\\' if bytes.get(i + 1) == Some(&b'\\') => i += 2,
b'"' => break,
_ => i += 1,
}
}

lex.bump(i + 1);
}

#[derive(PartialEq, Debug, Clone, Copy, Logos)]
#[logos(subpattern name = "[_a-zA-Z][-_0-9a-zA-Z]*")]
#[logos(subpattern path = "(?&name)(:(?&name))+|(?&name)")]
pub enum Token {
#[regex("\\n+")]
NewLines,

#[regex(r#""(\\[\\"]|[^"])*""#)]
#[token("\"", find_str_end)]
StringLiteral,

#[regex("\\-?\\d+")]
Expand Down Expand Up @@ -375,6 +391,13 @@ mod tests {
};
}

#[test]
fn str_escapes() {
cmp! { r#""hello" a"# => T::StringLiteral, T::Path, T::EOF };
cmp! { r#""hello\\" a"# => T::StringLiteral, T::Path, T::EOF };
cmp! { r#""hello\\\"" a"# => T::StringLiteral, T::Path, T::EOF };
}

#[test]
fn regexes() {
cmp! { "username" => T::Path };
Expand Down

0 comments on commit de6bbde

Please sign in to comment.