refactor(parser): reduce Token size from 32 to 16 bytes

crates/oxc_parser/src/cursor.rs

@@ -11,7 +11,7 @@ use crate::{
 
 pub struct ParserCheckpoint<'a> {
     lexer: LexerCheckpoint<'a>,
-    cur_token: Token<'a>,
+    cur_token: Token,
     prev_span_end: u32,
     errors_pos: usize,
 }
@@ -29,8 +29,8 @@ impl<'a> Parser<'a> {
     }
 
     /// Get current token
-    pub(crate) fn cur_token(&self) -> &Token<'a> {
-        &self.token
+    pub(crate) fn cur_token(&self) -> Token {
+        self.token
     }
 
     /// Get current Kind
@@ -47,12 +47,12 @@ impl<'a> Parser<'a> {
     }
 
     /// Get current string
-    pub(crate) fn cur_string(&self) -> Option<&str> {
-        self.cur_token().value.get_string()
+    pub(crate) fn cur_string(&self) -> &'a str {
+        self.lexer.get_string(self.token)
     }
 
     /// Peek next token, returns EOF for final peek
-    pub(crate) fn peek_token(&mut self) -> &Token {
+    pub(crate) fn peek_token(&mut self) -> Token {
         self.lexer.lookahead(1)
     }
 
@@ -67,7 +67,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Peek nth token
-    pub(crate) fn nth(&mut self, n: u8) -> &Token {
+    pub(crate) fn nth(&mut self, n: u8) -> Token {
         if n == 0 {
             return self.cur_token();
         }
@@ -94,7 +94,7 @@ impl<'a> Parser<'a> {
     /// whose code point sequence is the same as a `ReservedWord`.
     #[inline]
     fn test_escaped_keyword(&mut self, kind: Kind) {
-        if self.cur_token().escaped && kind.is_all_keyword() {
+        if self.cur_token().escaped() && kind.is_all_keyword() {
             let span = self.cur_token().span();
             self.error(diagnostics::EscapedKeyword(span));
         }

crates/oxc_parser/src/js/expression.rs

@@ -17,8 +17,7 @@ use super::{
 };
 use crate::{
     diagnostics,
-    lexer::{parse_big_int, parse_float, parse_int},
-    lexer::{Kind, TokenValue},
+    lexer::{parse_big_int, parse_float, parse_int, Kind},
     list::SeparatedList,
     Context, Parser,
 };
@@ -96,10 +95,7 @@ impl<'a> Parser<'a> {
 
     pub(crate) fn parse_identifier_kind(&mut self, kind: Kind) -> (Span, Atom) {
         let span = self.start_span();
-        let name = match std::mem::take(&mut self.token.value) {
-            TokenValue::String(value) => value,
-            TokenValue::None => "",
-        };
+        let name = self.cur_string();
         self.bump_remap(kind);
         (self.end_span(span), Atom::from(name))
     }
@@ -121,7 +117,7 @@ impl<'a> Parser<'a> {
     /// # Panics
     pub(crate) fn parse_private_identifier(&mut self) -> PrivateIdentifier {
         let span = self.start_span();
-        let name = Atom::from(self.cur_string().unwrap());
+        let name = Atom::from(self.cur_string());
         self.bump_any();
         PrivateIdentifier { span: self.end_span(span), name }
     }
@@ -349,9 +345,7 @@ impl<'a> Parser<'a> {
         if !self.at(Kind::Str) {
             return Err(self.unexpected());
         }
-        let TokenValue::String(value) = std::mem::take(&mut self.token.value) else {
-            unreachable!()
-        };
+        let value = self.cur_string();
         let span = self.start_span();
         self.bump_any();
         Ok(StringLiteral { span: self.end_span(span), value: value.into() })
@@ -454,8 +448,9 @@ impl<'a> Parser<'a> {
             _ => unreachable!(),
         };
 
-        // cooked = None when template literal has invalid escape sequence
-        let cooked = self.cur_string().map(Atom::from);
+        // `cooked = None` when template literal has invalid escape sequence
+        // This is matched by `is_valid_escape_sequence` in `Lexer::read_template_literal`
+        let cooked = self.cur_token().escaped_string_id.map(|_| self.cur_string());
 
         let raw = &self.cur_src()[1..self.cur_src().len() - end_offset as usize];
         let raw = Atom::from(if cooked.is_some() && raw.contains('\r') {
@@ -475,7 +470,11 @@ impl<'a> Parser<'a> {
         }
 
         let tail = matches!(cur_kind, Kind::TemplateTail | Kind::NoSubstitutionTemplate);
-        TemplateElement { span, tail, value: TemplateElementValue { raw, cooked } }
+        TemplateElement {
+            span,
+            tail,
+            value: TemplateElementValue { raw, cooked: cooked.map(Atom::from) },
+        }
     }
 
     /// Section 13.3 Meta Property

crates/oxc_parser/src/js/function.rs

@@ -50,7 +50,7 @@ impl<'a> Parser<'a> {
     }
 
     pub(crate) fn at_async_no_new_line(&mut self) -> bool {
-        self.at(Kind::Async) && !self.cur_token().escaped && !self.peek_token().is_on_new_line
+        self.at(Kind::Async) && !self.cur_token().escaped() && !self.peek_token().is_on_new_line
     }
 
     pub(crate) fn parse_function_body(&mut self) -> Result<Box<'a, FunctionBody<'a>>> {

crates/oxc_parser/src/js/statement.rs

@@ -127,7 +127,7 @@ impl<'a> Parser<'a> {
             Kind::Const if !(self.ts_enabled() && self.is_at_enum_declaration()) => {
                 self.parse_variable_statement(stmt_ctx)
             }
-            Kind::Let if !self.cur_token().escaped => self.parse_let(stmt_ctx),
+            Kind::Let if !self.cur_token().escaped() => self.parse_let(stmt_ctx),
             Kind::Await
                 if self.peek_kind() == Kind::Using && self.nth_kind(2).is_binding_identifier() =>
             {
@@ -276,7 +276,7 @@ impl<'a> Parser<'a> {
 
         let is_let_of = self.at(Kind::Let) && self.peek_at(Kind::Of);
         let is_async_of =
-            self.at(Kind::Async) && !self.cur_token().escaped && self.peek_at(Kind::Of);
+            self.at(Kind::Async) && !self.cur_token().escaped() && self.peek_at(Kind::Of);
         let expr_span = self.start_span();
 
         if self.at(Kind::RParen) {

crates/oxc_parser/src/jsx/mod.rs

@@ -360,14 +360,15 @@ impl<'a> Parser<'a> {
         }
         // we are at a valid normal Ident or Keyword, let's keep on lexing for `-`
         self.re_lex_jsx_identifier();
-        let name = Atom::from(self.cur_string().unwrap());
         self.bump_any();
-        Ok(self.ast.jsx_identifier(self.end_span(span), name))
+        let span = self.end_span(span);
+        let name = span.source_text(self.source_text);
+        Ok(self.ast.jsx_identifier(span, name.into()))
     }
 
     fn parse_jsx_text(&mut self) -> JSXText {
         let span = self.start_span();
-        let value = Atom::from(self.cur_string().unwrap());
+        let value = Atom::from(self.cur_string());
         self.bump_any();
         self.ast.jsx_text(self.end_span(span), value)
     }