css2: Support tokenizing %-tokens

robinlinden · Jan 12, 2025 · 860a1af · 860a1af
1 parent 630e8b9
commit 860a1af
Show file tree

Hide file tree

Showing 4 changed files with 38 additions and 11 deletions.
diff --git a/css2/token.h b/css2/token.h
@@ -1,5 +1,5 @@
 // SPDX-FileCopyrightText: 2022 Mikael Larsson <[email protected]>
-// SPDX-FileCopyrightText: 2024 Robin Lindén <[email protected]>
+// SPDX-FileCopyrightText: 2024-2025 Robin Lindén <[email protected]>
 //
 // SPDX-License-Identifier: BSD-2-Clause
 
@@ -69,7 +69,7 @@ struct NumberToken {
 };
 
 struct PercentageToken {
-    std::variant<int, double> data{};
+    std::variant<std::int32_t, double> data{};
     [[nodiscard]] bool operator==(PercentageToken const &) const = default;
 
     [[nodiscard]] constexpr bool is_integer() const { return std::holds_alternative<int>(data); }

diff --git a/css2/tokenizer.cpp b/css2/tokenizer.cpp
@@ -93,8 +93,7 @@ void Tokenizer::run() {
                         continue;
                     case '+': {
                         if (inputs_starts_number(*c)) {
-                            auto number = consume_number(*c);
-                            emit(NumberToken{number});
+                            emit(consume_a_numeric_token(*c));
                         } else {
                             emit(DelimToken{'+'});
                         }
@@ -105,8 +104,7 @@ void Tokenizer::run() {
                         continue;
                     case '-': {
                         if (inputs_starts_number(*c)) {
-                            auto number = consume_number(*c);
-                            emit(NumberToken{number});
+                            emit(consume_a_numeric_token(*c));
                             continue;
                         }
 
@@ -126,8 +124,7 @@ void Tokenizer::run() {
                     }
                     case '.': {
                         if (auto next_input = peek_input(0); is_digit(next_input)) {
-                            auto number = consume_number(*c);
-                            emit(NumberToken{number});
+                            emit(consume_a_numeric_token(*c));
                             continue;
                         }
 
@@ -162,9 +159,7 @@ void Tokenizer::run() {
                     case '7':
                     case '8':
                     case '9': {
-                        // TODO(robinlinden): https://www.w3.org/TR/css-syntax-3/#consume-a-numeric-token
-                        auto number = consume_number(*c);
-                        emit(NumberToken{number});
+                        emit(consume_a_numeric_token(*c));
                         continue;
                     }
                     default:
@@ -543,4 +538,15 @@ std::string Tokenizer::consume_an_escaped_code_point() {
     return std::string{*c};
 }
 
+Token Tokenizer::consume_a_numeric_token(char first_byte) {
+    // TODO(robinlinden): https://www.w3.org/TR/css-syntax-3/#consume-a-numeric-token
+    auto number = consume_number(first_byte);
+    if (peek_input(0) == '%') {
+        std::ignore = consume_next_input_character();
+        return PercentageToken{number};
+    }
+
+    return NumberToken{number};
+}
+
 } // namespace css2
diff --git a/css2/tokenizer.h b/css2/tokenizer.h
@@ -72,6 +72,7 @@ class Tokenizer {
 
     std::variant<std::int32_t, double> consume_number(char first_byte);
     std::string consume_an_escaped_code_point();
+    Token consume_a_numeric_token(char first_byte);
 };
 
 } // namespace css2

diff --git a/css2/tokenizer_test.cpp b/css2/tokenizer_test.cpp
@@ -439,6 +439,26 @@ int main() {
         expect_token(output, NumberToken{.data = 1});
     });
 
+    s.add_test("percentage: integer", [](etest::IActions &a) {
+        auto output = run_tokenizer(a, "13%");
+        expect_token(output, PercentageToken{.data = 13});
+    });
+
+    s.add_test("percentage: large", [](etest::IActions &a) {
+        auto output = run_tokenizer(a, "12147483647%");
+        expect_token(output, PercentageToken{std::numeric_limits<std::int32_t>::max()});
+    });
+
+    s.add_test("percentage: large negative", [](etest::IActions &a) {
+        auto output = run_tokenizer(a, "-12147483648%");
+        expect_token(output, PercentageToken{std::numeric_limits<std::int32_t>::min()});
+    });
+
+    s.add_test("percentage: number", [](etest::IActions &a) {
+        auto output = run_tokenizer(a, "13.25%");
+        expect_token(output, PercentageToken{.data = 13.25});
+    });
+
     s.add_test("plus: delim", [](etest::IActions &a) {
         auto output = run_tokenizer(a, "+hello");
         expect_token(output, DelimToken{'+'});