From c2394245225209c0f67ee9a89550aa42dab45a54 Mon Sep 17 00:00:00 2001
From: multipleof4 <planetrenox@protonmail.com>
Date: Fri, 26 Sep 2025 08:21:46 -0700
Subject: [PATCH] Feat: Create dedicated lexer module for tokens

---
 src/lexer.js | 37 +++++++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)
 create mode 100644 src/lexer.js

diff --git a/src/lexer.js b/src/lexer.js
new file mode 100644
index 0000000..ce5178e
--- /dev/null
+++ b/src/lexer.js
@@ -0,0 +1,37 @@
+import { createToken, Lexer } from 'chevrotain';
+
+// --- Category: Skipped ---
+export const WhiteSpace = createToken({ name: 'WhiteSpace', pattern: /[ \t]+/, group: Lexer.SKIPPED });
+export const NewLine = createToken({ name: 'NewLine', pattern: /\n|\r\n?/, group: Lexer.SKIPPED });
+export const Comment = createToken({ name: 'Comment', pattern: /\/\/[^\n\r]*/, group: Lexer.SKIPPED });
+
+// --- Category: Literals ---
+export const Number = createToken({ name: 'Number', pattern: /0|[1-9][0-9]*(\.[0-9]+)?/ });
+export const String = createToken({ name: 'String', pattern: /"(?:\\["\\]|[^\n"\\])*"/ });
+
+// --- Category: Brackets ---
+export const LBrace = createToken({ name: 'LBrace', pattern: /{/ });
+export const RBrace = createToken({ name: 'RBrace', pattern: /}/ });
+export const LParen = createToken({ name: 'LParen', pattern: /\(/ });
+export const RParen = createToken({ name: 'RParen', pattern: /\)/ });
+
+// --- Category: Operators & Punctuation ---
+export const Dot = createToken({ name: 'Dot', pattern: /\./ });
+export const Plus = createToken({ name: 'Plus', pattern: /\+/ });
+export const Comma = createToken({ name: 'Comma', pattern: /,/ });
+export const Colon = createToken({ name: 'Colon', pattern: /:/ });
+export const Eq = createToken({ name: 'Eq', pattern: /=/ });
+
+// --- Category: Identifiers ---
+export const Identifier = createToken({ name: 'Identifier', pattern: /[a-zA-Z_][a-zA-Z0-9_]*/ });
+
+// Order matters for the lexer
+export const allTokens = [
+    WhiteSpace, NewLine, Comment,
+    Number, String,
+    LBrace, RBrace, LParen, RParen,
+    Dot, Plus, Comma, Colon, Eq,
+    Identifier,
+];
+
+export const HiLexer = new Lexer(allTokens);