From 352f9e897eb1e595c18d2c23a99d027c17f1e643 Mon Sep 17 00:00:00 2001 From: John Lorentzson Date: Wed, 9 Jul 2025 14:37:47 +0200 Subject: [PATCH] Add TOKEN-ATOMIC for better tokenization statistics --- wip-duuqnd/user-side-compiler/tokenizer.lisp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/wip-duuqnd/user-side-compiler/tokenizer.lisp b/wip-duuqnd/user-side-compiler/tokenizer.lisp index b48aa06..15666ab 100644 --- a/wip-duuqnd/user-side-compiler/tokenizer.lisp +++ b/wip-duuqnd/user-side-compiler/tokenizer.lisp @@ -65,8 +65,10 @@ reading immediately. Should be a subset of *SPECIAL-TOKEN-CHARS*.") ;; Special syntax tokens +(defclass token-atomic (token) ()) + (defmacro define-atomic-token (name) - `(defclass ,name (token) ())) + `(defclass ,name (token-atomic) ())) (define-atomic-token token-end-of-statement)