pub fn tokenize(input: &str) -> Result<Vec<Token>>Expand description
Tokenizes the full input into a Vec of Tokens.
An empty string yields a single Token::Text with empty content.
ยงExamples
use runjucks_core::lexer::{tokenize, Token};
let tokens = tokenize("Hi {{ name }}").unwrap();
assert!(matches!(tokens[0], Token::Text(_)));
assert!(matches!(tokens[1], Token::Expression(_)));