This repository has been archived on 2026-09-13. You can view files and clone it, but cannot push or open issues or pull requests.
snp/src/tokenizer.rs

158 lines
4.5 KiB
Rust

//! Tokenizer module, tasked to parse a string and return a list of tokens
//!
//! As the dynamic formatter, the tokenizer is supposedly format agnostic
use regex::Regex;
use crate::errors::SnippetError;
/// A formatter, containing (or not) an argument to be subsitued with
#[derive(Debug)]
#[cfg_attr(test, derive(PartialEq))]
pub struct CustFormatter {
/// The positional id of the element to replace with
pub id: Option<usize>,
}
impl CustFormatter {
/// Create a CustomFormatter from a string, either void (positional) with a parsable (fixed)
fn new(text: &str) -> CustFormatter {
if text.is_empty() {
return CustFormatter { id: None };
}
CustFormatter {
id: Some(text.parse::<usize>().unwrap()),
}
}
/// Return an option of the Id
pub fn id(&self) -> Option<usize> {
self.id
}
}
/// The parsed snippet
#[derive(Debug)]
#[cfg_attr(test, derive(PartialEq))]
pub enum SnippetToken<'a> {
/// A subsitution token, containing either an index or nothing (and being "positional")
Substitution(CustFormatter),
/// A text token, the inbetween
Text(&'a str),
}
/// Assert where the read head is
enum Status {
/// Read head in inside a formatter
Formater,
/// Read head is outside a formatter
Text,
}
/// Add not empty values only
macro_rules! push_if_some {
($vec:ident, $val:expr) => {
if let Some(x) = $val {
$vec.push(x);
};
};
}
/// Create a Token list from the snippet given and return it as a vector of token
pub fn create_tokens_from_string(snippet: &str) -> Result<Vec<SnippetToken>, SnippetError> {
let re = Regex::new(r"\{\d*\}").unwrap();
let matches = re.find_iter(snippet);
let mut readhead = 0; // Starting to read at zero
let mut token_list = Vec::new();
for r in matches {
// Assume we matched text from read to here
push_if_some!(
token_list,
create_token_from_pos(snippet, Status::Text, readhead, Some(r.start()))
);
push_if_some!(
token_list,
create_token_from_pos(snippet, Status::Formater, r.start(), Some(r.end()),)
);
readhead = r.end();
}
push_if_some!(
token_list,
create_token_from_pos(snippet, Status::Text, readhead, None)
);
Ok(token_list)
}
/// Create a single Token from the current positions
fn create_token_from_pos(
text: &str,
status: Status,
start: usize,
stop: Option<usize>,
) -> Option<SnippetToken> {
let text = match (stop, status) {
(None, Status::Formater) => {
SnippetToken::Substitution(CustFormatter::new(&text[start + 1..]))
}
(None, Status::Text) => SnippetToken::Text(&text[start..]),
(Some(stop), Status::Formater) => {
SnippetToken::Substitution(CustFormatter::new(&text[start + 1..stop - 1]))
}
(Some(stop), Status::Text) => SnippetToken::Text(&text[start..stop]),
};
Some(text)
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn test_derive() {
println!("{:?}", CustFormatter::new("45"));
println!("{:?}", SnippetToken::Text("Bar"));
}
#[test]
fn test_create_token_from_pos() {
// All calls
let text = String::from("0123456789");
assert_eq!(
create_token_from_pos(&text, Status::Formater, 5, None),
Some(SnippetToken::Substitution(CustFormatter::new("6789")))
);
assert_eq!(
create_token_from_pos(&text, Status::Text, 7, None),
Some(SnippetToken::Text("789"))
);
assert_eq!(
create_token_from_pos(&text, Status::Formater, 2, Some(5)), //Need at least 2 digits
Some(SnippetToken::Substitution(CustFormatter::new("3")))
);
assert_eq!(
create_token_from_pos(&text, Status::Text, 0, Some(1)),
Some(SnippetToken::Text("0"))
);
}
#[test]
fn test_create_token_from_string() {
let v = create_tokens_from_string("{} {1} { {{}}").unwrap();
assert_eq!(
v,
vec![
SnippetToken::Text(""),
SnippetToken::Substitution(CustFormatter { id: None }),
SnippetToken::Text(" "),
SnippetToken::Substitution(CustFormatter { id: Some(1) }),
SnippetToken::Text(" { {"),
SnippetToken::Substitution(CustFormatter { id: None }),
SnippetToken::Text("}"),
]
);
}
}