//! Tokenizer module, tasked to parse a string and return a list of tokens //! //! As the dynamic formatter, the tokenizer is supposedly format agnostic use regex::Regex; use crate::errors::SnippetError; /// A formatter, containing (or not) an argument to be subsitued with #[derive(Debug)] #[cfg_attr(test, derive(PartialEq))] pub struct CustFormatter { /// The positional id of the element to replace with pub id: Option, } impl CustFormatter { /// Create a CustomFormatter from a string, either void (positional) with a parsable (fixed) fn new(text: &str) -> CustFormatter { if text.is_empty() { return CustFormatter { id: None }; } CustFormatter { id: Some(text.parse::().unwrap()), } } /// Return an option of the Id pub fn id(&self) -> Option { self.id } } /// The parsed snippet #[derive(Debug)] #[cfg_attr(test, derive(PartialEq))] pub enum SnippetToken<'a> { /// A subsitution token, containing either an index or nothing (and being "positional") Substitution(CustFormatter), /// A text token, the inbetween Text(&'a str), } /// Assert where the read head is enum Status { /// Read head in inside a formatter Formater, /// Read head is outside a formatter Text, } /// Add not empty values only macro_rules! push_if_some { ($vec:ident, $val:expr) => { if let Some(x) = $val { $vec.push(x); }; }; } /// Create a Token list from the snippet given and return it as a vector of token pub fn create_tokens_from_string(snippet: &str) -> Result, SnippetError> { let re = Regex::new(r"\{\d*\}").unwrap(); let matches = re.find_iter(snippet); let mut readhead = 0; // Starting to read at zero let mut token_list = Vec::new(); for r in matches { // Assume we matched text from read to here push_if_some!( token_list, create_token_from_pos(snippet, Status::Text, readhead, Some(r.start())) ); push_if_some!( token_list, create_token_from_pos(snippet, Status::Formater, r.start(), Some(r.end()),) ); readhead = r.end(); } push_if_some!( token_list, create_token_from_pos(snippet, Status::Text, readhead, None) ); Ok(token_list) } /// Create a single Token from the current positions fn create_token_from_pos( text: &str, status: Status, start: usize, stop: Option, ) -> Option { let text = match (stop, status) { (None, Status::Formater) => { SnippetToken::Substitution(CustFormatter::new(&text[start + 1..])) } (None, Status::Text) => SnippetToken::Text(&text[start..]), (Some(stop), Status::Formater) => { SnippetToken::Substitution(CustFormatter::new(&text[start + 1..stop - 1])) } (Some(stop), Status::Text) => SnippetToken::Text(&text[start..stop]), }; Some(text) } #[cfg(test)] mod test { use super::*; #[test] fn test_derive() { println!("{:?}", CustFormatter::new("45")); println!("{:?}", SnippetToken::Text("Bar")); } #[test] fn test_create_token_from_pos() { // All calls let text = String::from("0123456789"); assert_eq!( create_token_from_pos(&text, Status::Formater, 5, None), Some(SnippetToken::Substitution(CustFormatter::new("6789"))) ); assert_eq!( create_token_from_pos(&text, Status::Text, 7, None), Some(SnippetToken::Text("789")) ); assert_eq!( create_token_from_pos(&text, Status::Formater, 2, Some(5)), //Need at least 2 digits Some(SnippetToken::Substitution(CustFormatter::new("3"))) ); assert_eq!( create_token_from_pos(&text, Status::Text, 0, Some(1)), Some(SnippetToken::Text("0")) ); } #[test] fn test_create_token_from_string() { let v = create_tokens_from_string("{} {1} { {{}}").unwrap(); assert_eq!( v, vec![ SnippetToken::Text(""), SnippetToken::Substitution(CustFormatter { id: None }), SnippetToken::Text(" "), SnippetToken::Substitution(CustFormatter { id: Some(1) }), SnippetToken::Text(" { {"), SnippetToken::Substitution(CustFormatter { id: None }), SnippetToken::Text("}"), ] ); } }