158 lines
4.5 KiB
Rust
158 lines
4.5 KiB
Rust
//! Tokenizer module, tasked to parse a string and return a list of tokens
|
|
//!
|
|
//! As the dynamic formatter, the tokenizer is supposedly format agnostic
|
|
|
|
use regex::Regex;
|
|
|
|
use crate::errors::SnippetError;
|
|
|
|
/// A formatter, containing (or not) an argument to be subsitued with
|
|
#[derive(Debug)]
|
|
#[cfg_attr(test, derive(PartialEq))]
|
|
pub struct CustFormatter {
|
|
/// The positional id of the element to replace with
|
|
pub id: Option<usize>,
|
|
}
|
|
|
|
impl CustFormatter {
|
|
/// Create a CustomFormatter from a string, either void (positional) with a parsable (fixed)
|
|
fn new(text: &str) -> CustFormatter {
|
|
if text.is_empty() {
|
|
return CustFormatter { id: None };
|
|
}
|
|
CustFormatter {
|
|
id: Some(text.parse::<usize>().unwrap()),
|
|
}
|
|
}
|
|
|
|
/// Return an option of the Id
|
|
pub fn id(&self) -> Option<usize> {
|
|
self.id
|
|
}
|
|
}
|
|
|
|
/// The parsed snippet
|
|
#[derive(Debug)]
|
|
#[cfg_attr(test, derive(PartialEq))]
|
|
pub enum SnippetToken<'a> {
|
|
/// A subsitution token, containing either an index or nothing (and being "positional")
|
|
Substitution(CustFormatter),
|
|
/// A text token, the inbetween
|
|
Text(&'a str),
|
|
}
|
|
|
|
/// Assert where the read head is
|
|
enum Status {
|
|
/// Read head in inside a formatter
|
|
Formater,
|
|
/// Read head is outside a formatter
|
|
Text,
|
|
}
|
|
|
|
/// Add not empty values only
|
|
macro_rules! push_if_some {
|
|
($vec:ident, $val:expr) => {
|
|
if let Some(x) = $val {
|
|
$vec.push(x);
|
|
};
|
|
};
|
|
}
|
|
|
|
/// Create a Token list from the snippet given and return it as a vector of token
|
|
pub fn create_tokens_from_string(snippet: &str) -> Result<Vec<SnippetToken>, SnippetError> {
|
|
let re = Regex::new(r"\{\d*\}").unwrap();
|
|
let matches = re.find_iter(snippet);
|
|
let mut readhead = 0; // Starting to read at zero
|
|
let mut token_list = Vec::new();
|
|
for r in matches {
|
|
// Assume we matched text from read to here
|
|
push_if_some!(
|
|
token_list,
|
|
create_token_from_pos(snippet, Status::Text, readhead, Some(r.start()))
|
|
);
|
|
push_if_some!(
|
|
token_list,
|
|
create_token_from_pos(snippet, Status::Formater, r.start(), Some(r.end()),)
|
|
);
|
|
readhead = r.end();
|
|
}
|
|
push_if_some!(
|
|
token_list,
|
|
create_token_from_pos(snippet, Status::Text, readhead, None)
|
|
);
|
|
|
|
Ok(token_list)
|
|
}
|
|
|
|
/// Create a single Token from the current positions
|
|
fn create_token_from_pos(
|
|
text: &str,
|
|
status: Status,
|
|
start: usize,
|
|
stop: Option<usize>,
|
|
) -> Option<SnippetToken> {
|
|
let text = match (stop, status) {
|
|
(None, Status::Formater) => {
|
|
SnippetToken::Substitution(CustFormatter::new(&text[start + 1..]))
|
|
}
|
|
(None, Status::Text) => SnippetToken::Text(&text[start..]),
|
|
(Some(stop), Status::Formater) => {
|
|
SnippetToken::Substitution(CustFormatter::new(&text[start + 1..stop - 1]))
|
|
}
|
|
(Some(stop), Status::Text) => SnippetToken::Text(&text[start..stop]),
|
|
};
|
|
|
|
Some(text)
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod test {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn test_derive() {
|
|
println!("{:?}", CustFormatter::new("45"));
|
|
println!("{:?}", SnippetToken::Text("Bar"));
|
|
}
|
|
|
|
#[test]
|
|
fn test_create_token_from_pos() {
|
|
// All calls
|
|
let text = String::from("0123456789");
|
|
assert_eq!(
|
|
create_token_from_pos(&text, Status::Formater, 5, None),
|
|
Some(SnippetToken::Substitution(CustFormatter::new("6789")))
|
|
);
|
|
assert_eq!(
|
|
create_token_from_pos(&text, Status::Text, 7, None),
|
|
Some(SnippetToken::Text("789"))
|
|
);
|
|
assert_eq!(
|
|
create_token_from_pos(&text, Status::Formater, 2, Some(5)), //Need at least 2 digits
|
|
Some(SnippetToken::Substitution(CustFormatter::new("3")))
|
|
);
|
|
|
|
assert_eq!(
|
|
create_token_from_pos(&text, Status::Text, 0, Some(1)),
|
|
Some(SnippetToken::Text("0"))
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_create_token_from_string() {
|
|
let v = create_tokens_from_string("{} {1} { {{}}").unwrap();
|
|
assert_eq!(
|
|
v,
|
|
vec![
|
|
SnippetToken::Text(""),
|
|
SnippetToken::Substitution(CustFormatter { id: None }),
|
|
SnippetToken::Text(" "),
|
|
SnippetToken::Substitution(CustFormatter { id: Some(1) }),
|
|
SnippetToken::Text(" { {"),
|
|
SnippetToken::Substitution(CustFormatter { id: None }),
|
|
SnippetToken::Text("}"),
|
|
]
|
|
);
|
|
}
|
|
}
|