This commit is contained in:
vicky 2022-03-16 15:08:09 +01:00
commit d666719ecc
2 changed files with 104 additions and 23 deletions

View File

@ -137,32 +137,23 @@ def readable_markup(list_of_tokens):
for token in list_of_tokens:
if type(token) == str:
readable_string += token
elif token.label == 'text:em':
readable_string += _add_tag('em', readable_markup(token.content))
elif token.label == 'text:strong':
readable_string += _add_tag('b', readable_markup(token.content))
elif token.label == 'text:underline':
readable_string += _add_tag('u', readable_markup(token.content))
elif token.label == 'text:strikethrough':
readable_string += _add_tag('st', readable_markup(token.content))
elif token.label == 'text:custom_span':
readable_string += _add_tag('#', readable_markup(token.content))
elif token.label == 'text':
readable_string += ' ' + ' '.join(token.content)
else:
raise Exception('Unknown token type' + token.label)
return readable_string[1:]
readable_string += _add_tag(token)
return readable_string
def _add_tag(tag, content):
if content:
return f' <{tag}>{content}</{tag}>'
return ''
def _add_tag(token):
"""
Function used for testing and readability purposes. Replaces matched markup with html-like tags.
:param token: matched markup token in analysed text.
:return: returns readable string.
"""
if token.label == 'text':
return token.content[0]
if token.content:
return "<{} = '{}' />".format(token.label, ",".join(token.content))
# create_image_token = of_type(ImageToken)
# create_image_token()
# Base elements
quotes = pp.Word(r""""'""")
value = (quotes + pp.Word(pp.alphanums + r'.') + pp.match_previous_literal(quotes) ^
@ -216,7 +207,7 @@ il_link = pp.Literal('[') + ... + pp.Literal('](') + pp.QuotedString("'\"", esc_
# Oneline elements
# TODO: one_header should match any number of '#' I suggest either a div_element=Word("#") or a div_element=OneOrMore(pp.Literal('#')) with a pp.match_previous_literal(dive=_element) # noqa E501 (line too long)
one_header = (pp.Literal('#') + ... + pps(" "+'#')).add_parse_action(of_type(HeaderToken))
one_header = (pp.Literal('#') + ... + pps('#')).add_parse_action(of_type(HeaderToken))
# TODO: Don't use "^" (longest match wins) use "|" (first match wins) # TODO: add a ("name") ? # TODO: Suppress the "."
one_olist = pp.line_start + \

View File

@ -3,6 +3,9 @@ import pytest
import bootstraparse.modules.syntax as sy
##############################################################################################################
# PRE-PARSER
##############################################################################################################
# Dictionary of all label and associated token used in the Pre_parser
list_of_token_types = {
"alias": sy.AliasToken,
@ -48,6 +51,10 @@ expressions_to_match = {
}
##############################################################################################################
# PARSER
##############################################################################################################
# Testing the parser lexical elements
dict_advanced_syntax_input_and_expected_output = {
# Enhanced text # TODO : Test optional inline args
@ -127,7 +134,7 @@ dict_advanced_syntax_input_and_expected_output = {
("div>> [class='blue', 123#]{var='test', number=11}", (sy.UnimplementedToken(["div", ">>", sy.UnimplementedToken(["[", "class='blue'", ",", "123#", "]"]), sy.UnimplementedToken(["{", "var='test'", ",", "number=11", "}"])]))), # noqa E501 (line too long)
],
# Elements
# List Elements
"one_olist": [
# Matches a one-level ordered list element, must be at the beginning of the line
# Drop the "." from the match
@ -181,6 +188,53 @@ dict_advanced_syntax_input_and_expected_output = {
],
}
list_add_tag_input_and_expected_output = [
# Test cases for the add_tag function
# Input:
(sy.TextToken(["Text"]), "Text"),
(sy.EtEmToken(["*"]), "<text:em = '*' />"),
(sy.EtStrongToken(["**"]), "<text:strong = '**' />"),
(sy.EtUnderlineToken(["__"]), "<text:underline = '__' />"),
(sy.EtStrikethroughToken(["~~"]), "<text:strikethrough = '~~' />"),
(sy.StructuralElementStartToken(["div"]), "<se:start = 'div' />"),
(sy.StructuralElementEndToken(["div"]), "<se:end = 'div' />"),
(sy.EtCustomSpanToken(["(#12)"]), "<text:custom_span = '(#12)' />"),
(sy.HeaderToken(["#", "Text"]), "<header = '#,Text' />"),
]
list_of_text_input_and_readable_output = [
# Header
("# Text1 #", "one_header", "<header: '#,Text1' />"),
("## Text2 ##", "one_header", "<header: '##,Text2' />"),
# Text
("*Italic*", "et_em", "<em: *>Italic</em :*>"),
("__Underline__", "et_underline", "<underline: __>Underline</underline :__>"),
("**Bold**", "et_strong", "<strong: **>Bold</strong :**>"),
("~~Strikethrough~~", "et_strikethrough", "<strikethrough: ~~>Strikethrough</strikethrough :~~>"),
("Text *bold __underline__ still bold*", "enhanced_text", "<em: *><strong: **><underline: __><strikethrough: ~~>Text bold underline still bold</strikethrough :~~></underline :__></strong :**></em :*>"), # noqa E501 (line too long)
("[link]('http://www.google.com')", "il_link", "<link: [link](http://www.google.com)>"),
# Structural elements
("<<div", "se", "<div: <<div>"),
("div>>", "se", "<div: div>>>"),
# Lists
("- Text", "et_ulist", "<ulist: '-,Text' />"),
("12. Text", "et_olist", "<olist: '12,Text' />"),
("# Text", "et_olist", "<olist: '#,Text' />"),
# Tables
("| Text1 | Text2 |", "table", "<table: | Text1 | Text2 |>"),
("| Text1 |3 Text2 |", "table", "<table: | Text1 |3 Text2 |>"),
("|---|---|", "table_separator", "<table_separator: |---|---|>"),
("|:--|-:-|--:|--:|", "table_separator", "<table_separator: |:--|-:-|--:|--:|>"),
# Other Mostly for testing and coverage
("'", "quotes", "'"),
]
@pytest.mark.parametrize("token_name,token_class", list_of_token_types.items())
def test_token_type_exists(token_name, token_class):
@ -286,3 +340,39 @@ def test_advanced_expression_and_token_creation(expression, tokens):
assert result is not None
for token, expected_token in zip(result, expected_tokens):
assert token == expected_token
########################################################################################################################
# TEXT OUTPUT
########################################################################################################################
# Test the add tag function
@pytest.mark.parametrize("token, expected_output", list_add_tag_input_and_expected_output)
def test__add_tag(token, expected_output):
"""
Test that the add_tag function works correctly.
:param token: The token to test.
:param expected_output: The expected output.
:type token: sy.SemanticType
:type expected_output: str
"""
assert sy._add_tag(token) == expected_output
# Test text functions
@pytest.mark.parametrize("token_list, parsing_expression, expected", list_of_text_input_and_readable_output)
def test_readable_markup(token_list, parsing_expression, expected):
"""
Test that the readable markup is correctly generated.
:param token_list: The list of tokens.
:param parsing_expression: The parsing expression.
:param expected: The expected readable markup.
:type token_list: str
:type parsing_expression: str
:type expected: str
"""
expr = find_expression_from_str(parsing_expression)
assert isinstance(expr, pyparsing.ParserElement)
result = expr.parse_string(token_list)
assert result is not None
assert sy.readable_markup(result) == expected