lots of fixing

This commit is contained in:
vicky 2022-03-17 17:20:39 +01:00
parent 99f0fe26d4
commit a9b9d85ce1
2 changed files with 51 additions and 84 deletions

View File

@ -200,15 +200,24 @@ et_em = pp.Literal('*')('em').add_parse_action(of_type(EtEmToken))
et_strong = pp.Literal('**')('strong').add_parse_action(of_type(EtStrongToken)) et_strong = pp.Literal('**')('strong').add_parse_action(of_type(EtStrongToken))
et_underline = pp.Literal('__')('underline').add_parse_action(of_type(EtUnderlineToken)) et_underline = pp.Literal('__')('underline').add_parse_action(of_type(EtUnderlineToken))
et_strikethrough = pp.Literal('~~')('strikethrough').add_parse_action(of_type(EtStrikethroughToken)) et_strikethrough = pp.Literal('~~')('strikethrough').add_parse_action(of_type(EtStrikethroughToken))
et_custom_span = ('(#' + pp.Word(pp.nums)('span_id') + ')').set_name('custom_span') et_custom_span = (
pps('(#') + pp.Word(pp.nums)('span_id') + pps(')')
).set_name('custom_span').add_parse_action(of_type(EtCustomSpanToken))
markup = et_strong | et_em | et_strikethrough | et_underline | et_custom_span
# Multiline elements # Multiline elements
se_start = ('<<' + structural_elements).add_parse_action(of_type(StructuralElementStartToken)) se_start = (pps('<<') + structural_elements).add_parse_action(of_type(StructuralElementStartToken))
se_end = (structural_elements + '>>').add_parse_action(of_type(StructuralElementEndToken)) se_end = (structural_elements + pps('>>')).add_parse_action(of_type(StructuralElementEndToken))
se = se_end | se_start # Structural element se = se_end | se_start # Structural element
# Inline elements # Inline elements
il_link = pp.Literal('[') + ... + pp.Literal('](') + quotes + url_characters + pp.match_previous_literal(quotes) + ')' # [link text]('http://test.icule')
il_link = pp.common.url
# il_link = (
# pp.Literal('[') + pp.SkipTo('](')('text') + '](' +
# quotes + url_characters('url') + pp.match_previous_literal(quotes)
# + ')'
# ).add_parse_action(of_type(HyperlinkToken))
# Oneline elements # Oneline elements
one_header = ( one_header = (
@ -225,7 +234,10 @@ one_ulist = pp.line_start + (
).add_parse_action(of_type(EtUlistToken)) ).add_parse_action(of_type(EtUlistToken))
# Final elements # Final elements
enhanced_text = pp.OneOrMore(et_em | et_strikethrough | et_strong | et_underline | et_custom_span | text) enhanced_text = pp.OneOrMore(markup + text)
# pp.SkipTo(markup)('text').add_parse_action(of_type(TextToken)) + markup |
# pp.SkipTo(pp.line_end)("text").add_parse_action(of_type(TextToken))
# )
############################################################################## ##############################################################################
@ -250,55 +262,11 @@ if __name__ == '__main__': # pragma: no cover
pp.autoname_elements() pp.autoname_elements()
list_strings = [ list_strings = [
"Normal text", "'http://test.icule'",
"Normal text *em text*",
"Normal text **strong text**",
"Normal text __underline text__",
"Normal text ~~strikethrough text~~",
"Normal text (#123) custom span text(#123)",
"**Strong text __underline__** normal text",
"Normal text **strong text *em text*** *em text **strong text*** normal text",
"<<div",
"div>>",
"[link text]('http://test.icule')",
"# header #",
"1. ordered list",
"- unordered list",
] ]
# div_start.parse_string(list_strings[8])
# div_end.parse_string(list_strings[9])
# il_link.parse_string(list_strings[10])
# one_header.parse_string(list_strings[11])
# one_olist.parse_string(list_strings[12])
# one_ulist.parse_string(list_strings[13])
# enhanced_text.create_diagram("../../../dev_outputs/diagram.html") rich.inspect(il_link.parse_string(list_strings[0]))
# for string in list_strings:
# print('Input string:', string)
# output = enhanced_text.parseString(string)
# # rich.inspect(output)
# print('Output string:', readable_markup(output))
# print()
if not os.path.exists('../../../dev_outputs/'): if not os.path.exists('../../../dev_outputs/'):
os.mkdir('../../../dev_outputs/') os.mkdir('../../../dev_outputs/')
enhanced_text.create_diagram("../../../dev_outputs/diagram.html") enhanced_text.create_diagram("../../../dev_outputs/diagram.html")
import timeit
base_time = {}
for string in list_strings:
print('Input string:', string)
n_time = timeit.timeit("enhanced_text.parseString(string)", number=100, globals=globals())
base_time[string] = n_time
output = enhanced_text.parseString(string)
# rich.inspect(output)
rich.print(f'[] Output string: {readable_markup(output)} in {n_time:.2}s)')
print()
time = timeit.timeit("enhanced_text.parseString(mega_string)", number=5, globals=globals())
rich.print(f'[]Mega time: {time}')
base_time['mega_string'] = time
print('Base time:')
for string, time in base_time.items():
print(f' "{string}": {time},')

View File

@ -56,31 +56,31 @@ expressions_to_match = {
############################################################################################################## ##############################################################################################################
# Testing the parser lexical elements # Testing the parser lexical elements
# test_advanced_expression_and_token_creation
dict_advanced_syntax_input_and_expected_output = { dict_advanced_syntax_input_and_expected_output = {
# Enhanced text # TODO : Test optional inline args # Enhanced text # TODO : Test optional inline args
"et_em": [ "et_em": [
# Single em element, should only match the first "*" # Single em element, should only match the first "*"
("*", (sy.EtEmToken("*"), )), ("*", (sy.EtEmToken(["*"]), )),
], ],
"et_strong": [ "et_strong": [
# Single strong element, should only match the first "**" # Single strong element, should only match the first "**"
("**", sy.EtStrongToken("**"), ), ("**", (sy.EtStrongToken(["**"]), )),
], ],
"et_underline": [ "et_underline": [
# Single underline element, should only match the first "__" # Single underline element, should only match the first "__"
("__", (sy.EtUnderlineToken("__"), )), ("__", (sy.EtUnderlineToken(["__"]), )),
], ],
"et_strikethrough": [ "et_strikethrough": [
# Single strikethrough element, should only match the first "~~" # Single strikethrough element, should only match the first "~~"
("~~", (sy.EtStrikethroughToken("~~"), )), ("~~", (sy.EtStrikethroughToken(["~~"]), )),
], ],
"et_custom_span": [ "et_custom_span": [
# Single custom span element, should only match the first "(#number)" # Single custom span element, should only match the first "(#number)"
("(#12345)", (sy.EtCustomSpanToken("#12345"), )), ("(#12345)", (sy.EtCustomSpanToken(["12345"]), )),
], ],
"il_link": [ "il_link": [
# Single link element, should only match the first "[link_name](link)" # Single link element, should only match the first "[link_name](link)"
# TODO: Text tokens ? or Link token and hyperlink token ?
("[text_link]('text://www.website.com/link.html')", ("[text_link]('text://www.website.com/link.html')",
sy.HyperlinkToken("[text_link]('text://www.website.com/link.html')")) sy.HyperlinkToken("[text_link]('text://www.website.com/link.html')"))
], ],
@ -92,7 +92,7 @@ dict_advanced_syntax_input_and_expected_output = {
("**test**", (sy.EtStrongToken(["**"]), sy.TextToken(["test"]), sy.EtStrongToken(["**"]))), ("**test**", (sy.EtStrongToken(["**"]), sy.TextToken(["test"]), sy.EtStrongToken(["**"]))),
("__test__", (sy.EtUnderlineToken(["__"]), sy.TextToken(["test"]), sy.EtUnderlineToken(["__"]))), ("__test__", (sy.EtUnderlineToken(["__"]), sy.TextToken(["test"]), sy.EtUnderlineToken(["__"]))),
("~~test~~", (sy.EtStrikethroughToken(["~~"]), sy.TextToken(["test"]), sy.EtStrikethroughToken(["~~"]))), ("~~test~~", (sy.EtStrikethroughToken(["~~"]), sy.TextToken(["test"]), sy.EtStrikethroughToken(["~~"]))),
("(#12345)", (sy.EtCustomSpanToken(["#12345"]), )), ("(#12345)", (sy.EtCustomSpanToken(["12345"]), )),
("*test*test*", (sy.EtEmToken(["*"]), sy.TextToken(["test"]), sy.EtEmToken(["*"]), sy.TextToken(["test"]), ("*test*test*", (sy.EtEmToken(["*"]), sy.TextToken(["test"]), sy.EtEmToken(["*"]), sy.TextToken(["test"]),
sy.EtEmToken(["*"]))), sy.EtEmToken(["*"]))),
("**test**test**", (sy.EtStrongToken(["**"]), sy.TextToken(["test"]), sy.EtStrongToken(["**"]), sy.TextToken(["test"]), ("**test**test**", (sy.EtStrongToken(["**"]), sy.TextToken(["test"]), sy.EtStrongToken(["**"]), sy.TextToken(["test"]),
@ -101,13 +101,12 @@ dict_advanced_syntax_input_and_expected_output = {
sy.TextToken(["test"]), sy.EtUnderlineToken(["__"]), )), sy.TextToken(["test"]), sy.EtUnderlineToken(["__"]), )),
("~~test~~test~~", (sy.EtStrikethroughToken(["~~"]), sy.TextToken(["test"]), sy.EtStrikethroughToken(["~~"]), ("~~test~~test~~", (sy.EtStrikethroughToken(["~~"]), sy.TextToken(["test"]), sy.EtStrikethroughToken(["~~"]),
sy.TextToken(["test"]), sy.EtStrikethroughToken(["~~"]), )), sy.TextToken(["test"]), sy.EtStrikethroughToken(["~~"]), )),
("(#12345)test(#12345)", (sy.EtCustomSpanToken(["#12345"]), sy.TextToken(["test"]), ("(#12345)test(#12345)", (sy.EtCustomSpanToken(["12345"]), sy.TextToken(["test"]),
sy.EtCustomSpanToken(["#12345"]), )), sy.EtCustomSpanToken(["12345"]), )),
("__test(#12)test**test__test~~", (sy.EtUnderlineToken("__"), sy.TextToken(["test"]), ("__test(#12)test**test__test~~", (sy.EtUnderlineToken(["__"]), sy.TextToken(["test"]),
sy.EtCustomSpanToken(["#12"]), sy.TextToken(["test"]), sy.EtStrongToken(["**"]), sy.EtCustomSpanToken(["12"]), sy.TextToken(["test"]), sy.EtStrongToken(["**"]),
sy.TextToken(["test"]), sy.EtStrikethroughToken(["~~"]), sy.TextToken(["test"]), sy.EtUnderlineToken(["__"]),
sy.TextToken(["test"]), sy.EtUnderlineToken(["__"]), )), sy.TextToken(["test"]), sy.EtStrikethroughToken(["~~"]), )),
("#. List of elements", (sy.EtUlistToken(["#", "List of elements"]), )),
("Test text with a link [link_name](link)", (sy.TextToken(["Test text with a link "]), ("Test text with a link [link_name](link)", (sy.TextToken(["Test text with a link "]),
sy.HyperlinkToken(["link_name", "link"]), )), sy.HyperlinkToken(["link_name", "link"]), )),
("Test text with a link [link_name](link) and a *bold* text", (sy.TextToken(["Test text with a link "]), ("Test text with a link [link_name](link) and a *bold* text", (sy.TextToken(["Test text with a link "]),
@ -120,16 +119,16 @@ dict_advanced_syntax_input_and_expected_output = {
# Structural Elements # Structural Elements
"se_start": [ "se_start": [
# Matches a start of a structural element # Matches a start of a structural element
("<<div", (sy.StructuralElementStartToken("div"), )), ("<<div", (sy.StructuralElementStartToken(["div"]), )),
], ],
"se_end": [ "se_end": [
("div>>", (sy.StructuralElementEndToken("div"), )), ("div>>", (sy.StructuralElementEndToken(["div"]), )),
], ],
"se": [ "se": [
# Matches a div element, must be at the beginning of the line, the closing div can be with arguments # Matches a div element, must be at the beginning of the line, the closing div can be with arguments
("div>> [class='blue', 123#]{var='test', number=11}", (sy.StructuralElementEndToken(["div", sy.OptionalToken(["[", "class='blue'", ",", "123#", "]","{", "var='test'", ",", "number=11", "}"])]))), # noqa E501 (line too long) ("div>> [class='blue', 123#]{var='test', number=11}", [sy.StructuralElementEndToken(["div", [sy.OptionalToken(["[", "class='blue'", ",", "123#", "]","{", "var='test'", ",", "number=11", "}"])]])]), # noqa E501 (line too long)
("div>> [class='blue', 123#]", (sy.StructuralElementEndToken(["div", sy.OptionalToken(["[", "class='blue'", ",", "123#", "]"])]))), # noqa E501 (line too long) ("div>> [class='blue', 123#]", [sy.StructuralElementEndToken(["div", [sy.OptionalToken(["[", "class='blue'", ",", "123#", "]"])]])]), # noqa E501 (line too long)
("div>> {var='test', number=11}", (sy.StructuralElementEndToken(["div", sy.OptionalToken(["{", "var='test'", ",", "number=11", "}"])]))), # noqa E501 (line too long) ("div>> {var='test', number=11}", [sy.StructuralElementEndToken(["div", [sy.OptionalToken(["{", "var='test'", ",", "number=11", "}"])]])]), # noqa E501 (line too long)
], ],
# List Elements # List Elements
@ -186,6 +185,7 @@ dict_advanced_syntax_input_and_expected_output = {
], ],
} }
# test__add_tag
list_add_tag_input_and_expected_output = [ list_add_tag_input_and_expected_output = [
# Test cases for the add_tag function # Test cases for the add_tag function
# Input: # Input:
@ -200,28 +200,27 @@ list_add_tag_input_and_expected_output = [
(sy.HeaderToken(["#", "Text"]), "<header = '#,Text' />"), (sy.HeaderToken(["#", "Text"]), "<header = '#,Text' />"),
] ]
# test_readable_markup
list_of_text_input_and_readable_output = [ list_of_text_input_and_readable_output = [
# Header # Header
("# Text1 #", "one_header", "<header: '#,Text1 ' />"), ("# Text1 #", "one_header", "<header = '#,Text1 ' />"),
("## Text2 ##", "one_header", "<header: '##,Text2 ' />"), ("## Text2 ##", "one_header", "<header = '##,Text2 ' />"),
# Text # Text
("*Italic*", "et_em", "<em: *>Italic</em :*>"), ("*Italic*", "et_em", "<text:em = '*' />"),
("__Underline__", "et_underline", "<underline: __>Underline</underline :__>"), ("__Underline__", "et_underline", "<text:underline = '__' />"),
("**Bold**", "et_strong", "<strong: **>Bold</strong :**>"), ("**Bold**", "et_strong", "<text:strong = '**' />"),
("~~Strikethrough~~", "et_strikethrough", "<strikethrough: ~~>Strikethrough</strikethrough :~~>"), ("~~Strikethrough~~", "et_strikethrough", "<text:strikethrough = '~~' />"),
("Text *bold __underline__ still bold*", "enhanced_text", "<em: *><strong: **><underline: __><strikethrough: ~~>Text bold underline still bold</strikethrough :~~></underline :__></strong :**></em :*>"), # noqa E501 (line too long) # ("Text *bold __underline__ still bold*", "enhanced_text", "<em: *><strong: **><underline: __><strikethrough: ~~>Text bold underline still bold</strikethrough :~~></underline :__></strong :**></em :*>"), # noqa E501 (line too long)
("[link]('http://www.google.com')", "il_link", "<link: [link](http://www.google.com)>"), ("[link]('http://www.google.com')", "il_link", "<link: [link](http://www.google.com)>"),
# Structural elements # Structural elements
("<<div", "se", "<div: <<div>"), ("<<div", "se", "<se:start = 'div' />"),
("div>>", "se", "<div: div>>>"), ("div>>", "se", "<se:end = 'div' />"),
# Lists # Lists
("- Text", "il_ulist", "<ulist: '-,Text' />"), ("- Text", "one_ulist", "<list:ulist = '-,Text' />"),
("12. Text", "il_olist", "<olist: '12,Text' />"), ("#. Text", "one_olist", "<list:olist = '#,Text' />"),
("# Text", "il_olist", "<olist: '#,Text' />"),
# Tables # Tables
("| Text1 | Text2 |", "table", "<table: | Text1 | Text2 |>"), ("| Text1 | Text2 |", "table", "<table: | Text1 | Text2 |>"),