From a9b9d85ce149dc6f5217dbb8edb2d7b7b73077f5 Mon Sep 17 00:00:00 2001 From: vicky Date: Thu, 17 Mar 2022 17:20:39 +0100 Subject: [PATCH] lots of fixing --- src/bootstraparse/modules/syntax.py | 72 ++++++++--------------------- tests/modules/test_syntax.py | 63 +++++++++++++------------ 2 files changed, 51 insertions(+), 84 deletions(-) diff --git a/src/bootstraparse/modules/syntax.py b/src/bootstraparse/modules/syntax.py index 0a8d327..cd9e342 100644 --- a/src/bootstraparse/modules/syntax.py +++ b/src/bootstraparse/modules/syntax.py @@ -200,15 +200,24 @@ et_em = pp.Literal('*')('em').add_parse_action(of_type(EtEmToken)) et_strong = pp.Literal('**')('strong').add_parse_action(of_type(EtStrongToken)) et_underline = pp.Literal('__')('underline').add_parse_action(of_type(EtUnderlineToken)) et_strikethrough = pp.Literal('~~')('strikethrough').add_parse_action(of_type(EtStrikethroughToken)) -et_custom_span = ('(#' + pp.Word(pp.nums)('span_id') + ')').set_name('custom_span') +et_custom_span = ( + pps('(#') + pp.Word(pp.nums)('span_id') + pps(')') +).set_name('custom_span').add_parse_action(of_type(EtCustomSpanToken)) +markup = et_strong | et_em | et_strikethrough | et_underline | et_custom_span # Multiline elements -se_start = ('<<' + structural_elements).add_parse_action(of_type(StructuralElementStartToken)) -se_end = (structural_elements + '>>').add_parse_action(of_type(StructuralElementEndToken)) +se_start = (pps('<<') + structural_elements).add_parse_action(of_type(StructuralElementStartToken)) +se_end = (structural_elements + pps('>>')).add_parse_action(of_type(StructuralElementEndToken)) se = se_end | se_start # Structural element # Inline elements -il_link = pp.Literal('[') + ... + pp.Literal('](') + quotes + url_characters + pp.match_previous_literal(quotes) + ')' +# [link text]('http://test.icule') +il_link = pp.common.url +# il_link = ( +# pp.Literal('[') + pp.SkipTo('](')('text') + '](' + +# quotes + url_characters('url') + pp.match_previous_literal(quotes) +# + ')' +# ).add_parse_action(of_type(HyperlinkToken)) # Oneline elements one_header = ( @@ -225,7 +234,10 @@ one_ulist = pp.line_start + ( ).add_parse_action(of_type(EtUlistToken)) # Final elements -enhanced_text = pp.OneOrMore(et_em | et_strikethrough | et_strong | et_underline | et_custom_span | text) +enhanced_text = pp.OneOrMore(markup + text) +# pp.SkipTo(markup)('text').add_parse_action(of_type(TextToken)) + markup | +# pp.SkipTo(pp.line_end)("text").add_parse_action(of_type(TextToken)) +# ) ############################################################################## @@ -250,55 +262,11 @@ if __name__ == '__main__': # pragma: no cover pp.autoname_elements() list_strings = [ - "Normal text", - "Normal text *em text*", - "Normal text **strong text**", - "Normal text __underline text__", - "Normal text ~~strikethrough text~~", - "Normal text (#123) custom span text(#123)", - "**Strong text __underline__** normal text", - "Normal text **strong text *em text*** *em text **strong text*** normal text", - "<>", - "[link text]('http://test.icule')", - "# header #", - "1. ordered list", - "- unordered list", + "'http://test.icule'", ] - # div_start.parse_string(list_strings[8]) - # div_end.parse_string(list_strings[9]) - # il_link.parse_string(list_strings[10]) - # one_header.parse_string(list_strings[11]) - # one_olist.parse_string(list_strings[12]) - # one_ulist.parse_string(list_strings[13]) - # enhanced_text.create_diagram("../../../dev_outputs/diagram.html") - # for string in list_strings: - # print('Input string:', string) - # output = enhanced_text.parseString(string) - # # rich.inspect(output) - # print('Output string:', readable_markup(output)) - # print() + rich.inspect(il_link.parse_string(list_strings[0])) if not os.path.exists('../../../dev_outputs/'): os.mkdir('../../../dev_outputs/') - enhanced_text.create_diagram("../../../dev_outputs/diagram.html") - - import timeit - base_time = {} - for string in list_strings: - print('Input string:', string) - n_time = timeit.timeit("enhanced_text.parseString(string)", number=100, globals=globals()) - base_time[string] = n_time - output = enhanced_text.parseString(string) - # rich.inspect(output) - rich.print(f'[] Output string: {readable_markup(output)} in {n_time:.2}s)') - print() - - time = timeit.timeit("enhanced_text.parseString(mega_string)", number=5, globals=globals()) - rich.print(f'[]Mega time: {time}') - base_time['mega_string'] = time - - print('Base time:') - for string, time in base_time.items(): - print(f' "{string}": {time},') + enhanced_text.create_diagram("../../../dev_outputs/diagram.html") \ No newline at end of file diff --git a/tests/modules/test_syntax.py b/tests/modules/test_syntax.py index 21d9e0e..14d3dc0 100644 --- a/tests/modules/test_syntax.py +++ b/tests/modules/test_syntax.py @@ -56,31 +56,31 @@ expressions_to_match = { ############################################################################################################## # Testing the parser lexical elements +# test_advanced_expression_and_token_creation dict_advanced_syntax_input_and_expected_output = { # Enhanced text # TODO : Test optional inline args "et_em": [ # Single em element, should only match the first "*" - ("*", (sy.EtEmToken("*"), )), + ("*", (sy.EtEmToken(["*"]), )), ], "et_strong": [ # Single strong element, should only match the first "**" - ("**", sy.EtStrongToken("**"), ), + ("**", (sy.EtStrongToken(["**"]), )), ], "et_underline": [ # Single underline element, should only match the first "__" - ("__", (sy.EtUnderlineToken("__"), )), + ("__", (sy.EtUnderlineToken(["__"]), )), ], "et_strikethrough": [ # Single strikethrough element, should only match the first "~~" - ("~~", (sy.EtStrikethroughToken("~~"), )), + ("~~", (sy.EtStrikethroughToken(["~~"]), )), ], "et_custom_span": [ # Single custom span element, should only match the first "(#number)" - ("(#12345)", (sy.EtCustomSpanToken("#12345"), )), + ("(#12345)", (sy.EtCustomSpanToken(["12345"]), )), ], "il_link": [ # Single link element, should only match the first "[link_name](link)" - # TODO: Text tokens ? or Link token and hyperlink token ? ("[text_link]('text://www.website.com/link.html')", sy.HyperlinkToken("[text_link]('text://www.website.com/link.html')")) ], @@ -92,7 +92,7 @@ dict_advanced_syntax_input_and_expected_output = { ("**test**", (sy.EtStrongToken(["**"]), sy.TextToken(["test"]), sy.EtStrongToken(["**"]))), ("__test__", (sy.EtUnderlineToken(["__"]), sy.TextToken(["test"]), sy.EtUnderlineToken(["__"]))), ("~~test~~", (sy.EtStrikethroughToken(["~~"]), sy.TextToken(["test"]), sy.EtStrikethroughToken(["~~"]))), - ("(#12345)", (sy.EtCustomSpanToken(["#12345"]), )), + ("(#12345)", (sy.EtCustomSpanToken(["12345"]), )), ("*test*test*", (sy.EtEmToken(["*"]), sy.TextToken(["test"]), sy.EtEmToken(["*"]), sy.TextToken(["test"]), sy.EtEmToken(["*"]))), ("**test**test**", (sy.EtStrongToken(["**"]), sy.TextToken(["test"]), sy.EtStrongToken(["**"]), sy.TextToken(["test"]), @@ -101,13 +101,12 @@ dict_advanced_syntax_input_and_expected_output = { sy.TextToken(["test"]), sy.EtUnderlineToken(["__"]), )), ("~~test~~test~~", (sy.EtStrikethroughToken(["~~"]), sy.TextToken(["test"]), sy.EtStrikethroughToken(["~~"]), sy.TextToken(["test"]), sy.EtStrikethroughToken(["~~"]), )), - ("(#12345)test(#12345)", (sy.EtCustomSpanToken(["#12345"]), sy.TextToken(["test"]), - sy.EtCustomSpanToken(["#12345"]), )), - ("__test(#12)test**test__test~~", (sy.EtUnderlineToken("__"), sy.TextToken(["test"]), - sy.EtCustomSpanToken(["#12"]), sy.TextToken(["test"]), sy.EtStrongToken(["**"]), - sy.TextToken(["test"]), sy.EtStrikethroughToken(["~~"]), - sy.TextToken(["test"]), sy.EtUnderlineToken(["__"]), )), - ("#. List of elements", (sy.EtUlistToken(["#", "List of elements"]), )), + ("(#12345)test(#12345)", (sy.EtCustomSpanToken(["12345"]), sy.TextToken(["test"]), + sy.EtCustomSpanToken(["12345"]), )), + ("__test(#12)test**test__test~~", (sy.EtUnderlineToken(["__"]), sy.TextToken(["test"]), + sy.EtCustomSpanToken(["12"]), sy.TextToken(["test"]), sy.EtStrongToken(["**"]), + sy.TextToken(["test"]), sy.EtUnderlineToken(["__"]), + sy.TextToken(["test"]), sy.EtStrikethroughToken(["~~"]), )), ("Test text with a link [link_name](link)", (sy.TextToken(["Test text with a link "]), sy.HyperlinkToken(["link_name", "link"]), )), ("Test text with a link [link_name](link) and a *bold* text", (sy.TextToken(["Test text with a link "]), @@ -120,16 +119,16 @@ dict_advanced_syntax_input_and_expected_output = { # Structural Elements "se_start": [ # Matches a start of a structural element - ("<>", (sy.StructuralElementEndToken("div"), )), + ("div>>", (sy.StructuralElementEndToken(["div"]), )), ], "se": [ # Matches a div element, must be at the beginning of the line, the closing div can be with arguments - ("div>> [class='blue', 123#]{var='test', number=11}", (sy.StructuralElementEndToken(["div", sy.OptionalToken(["[", "class='blue'", ",", "123#", "]","{", "var='test'", ",", "number=11", "}"])]))), # noqa E501 (line too long) - ("div>> [class='blue', 123#]", (sy.StructuralElementEndToken(["div", sy.OptionalToken(["[", "class='blue'", ",", "123#", "]"])]))), # noqa E501 (line too long) - ("div>> {var='test', number=11}", (sy.StructuralElementEndToken(["div", sy.OptionalToken(["{", "var='test'", ",", "number=11", "}"])]))), # noqa E501 (line too long) + ("div>> [class='blue', 123#]{var='test', number=11}", [sy.StructuralElementEndToken(["div", [sy.OptionalToken(["[", "class='blue'", ",", "123#", "]","{", "var='test'", ",", "number=11", "}"])]])]), # noqa E501 (line too long) + ("div>> [class='blue', 123#]", [sy.StructuralElementEndToken(["div", [sy.OptionalToken(["[", "class='blue'", ",", "123#", "]"])]])]), # noqa E501 (line too long) + ("div>> {var='test', number=11}", [sy.StructuralElementEndToken(["div", [sy.OptionalToken(["{", "var='test'", ",", "number=11", "}"])]])]), # noqa E501 (line too long) ], # List Elements @@ -186,6 +185,7 @@ dict_advanced_syntax_input_and_expected_output = { ], } +# test__add_tag list_add_tag_input_and_expected_output = [ # Test cases for the add_tag function # Input: @@ -200,28 +200,27 @@ list_add_tag_input_and_expected_output = [ (sy.HeaderToken(["#", "Text"]), "
"), ] - +# test_readable_markup list_of_text_input_and_readable_output = [ # Header - ("# Text1 #", "one_header", ""), - ("## Text2 ##", "one_header", ""), + ("# Text1 #", "one_header", "
"), + ("## Text2 ##", "one_header", "
"), # Text - ("*Italic*", "et_em", "Italic"), - ("__Underline__", "et_underline", "Underline"), - ("**Bold**", "et_strong", "Bold"), - ("~~Strikethrough~~", "et_strikethrough", "Strikethrough"), - ("Text *bold __underline__ still bold*", "enhanced_text", "Text bold underline still bold"), # noqa E501 (line too long) + ("*Italic*", "et_em", ""), + ("__Underline__", "et_underline", ""), + ("**Bold**", "et_strong", ""), + ("~~Strikethrough~~", "et_strikethrough", ""), + # ("Text *bold __underline__ still bold*", "enhanced_text", "Text bold underline still bold"), # noqa E501 (line too long) ("[link]('http://www.google.com')", "il_link", ""), # Structural elements - ("<"), - ("div>>", "se", ">>"), + ("<"), + ("div>>", "se", ""), # Lists - ("- Text", "il_ulist", ""), - ("12. Text", "il_olist", ""), - ("# Text", "il_olist", ""), + ("- Text", "one_ulist", ""), + ("#. Text", "one_olist", ""), # Tables ("| Text1 | Text2 |", "table", ""),