diff --git a/src/bootstraparse/modules/syntax.py b/src/bootstraparse/modules/syntax.py index cd9e342..d69667e 100644 --- a/src/bootstraparse/modules/syntax.py +++ b/src/bootstraparse/modules/syntax.py @@ -1,7 +1,9 @@ # Dedicated module for the syntax of all the parsing import os +from itertools import zip_longest + import pyparsing as pp -import rich +# import rich pps = pp.Suppress @@ -16,6 +18,20 @@ class SemanticType: def __init__(self, content): self.content = content + def to_markup(self): + """ + Function used for testing and readability purposes. Replaces matched markup with html-like tags. + """ + if self.content: + return_list = [] + for elt in self.content: + try: + return_list.append(elt.to_markup()) + except Exception: + return_list.append(str(elt)) + return f"<{self.label} = '{','.join(return_list)}' />" + return f'<[NOC] {self.label} />' + def __str__(self): if type(self.content) == str: return f'{self.label}[{self.content}]' @@ -27,13 +43,32 @@ class SemanticType: def __eq__(self, other): if type(other) == type(self): - for e1, e2 in zip(self.content, other.content): + for e1, e2 in zip_longest(self.content, other.content): if e1 != e2: return False return True return False +class ExplicitSemanticType(SemanticType): + """ + Explicit semantic type, the label is the only information we need. + """ + def to_markup(self): + return f'<{self.label} />' + + +class EmptySemanticType(SemanticType): + """ + Empty semantic type, the content is the only information we need. + """ + def to_markup(self): + if type(self.content) == str: + return f'{self.content}' + else: + return " ".join(map(str, self.content)) + + class UnimplementedToken(SemanticType): label = "unimplemented" @@ -46,27 +81,27 @@ class ImageToken(SemanticType): label = "image" -class TextToken(SemanticType): +class TextToken(EmptySemanticType): label = "text" -class EnhancedToken(SemanticType): +class EnhancedToken(ExplicitSemanticType): label = "text:enhanced" -class EtEmToken(SemanticType): +class EtEmToken(ExplicitSemanticType): label = "text:em" -class EtStrongToken(SemanticType): +class EtStrongToken(ExplicitSemanticType): label = "text:strong" -class EtUnderlineToken(SemanticType): +class EtUnderlineToken(ExplicitSemanticType): label = "text:underline" -class EtStrikethroughToken(SemanticType): +class EtStrikethroughToken(ExplicitSemanticType): label = "text:strikethrough" @@ -141,25 +176,16 @@ def readable_markup(list_of_tokens): :return: returns readable string. :rtype: string """ - readable_string = '' + readable_list = [] for token in list_of_tokens: if type(token) == str: - readable_string += token + readable_list.append(token) else: - readable_string += _add_tag(token) - return readable_string - - -def _add_tag(token): - """ - Function used for testing and readability purposes. Replaces matched markup with html-like tags. - :param token: matched markup token in analysed text. - :return: returns readable string. - """ - if token.label == 'text': - return token.content[0] - if token.content: - return "<{} = '{}' />".format(token.label, ",".join(token.content)) + try: + readable_list.append(token.to_markup()) + except Exception: + readable_list.append(str(token)) + return " ".join(readable_list) # Base elements @@ -194,6 +220,10 @@ optional = ( pp.Opt(html_insert)("html_insert") + pp.Opt(var)("var") )("optional").add_parse_action(of_type(OptionalToken)) +# Inline elements +il_link = pp.Regex( + r"""\[(?P.+)\]\(['"](?P[a-zA-Z-_:\/=@#!%\?\d\(\)\.]+)['"]\)""" +).add_parse_action(of_type(HyperlinkToken)) # Enhanced text elements et_em = pp.Literal('*')('em').add_parse_action(of_type(EtEmToken)) @@ -203,22 +233,15 @@ et_strikethrough = pp.Literal('~~')('strikethrough').add_parse_action(of_type(Et et_custom_span = ( pps('(#') + pp.Word(pp.nums)('span_id') + pps(')') ).set_name('custom_span').add_parse_action(of_type(EtCustomSpanToken)) -markup = et_strong | et_em | et_strikethrough | et_underline | et_custom_span + +# markup sums up all in-line elements +markup = il_link | et_strong | et_em | et_strikethrough | et_underline | et_custom_span # Multiline elements se_start = (pps('<<') + structural_elements).add_parse_action(of_type(StructuralElementStartToken)) se_end = (structural_elements + pps('>>')).add_parse_action(of_type(StructuralElementEndToken)) se = se_end | se_start # Structural element -# Inline elements -# [link text]('http://test.icule') -il_link = pp.common.url -# il_link = ( -# pp.Literal('[') + pp.SkipTo('](')('text') + '](' + -# quotes + url_characters('url') + pp.match_previous_literal(quotes) -# + ')' -# ).add_parse_action(of_type(HyperlinkToken)) - # Oneline elements one_header = ( header_element + pp.SkipTo(pp.match_previous_literal(header_element)) @@ -234,10 +257,9 @@ one_ulist = pp.line_start + ( ).add_parse_action(of_type(EtUlistToken)) # Final elements -enhanced_text = pp.OneOrMore(markup + text) -# pp.SkipTo(markup)('text').add_parse_action(of_type(TextToken)) + markup | -# pp.SkipTo(pp.line_end)("text").add_parse_action(of_type(TextToken)) -# ) +enhanced_text = pp.ZeroOrMore( + markup | pp.SkipTo(markup)('text').add_parse_action(of_type(TextToken)) + markup +) + pp.Opt(pp.rest_of_line("text").add_parse_action(of_type(TextToken))) ############################################################################## @@ -253,7 +275,7 @@ line_to_replace = pp.OneOrMore( pp.SkipTo(image ^ alias)('text').add_parse_action(of_type(TextToken)) ^ image.add_parse_action(of_type(ImageToken)) ^ alias.add_parse_action(of_type(AliasToken)) -) ^ pp.SkipTo(pp.lineEnd)('text').add_parse_action(of_type(TextToken)) +) ^ pp.rest_of_line('text').add_parse_action(of_type(TextToken)) ############################################################################## # Temporary tests @@ -261,12 +283,6 @@ line_to_replace = pp.OneOrMore( if __name__ == '__main__': # pragma: no cover pp.autoname_elements() - list_strings = [ - "'http://test.icule'", - ] - - rich.inspect(il_link.parse_string(list_strings[0])) - if not os.path.exists('../../../dev_outputs/'): os.mkdir('../../../dev_outputs/') - enhanced_text.create_diagram("../../../dev_outputs/diagram.html") \ No newline at end of file + enhanced_text.create_diagram("../../../dev_outputs/diagram.html") diff --git a/tests/modules/test_syntax.py b/tests/modules/test_syntax.py index d9bd192..d2302c5 100644 --- a/tests/modules/test_syntax.py +++ b/tests/modules/test_syntax.py @@ -58,7 +58,7 @@ expressions_to_match = { # Testing the parser lexical elements # test_advanced_expression_and_token_creation dict_advanced_syntax_input_and_expected_output = { - # Enhanced text # TODO : Test optional inline args + # Enhanced text "et_em": [ # Single em element, should only match the first "*" ("*", (sy.EtEmToken(["*"]), )), @@ -80,9 +80,9 @@ dict_advanced_syntax_input_and_expected_output = { ("(#12345)", (sy.EtCustomSpanToken(["12345"]), )), ], "il_link": [ - # Single link element, should only match the first "[link_name](link)" + # Single link element, should only match the first "[link_name]('link')" ("[text_link]('text://www.website.com/link.html')", - sy.HyperlinkToken("[text_link]('text://www.website.com/link.html')")) + [sy.HyperlinkToken(["[text_link]('text://www.website.com/link.html')"])]) ], # Enhanced Text @@ -107,12 +107,11 @@ dict_advanced_syntax_input_and_expected_output = { sy.EtCustomSpanToken(["12"]), sy.TextToken(["test"]), sy.EtStrongToken(["**"]), sy.TextToken(["test"]), sy.EtUnderlineToken(["__"]), sy.TextToken(["test"]), sy.EtStrikethroughToken(["~~"]), )), - ("Test text with a link [link_name](link)", (sy.TextToken(["Test text with a link "]), - sy.HyperlinkToken(["link_name", "link"]), )), - ("Test text with a link [link_name](link) and a *bold* text", (sy.TextToken(["Test text with a link "]), - sy.HyperlinkToken(["link_name", "link"]), - sy.TextToken([" and a "]), sy.EtEmToken(["*"]), - sy.TextToken(["bold"]), sy.EtEmToken(["*"]), )), + ("Test text with a link [link_name]('link')", (sy.TextToken(["Test text with a link"]), sy.HyperlinkToken(["[link_name]('link')"]), )), # noqa E501 (line too long) + ("Test text with a link [link_name](\"link\") and a *bold* text", (sy.TextToken(["Test text with a link"]), + sy.HyperlinkToken(['[link_name]("link")']), + sy.TextToken(["and a"]), sy.EtEmToken(["*"]), + sy.TextToken(["bold"]), sy.EtEmToken(["*"]), )), ], @@ -190,10 +189,10 @@ list_add_tag_input_and_expected_output = [ # Test cases for the add_tag function # Input: (sy.TextToken(["Text"]), "Text"), - (sy.EtEmToken(["*"]), ""), - (sy.EtStrongToken(["**"]), ""), - (sy.EtUnderlineToken(["__"]), ""), - (sy.EtStrikethroughToken(["~~"]), ""), + (sy.EtEmToken(["*"]), ""), + (sy.EtStrongToken(["**"]), ""), + (sy.EtUnderlineToken(["__"]), ""), + (sy.EtStrikethroughToken(["~~"]), ""), (sy.StructuralElementStartToken(["div"]), ""), (sy.StructuralElementEndToken(["div"]), ""), (sy.EtCustomSpanToken(["(#12)"]), ""), @@ -207,13 +206,12 @@ list_of_text_input_and_readable_output = [ ("## Text2 ##", "one_header", "
"), # Text - ("*Italic*", "et_em", ""), - ("__Underline__", "et_underline", ""), - ("**Bold**", "et_strong", ""), - ("~~Strikethrough~~", "et_strikethrough", ""), - ("Text *bold __underline__ still bold*", "enhanced_text", "Text bold underline still bold"), # noqa E501 (line too long) - - ("[link]('http://www.google.com')", "il_link", ""), + ("*Italic*", "et_em", ""), + ("__Underline__", "et_underline", ""), + ("**Bold**", "et_strong", ""), + ("~~Strikethrough~~", "et_strikethrough", ""), + ("Text *bold __underline__ still bold*", "enhanced_text", "Text bold underline still bold "), # noqa E501 (line too long) + ("[link]('http://www.google.com')", "il_link", ""), # Structural elements ("<"), @@ -291,6 +289,33 @@ def test_semantic_type(): assert strings_in_token(st, ["[1, 2, 3]", "test"]) +def test_semantic_type_eq(): + """ + Test that the semantic type is correctly created. + """ + st1 = sy.SemanticType("test") + st2 = sy.SemanticType("test") + assert st1 == st2 + st1 = sy.SemanticType(["test"]) + st2 = sy.SemanticType(["test2"]) + assert st1 != st2 + st1 = sy.SemanticType("test") + st2 = sy.SemanticType([1, 2, 3]) + assert st1 != st2 + st1 = sy.SemanticType([1, 2, 3]) + st2 = sy.SemanticType([1, 2, 3]) + assert st1 == st2 + st1 = sy.SemanticType([1, 2, 3]) + st2 = sy.SemanticType([1, 2, 3, 4]) + assert st1 != st2 + st1 = sy.SemanticType([1, 2, 3, 4]) + st2 = sy.SemanticType([1, 2, 3]) + assert st1 != st2 + st1 = sy.SemanticType([1, 2, 3]) + st2 = sy.SemanticType([1, 2, 3, 4]) + assert st1 != st2 + + def find_expression_from_str(expression_str): """ Find the expression from its string representation. @@ -354,7 +379,7 @@ def test__add_tag(token, expected_output): :type token: sy.SemanticType :type expected_output: str """ - assert sy._add_tag(token) == expected_output + assert token.to_markup() == expected_output # Test text functions @@ -374,3 +399,18 @@ def test_readable_markup(token_list, parsing_expression, expected): result = expr.parse_string(token_list) assert result is not None assert sy.readable_markup(result) == expected + + +def test_weird_markup(): + """ + Test that the readable markup is correctly generated. + """ + assert sy.readable_markup([sy.EmptySemanticType("weird")]) == "weird" + empty_token = sy.SemanticType(None) + empty_token.label = "weird" + assert sy.readable_markup([empty_token]) == "<[NOC] weird />" + + class WeirdClass: + def __str__(self): + return "weird" + assert sy.readable_markup([sy.EmptySemanticType("weird"), empty_token, WeirdClass()]) == "weird <[NOC] weird /> weird"