diff --git a/src/bootstraparse/modules/syntax.py b/src/bootstraparse/modules/syntax.py index b0c83fc..9f432d5 100644 --- a/src/bootstraparse/modules/syntax.py +++ b/src/bootstraparse/modules/syntax.py @@ -98,10 +98,6 @@ class TableRowToken(SemanticType): label = "table:row" -class TableHeaderToken(SemanticType): - label = "table:header" - - class TableCellToken(SemanticType): label = "table:cell" @@ -110,10 +106,6 @@ class TableSeparatorToken(SemanticType): label = "table:separator" -class TableSeparatorToken(SemanticType): - label = "table:separator" - - def of_type(token_class): """ Function creating a custom function for generating the given Token type. @@ -185,6 +177,7 @@ structural_elements = (pp.Word('div') ^ pp.Word('article') ^ pp.Word('aside') ^ # Optional elements optional = (pp.Opt(html_insert)("html_insert") + pp.Opt(var)("var"))("optional") +# TODO : Everywhere : Avoid text at all costs since it is not a valid token, make a new token for it or use pp.SkipTo or (...) # Enhanced text elements # TODO: Add all markups so that they can be parsed, they no longer need to be linked # et_em = ('*' + enhanced_text + '*')('em').add_parse_action(of_type(EtEmToken)) # et_strong = ('**' + enhanced_text + '**')('strong').add_parse_action(of_type(EtStrongToken)) @@ -194,11 +187,12 @@ optional = (pp.Opt(html_insert)("html_insert") + pp.Opt(var)("var"))("optional") # et_custom_span = \ # (custom_span + enhanced_text + pp.match_previous_literal(custom_span))('custom_span')\ # .add_parse_action(of_type(EtCustomSpanToken)) -# enhanced_text <<= (text | et_strong | et_em | et_underline | et_strikethrough | et_custom_span) + pp.Opt(enhanced_text) +# enhanced_text <== (text | et_strong | et_em | et_underline | +# et_strikethrough | et_custom_span) + pp.Opt(enhanced_text) # Multiline elements -se_start = '~~' + structural_elements -se_end = text + '~~' +se_start = '~~' + structural_elements # TODO : It's not a text, rather a keyword and add ("name") and .add_parse_action(of_type(TextToken)) +se_end = text + '~~' # TODO : Idem # Inline elements il_link = '[' + text + ']' + '(' + quotes + http_characters + pp.match_previous_literal(quotes) + ')' diff --git a/tests/modules/test_syntax.py b/tests/modules/test_syntax.py index 3424070..a7944f8 100644 --- a/tests/modules/test_syntax.py +++ b/tests/modules/test_syntax.py @@ -3,13 +3,13 @@ import pytest import bootstraparse.modules.syntax as sy -# Dictionary of all label and associated token used in the language +# Dictionary of all label and associated token used in the Pre_parser list_of_token_types = { "alias": sy.AliasToken, "image": sy.ImageToken, } -# Dictionary of all lexical elements and a list of matching strings +# Dictionary of all lexical elements and a list of matching strings for the Pre_parser expressions_to_match = { # Base elements "quotes": ["'hi, there'", "'hi, there'"], @@ -48,30 +48,39 @@ expressions_to_match = { } +# Testing the parser lexical elements dict_advanced_syntax_input_and_expected_output = { # Enhanced text # TODO : Test optional inline args "et_em": [ + # Single em element, should only match the first "*" ("*", (sy.EtEmToken("*"), )), ], "et_strong": [ + # Single strong element, should only match the first "**" ("**", sy.EtEmToken("**"), ), ], "et_underline": [ + # Single underline element, should only match the first "__" ("__", (sy.EtUnderlineToken("__"), )), ], "et_strikethrough": [ + # Single strikethrough element, should only match the first "~~" ("~~", (sy.EtStrikethroughToken("~~"), )), ], "et_custom_span": [ + # Single custom span element, should only match the first "(#number)" ("(#12345)", (sy.EtCustomSpanToken("#12345"), )), ], "il_link": [ + # Single link element, should only match the first "[link_name](link)" + # TODO: Text tokens ? or Link token and hypelink token ? ("[text_link]('text://www.website.com/link.html')", - sy.UnimplementedToken("[text_link]('text://www.website.com/link.html')")) + sy.HyperlinkToken("[text_link]('text://www.website.com/link.html')")) ], # Enhanced Text "enhanced_text": [ + # Matches a line of text, with or without inline elements ("*test*", (sy.EtEmToken(["*"]), sy.TextToken(["test"]), sy.EtEmToken(["*"]))), ("**test**", (sy.EtEmToken(["**"]), sy.TextToken(["test"]), sy.EtEmToken(["**"]))), ("__test__", (sy.EtUnderlineToken(["__"]), sy.TextToken(["test"]), sy.EtUnderlineToken(["__"]))), @@ -92,25 +101,35 @@ dict_advanced_syntax_input_and_expected_output = { sy.TextToken(["test"]), sy.EtStrikethroughToken(["~~"]), sy.TextToken(["test"]), sy.EtUnderlineToken(["__"]), )), ("12. List", (sy.EtUlistToken(["12."]), sy.TextToken([" List"]), )), - + ("Test text with a link [link_name](link)", (sy.TextToken(["Test text with a link "]), + sy.HyperlinkToken(["link_name", "link"]), )), + ("Test text with a link [link_name](link) and a *bold* text", (sy.TextToken(["Test text with a link "]), + sy.HyperlinkToken(["link_name", "link"]), + sy.TextToken([" and a "]), sy.EtEmToken(["*"]), + sy.TextToken(["bold"]), sy.EtEmToken(["*"]), )), ], # Divs "et_div": [ - ("<>", (sy.TextToken("div"), sy.UnimplementedToken(">>"))), + # Matches a div element, must be at the beginning of the line, the closing div can be with arguments + # TODO: Implement a specific div token + ("<>", (sy.UnimplementedToken(["div", ">>"]))), ], # Elements "one_olist": [ + # Matches a one-level ordered list element, must be at the beginning of the line ("12. Text", (sy.TextToken(["12.", sy.TextToken(["Text"])]), )), ], "one_ulist": [ + # Matches a one-level unordered list element, must be at the beginning of the line ("- Text", [sy.EtUlistToken(["-", sy.TextToken(["Text"])]), ]), ], # Headers "one_header": [ + # Matches a one-level header element, must be at the beginning of the line # TODO: Split into different tokens ? ("# Text1 #", (sy.HeaderToken(["#", "Text1"]), )), ("## Text2 ##", (sy.HeaderToken(["##", "Text2"]), )), ("### Text3 ###", (sy.HeaderToken(["###", "Text3"]), )), @@ -119,6 +138,20 @@ dict_advanced_syntax_input_and_expected_output = { ("###### Text6 ######", (sy.HeaderToken(["######", "Text6"]), )), ], # Tables + "table_row": [ + # Matches a table element, must be at the beginning of the line # TODO: add TableCellToken? + ("| Text1 | Text2 |", (sy.TableRowToken(["|", "Text1", "|", "Text2", "|"]), )), + ("| Text1 | Text2 | Text3 |", (sy.TableRowToken(["|", "Text1", "|", "Text2", "|", "Text3", "|"]), )), + ("|2 Text1 | Text2 |", (sy.TableRowToken(["|2", "Text1", "|", "Text2", "|"]), )), + ("|2 Text1 |3 Text2 |", (sy.TableRowToken(["|2", "Text1", "|3", "Text2", "|"]), )), + ("|2 Text1 |3 Text2 |4 Text3 |", (sy.TableRowToken(["|2", "Text1", "|3", "Text2", "|4", "Text3", "|"]), )), + ], + "table_separator": [ + # Matches a table element, must be at the beginning of the line # TODO: Tokens ? + ("|---|---|", (sy.TableSeparatorToken(["|", "---", "|", "---", "|"]), )), + ("|---|---|---|", (sy.TableSeparatorToken(["|", "---", "|", "---", "|", "---", "|"]), )), + ("|:--|-:-|--:|--:|", (sy.TableSeparatorToken(["|", ":--", "|", "-:-", "|", "--:", "|", "--:", "|"]), )), + ], }