diff --git a/src/bootstraparse/modules/syntax.py b/src/bootstraparse/modules/syntax.py index 5aa1de7..a1a7684 100644 --- a/src/bootstraparse/modules/syntax.py +++ b/src/bootstraparse/modules/syntax.py @@ -27,7 +27,7 @@ class SemanticType: for elt in self.content: try: return_list.append(elt.to_markup()) - except Exception: + except Exception: # noqa E722 (This function is used for testing only) return_list.append(str(elt)) return f"<{self.label} = '{','.join(return_list)}' />" return f'<[NOC] {self.label} />' @@ -204,8 +204,8 @@ def reparse(parse_element): :param parse_element: pp object defining desired match :return: returns a function parsing given match with specified parsing element """ - def _reparse(original_string, _, __): - return parse_element.parse_string(original_string) + def _reparse(__, _, tokens): + return parse_element.parseString(tokens[0]) # TODO: fix this with a proper solution return _reparse @@ -224,7 +224,7 @@ def readable_markup(list_of_tokens): else: try: readable_list.append(token.to_markup()) - except Exception: + except Exception: # noqa E722 (This function is used for testing and readability purposes) readable_list.append(str(token)) return " ".join(readable_list) @@ -322,12 +322,12 @@ one_display = ( ).add_parse_action(of_type(DisplayToken)) one_olist = pp.line_start + ( pps(pp.Literal('#.')) + ( - (pp.SkipTo(optional)('text').add_parse_action(of_type(TextToken)) + optional) | # TODO: add reparse + (pp.SkipTo(optional)('text').add_parse_action(reparse(enhanced_text)) + optional) | enhanced_text) ).add_parse_action(of_type(EtOlistToken)) one_ulist = pp.line_start + ( pps(pp.Literal('-')) + ( - (pp.SkipTo(optional)('text').add_parse_action(of_type(TextToken)) + optional) | # TODO: add reparse + (pp.SkipTo(optional)('text').add_parse_action(reparse(enhanced_text)) + optional) | enhanced_text) ).add_parse_action(of_type(EtUlistToken)) diff --git a/tests/modules/test_syntax.py b/tests/modules/test_syntax.py index 106b07f..8e14adf 100644 --- a/tests/modules/test_syntax.py +++ b/tests/modules/test_syntax.py @@ -501,6 +501,31 @@ dict_advanced_syntax_input_and_expected_output = { ("|---|---|", (sy.TableSeparatorToken(["---", "---"]),), __GL()), ("- Text", (sy.EtUlistToken([sy.TextToken(["Text"])]),), __GL()), ("div>>", (sy.StructuralElementEndToken(["div"]),), __GL()), + ("#. Text with *em* and __underline__ and {arguments}{{super-arguments}}[a=1,'ui']", ( + sy.EtOlistToken([ + sy.TextToken(["Text with"]), + sy.EtEmToken(["*"]), + sy.TextToken(["em"]), + sy.EtEmToken(["*"]), + sy.TextToken(["and"]), + sy.EtUnderlineToken(["__"]), + sy.TextToken(["underline"]), + sy.EtUnderlineToken(["__"]), + sy.TextToken(["and"]), + sy.OptionalToken([ + sy.OptionalInsertToken([ + "arguments", + ]), + sy.OptionalClassToken([ + "super-arguments", + ]), + sy.OptionalVarToken([ + sy.BeAssignToken([["a", 1]]), + sy.BeValueToken(["ui"]), + ]), + ]), + ]), + ), __GL(),), ], } @@ -802,11 +827,19 @@ def test_reparse(original_string, reparse_with, expected_output, line_test): print(f"With expression: '{reparse_with}'") print("Defined at %(filename)s:%(lineno)d" % {'filename': __syntax_file, 'lineno': __definition_of_syntax_elements[reparse_with]}) - print(f"Expected: {expected_output}") + print(f"Expected: {expected_output} (len:{len(expected_output)})") expression = find_expression_from_str(str(reparse_with)) - reparse_action = pyparsing.Word(pyparsing.alphanums).add_parse_action(sy.reparse(expression)) + reparse_action = pyparsing.Word(pyparsing.printables+" ").add_parse_action(sy.reparse(expression)) reparse_result = reparse_action.parseString(str(original_string)) + print(f"Found: {reparse_result} (len:{len(reparse_result)}).") + assert reparse_result is not None for expected, result in zip_longest(expected_output, reparse_result): - assert expected == result + assert result == expected + + +def test_empty_token(): + """Test that empty tokens are correctly dropped.""" + + assert sy.of_type(sy.EmptySemanticType)("", [], []) is None