From 98e86d2626bdcbfe6e47b283e10a0fc93d663478 Mon Sep 17 00:00:00 2001 From: Maximilien Rigaut Date: Tue, 7 Jun 2022 23:13:09 +0200 Subject: [PATCH] First steps for codetoken --- src/bootstraparse/modules/syntax.py | 15 ++++++++++++++- tests/modules/test_syntax.py | 5 +++++ tests/modules/test_syntax_context.py | 1 + tests/modules/test_syntax_to_original.py | 16 ++++++++++++++++ 4 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 tests/modules/test_syntax_to_original.py diff --git a/src/bootstraparse/modules/syntax.py b/src/bootstraparse/modules/syntax.py index d3dd682..a3bdee8 100644 --- a/src/bootstraparse/modules/syntax.py +++ b/src/bootstraparse/modules/syntax.py @@ -48,6 +48,12 @@ class SemanticType: return f"<{self.label} = '{','.join(return_list)}' />" return f'<[NOC] {self.label} />' + def to_original(self): + """ + Return the original string of the token. + """ + return " ".join(self.content) + def __str__(self): if type(self.content) == str: return f'{self.label}[{self.content}]' @@ -305,6 +311,10 @@ class BlockQuoteAuthorToken(SemanticType): label = "bq:author" +class CodeToken(ExplicitSemanticType, TokensToMatch): + label = "code" + + class Linebreak(ExplicitSemanticType): label = "linebreak" @@ -455,8 +465,11 @@ et_custom_span = ( pps('(#') + pp.Word(pp.nums)('span_id') + pps(')') ).set_name('custom_span').add_parse_action(of_type(EtCustomSpanToken)) +# Code Token +code = pp.Literal('```')('code').add_parse_action(of_type(CodeToken)) + # markup sums up all in-line elements -markup = il_link | et_strong | et_em | et_strikethrough | et_underline | et_custom_span +markup = il_link | et_strong | et_em | et_strikethrough | et_underline | et_custom_span | code # not quite correct but good enough for now # noqa : E501 enhanced_text = pp.ZeroOrMore( markup | pp.SkipTo(markup)('text').add_parse_action(of_type(TextToken)) + markup ) + pp.Opt(pp.Regex(r'.+')("text").add_parse_action(of_type(TextToken))) diff --git a/tests/modules/test_syntax.py b/tests/modules/test_syntax.py index a3e8ad4..f713087 100644 --- a/tests/modules/test_syntax.py +++ b/tests/modules/test_syntax.py @@ -551,6 +551,7 @@ dict_advanced_syntax_input_and_expected_output = { ]), ]), ), __GL(),), + ("```Code```", (sy.CodeToken(["```"]), sy.TextToken(["Code"]), sy.CodeToken(["```"])), __GL()), ], "blockquote": [ ("> Text", (sy.BlockQuoteToken([sy.TextToken(["Text"])]),), __GL()), @@ -590,6 +591,10 @@ dict_advanced_syntax_input_and_expected_output = { sy.BlockQuoteAuthorToken(["Author Name"]), ), __GL()), ], + "code": [ + ("```\nText\n```", (sy.CodeToken(["```"]),), __GL()), + ("```\nText\nText\n```", (sy.CodeToken(["```"]),), __GL()), + ], } # Cursed zipping oneline diff --git a/tests/modules/test_syntax_context.py b/tests/modules/test_syntax_context.py index bfd228f..928308c 100644 --- a/tests/modules/test_syntax_context.py +++ b/tests/modules/test_syntax_context.py @@ -32,6 +32,7 @@ _type_counterpart_container = [ (syntax.TableSeparatorToken, None, "error", tools.__GLk(1)), (syntax.BlockQuoteToken, None, "error", tools.__GLk(1)), (syntax.BlockQuoteAuthorToken, None, "error", tools.__GLk(1)), + (syntax.CodeToken, "code", "token", tools.__GLk(1)), (syntax.Linebreak, "linebreak", "self", tools.__GLk(1)), ] diff --git a/tests/modules/test_syntax_to_original.py b/tests/modules/test_syntax_to_original.py new file mode 100644 index 0000000..490fda8 --- /dev/null +++ b/tests/modules/test_syntax_to_original.py @@ -0,0 +1,16 @@ +import pytest + +from bootstraparse.modules import syntax, error_mngr, tools, context_mngr +from bootstraparse.modules.tools import __GL, __module_path, find_variables_in_file + +__XF = pytest.mark.xfail + +_list_original_strings = [ + "string", +] + + +def test_to_original(): + """Test the .to_original() method of SemanticTypes.""" + token = syntax.SemanticType(["test"]) + assert token.to_original() == 'test'