First steps for codetoken

This commit is contained in:
Maximilien Rigaut 2022-06-07 23:13:09 +02:00
parent 3e910f5c95
commit 98e86d2626
No known key found for this signature in database
GPG Key ID: 4F14B23709ADA5B1
4 changed files with 36 additions and 1 deletions

View File

@ -48,6 +48,12 @@ class SemanticType:
return f"<{self.label} = '{','.join(return_list)}' />"
return f'<[NOC] {self.label} />'
def to_original(self):
"""
Return the original string of the token.
"""
return " ".join(self.content)
def __str__(self):
if type(self.content) == str:
return f'{self.label}[{self.content}]'
@ -305,6 +311,10 @@ class BlockQuoteAuthorToken(SemanticType):
label = "bq:author"
class CodeToken(ExplicitSemanticType, TokensToMatch):
label = "code"
class Linebreak(ExplicitSemanticType):
label = "linebreak"
@ -455,8 +465,11 @@ et_custom_span = (
pps('(#') + pp.Word(pp.nums)('span_id') + pps(')')
).set_name('custom_span').add_parse_action(of_type(EtCustomSpanToken))
# Code Token
code = pp.Literal('```')('code').add_parse_action(of_type(CodeToken))
# markup sums up all in-line elements
markup = il_link | et_strong | et_em | et_strikethrough | et_underline | et_custom_span
markup = il_link | et_strong | et_em | et_strikethrough | et_underline | et_custom_span | code # not quite correct but good enough for now # noqa : E501
enhanced_text = pp.ZeroOrMore(
markup | pp.SkipTo(markup)('text').add_parse_action(of_type(TextToken)) + markup
) + pp.Opt(pp.Regex(r'.+')("text").add_parse_action(of_type(TextToken)))

View File

@ -551,6 +551,7 @@ dict_advanced_syntax_input_and_expected_output = {
]),
]),
), __GL(),),
("```Code```", (sy.CodeToken(["```"]), sy.TextToken(["Code"]), sy.CodeToken(["```"])), __GL()),
],
"blockquote": [
("> Text", (sy.BlockQuoteToken([sy.TextToken(["Text"])]),), __GL()),
@ -590,6 +591,10 @@ dict_advanced_syntax_input_and_expected_output = {
sy.BlockQuoteAuthorToken(["Author Name"]),
), __GL()),
],
"code": [
("```\nText\n```", (sy.CodeToken(["```"]),), __GL()),
("```\nText\nText\n```", (sy.CodeToken(["```"]),), __GL()),
],
}
# Cursed zipping oneline

View File

@ -32,6 +32,7 @@ _type_counterpart_container = [
(syntax.TableSeparatorToken, None, "error", tools.__GLk(1)),
(syntax.BlockQuoteToken, None, "error", tools.__GLk(1)),
(syntax.BlockQuoteAuthorToken, None, "error", tools.__GLk(1)),
(syntax.CodeToken, "code", "token", tools.__GLk(1)),
(syntax.Linebreak, "linebreak", "self", tools.__GLk(1)),
]

View File

@ -0,0 +1,16 @@
import pytest
from bootstraparse.modules import syntax, error_mngr, tools, context_mngr
from bootstraparse.modules.tools import __GL, __module_path, find_variables_in_file
__XF = pytest.mark.xfail
_list_original_strings = [
"string",
]
def test_to_original():
"""Test the .to_original() method of SemanticTypes."""
token = syntax.SemanticType(["test"])
assert token.to_original() == 'test'