Added tokens for basic elements.

Some more PEP and testing shenanigans.
This commit is contained in:
Maximilien Rigaut 2022-03-19 10:12:51 +01:00
parent 5694596f90
commit 76f4f139bd
3 changed files with 90 additions and 25 deletions

View File

@ -44,11 +44,18 @@ class SemanticType:
def __eq__(self, other):
if type(other) == type(self):
for e1, e2 in zip_longest(self.content, other.content):
if e1 != e2:
if isinstance(e1, (pp.ParseResults, list)): # if it's a list, we need to compare each element
for elt1, elt2 in zip_longest(e1, e2):
if elt1 != elt2:
return False
elif e1 != e2:
return False
return True
return False
def __ne__(self, other):
return not self.__eq__(other)
class ExplicitSemanticType(SemanticType):
"""
@ -157,6 +164,22 @@ class OptionalToken(SemanticType):
label = "optional"
class OptionalInsertToken(SemanticType):
label = "optional:insert"
class OptionalVarToken(SemanticType):
label = "optional:var"
class BeAssignToken(SemanticType):
label = "be:assign"
class BeValueToken(SemanticType):
label = "be:var"
def of_type(token_class):
"""
Function creating a custom function for generating the given Token type.
@ -192,25 +215,31 @@ def readable_markup(list_of_tokens):
# Base elements
quotes = pp.Word(r""""'""")
value = (quotes + pp.Word(pp.alphanums + r'.') + pp.match_previous_literal(quotes) ^
value = (pps(quotes) + pp.Word(pp.alphanums + r'.') + pps(pp.match_previous_literal(quotes)) ^
pp.common.fnumber)("value")
assignation = pp.Group(pp.common.identifier('var_name') + '=' + value('var_value'))("assignation")
assignation = pp.Group(
pp.common.identifier('var_name') + pps('=') + value('var_value')
)("assignation") # TODO: More extensive testing
text = pp.OneOrMore(pp.Word(pp.alphanums))('text').add_parse_action(of_type(TextToken))
url_characters = pp.common.url
# Composite elements
var = '[' + pp.delimitedList(assignation ^ value)("list_vars").set_name("list_vars") + ']'
var = pps('[') + pp.delimitedList(
assignation.add_parse_action(of_type(BeAssignToken)) ^
value.add_parse_action(of_type(BeValueToken))
)("list_vars").set_name("list_vars") + pps(']') # TODO: More extensive testing
# Specific elements
image_element = ('@{' + pp.common.identifier('image_name') + '}')("image_element")
alias_element = ('@[' + pp.common.identifier('alias_name') + ']')("alias_element")
expression = pp.Word(pp.alphanums + r'=+-_\'",;:!<> ')
html_insert = '{' + expression('html_insert') + '}'
html_insert = pps('{') + expression('html_insert') + pps('}') # TODO: More extensive testing
# Optional elements
optional = (
pp.Opt(html_insert)("html_insert") & pp.Opt(var)("var")
)("optional").add_parse_action(of_type(OptionalToken))
pp.Opt(html_insert)("html_insert").add_parse_action(of_type(OptionalInsertToken)) &
pp.Opt(var)("var").add_parse_action(of_type(OptionalVarToken))
)("optional").add_parse_action(of_type(OptionalToken)) # Macro OptionnalToken might be redundant
# Structural elements
structural_elements = (

View File

@ -206,7 +206,7 @@ def test_preparser_content(filename, content):
pp.make_import_list()
assert pp.export_with_imports().read() == content
# todo: test import in sub-folders
# TODO: Test import in sub-folders
@pytest.mark.xfail(reason="Not implemented")

View File

@ -168,16 +168,33 @@ dict_advanced_syntax_input_and_expected_output = {
# Matches a div element, must be at the beginning of the line, the closing div can be with arguments
("div>> [class='blue', 123]{var='test', number=11}", (
sy.StructuralElementEndToken(["div"]),
sy.OptionalToken(["[", ["class", "=", "'blue'"], 123, "]", "{", "var='test'", "number=11", "}"]),
), __GL(), __XF), # Bad optional implementation
# ("div>> [class='blue', 123#]",
# [sy.StructuralElementEndToken(["div", [sy.OptionalToken(["[", "class='blue'", ",", "123#", "]"])]])]),
# ("div>> {var='test', number=11}",
# [sy.StructuralElementEndToken(["div", [sy.OptionalToken(["{", "var='test'", ",", "number=11", "}"])]])]),
sy.OptionalToken([
sy.OptionalVarToken([
sy.BeAssignToken([["class", "blue"]]),
sy.BeValueToken([123.0]),
]),
sy.OptionalInsertToken(["var='test', number=11"]),
]),
), __GL(),),
("div>> [class='blue', 123]", (
sy.StructuralElementEndToken(["div"]),
sy.OptionalToken([
sy.OptionalVarToken([
sy.BeAssignToken([["class", "blue"]]),
sy.BeValueToken([123.0]),
]),
]),
), __GL(),),
("div>> {var='test', number=11}", (
sy.StructuralElementEndToken(["div"]),
sy.OptionalToken([
sy.OptionalInsertToken(["var='test', number=11"]),
]),
), __GL(),),
],
# List Elements
# TODO: Test markup and add optionnals in lists
"one_olist": [
# Matches a one-level ordered list element, must be at the beginning of the line
# Drop the "." from the match
@ -220,13 +237,25 @@ dict_advanced_syntax_input_and_expected_output = {
("- Text7 [class='blue', 123]{var='test', number=11}", (
sy.EtUlistToken([
sy.TextToken(["Text7"]),
sy.OptionalToken(["[", "class='blue'", ",", "123#", "]", "{", "var='test'", ",", "number=11", "}"])
sy.OptionalToken([
sy.OptionalVarToken([
sy.BeAssignToken([["class", "blue"]]),
sy.BeValueToken([123.0]),
]),
sy.OptionalInsertToken(["var='test', number=11"]),
]),
]),
), __GL(), __XF),
("#. Text8 [class='blue', 123]{var='test', number=11}", (
sy.EtOlistToken([
sy.TextToken(["Text8"]),
sy.OptionalToken(["[", "class='blue'", ",", "123", "]", "{", "var='test'", ",", "number=11", "}"])
sy.OptionalToken([
sy.OptionalVarToken([
sy.BeAssignToken([["class", "blue"]]),
sy.BeValueToken([123.0]),
]),
sy.OptionalInsertToken(["var='test', number=11"]),
]),
]),
), __GL(), __XF),
("! Display9 ! [class='blue', 123]{var='test', number=11}", (
@ -234,9 +263,12 @@ dict_advanced_syntax_input_and_expected_output = {
"!",
"Display9",
sy.OptionalToken([
"[", "class='blue'", ",", "123", "]",
"{", "var='test'", ",", "number=11", "}"
])
sy.OptionalVarToken([
sy.BeAssignToken([["class", "blue"]]),
sy.BeValueToken([123.0]),
]),
sy.OptionalInsertToken(["var='test', number=11"]),
]),
]),
), __GL(), __XF), # Bad optional implementation
("## Text10 ## [class='blue', 123]{var='test', number=11}", (
@ -244,8 +276,12 @@ dict_advanced_syntax_input_and_expected_output = {
"##",
"Text10",
sy.OptionalToken([
"[", "class='blue'", ",", "123", "]",
"{", "var='test'", ",", "number=11", "}"])
sy.OptionalVarToken([
sy.BeAssignToken([["class", "blue"]]),
sy.BeValueToken([123.0]),
]),
sy.OptionalInsertToken(["var='test', number=11"]),
]),
]),
), __GL(), __XF), # Bad optional implementation
],
@ -302,15 +338,15 @@ dict_advanced_syntax_input_and_expected_output = {
# Match any line parsed by the parser (can match header, list table etc...) this is the main syntax element
("# Text1 #", (sy.HeaderToken(["#", "Text1 "]),), __GL()),
("Text *em __underline__ still em*", (
sy.TextToken(["Text "]),
sy.TextToken(["Text"]),
sy.EtEmToken(["*"]),
sy.TextToken(["em "]),
sy.TextToken(["em"]),
sy.EtUnderlineToken(["__"]),
sy.TextToken(["underline"]),
sy.EtUnderlineToken(["__"]),
sy.TextToken(["still em"]),
sy.EtEmToken(["*"]),
), __GL(), __XF), # Bad optional implementation
), __GL(),),
("|2 Text1 | Text2 |", (
sy.TableRowToken(["2", sy.TableCellToken(["Text1 "]), sy.TableCellToken(["Text2 "])]),
), __GL()),