Merged with origin, corrected some tests and reached 100% code coverage

This commit is contained in:
Maximilien Rigaut 2022-03-17 19:53:53 +01:00
commit 5bc833990b
2 changed files with 123 additions and 67 deletions

View File

@ -1,7 +1,9 @@
# Dedicated module for the syntax of all the parsing # Dedicated module for the syntax of all the parsing
import os import os
from itertools import zip_longest
import pyparsing as pp import pyparsing as pp
import rich # import rich
pps = pp.Suppress pps = pp.Suppress
@ -16,6 +18,20 @@ class SemanticType:
def __init__(self, content): def __init__(self, content):
self.content = content self.content = content
def to_markup(self):
"""
Function used for testing and readability purposes. Replaces matched markup with html-like tags.
"""
if self.content:
return_list = []
for elt in self.content:
try:
return_list.append(elt.to_markup())
except Exception:
return_list.append(str(elt))
return f"<{self.label} = '{','.join(return_list)}' />"
return f'<[NOC] {self.label} />'
def __str__(self): def __str__(self):
if type(self.content) == str: if type(self.content) == str:
return f'{self.label}[{self.content}]' return f'{self.label}[{self.content}]'
@ -27,13 +43,32 @@ class SemanticType:
def __eq__(self, other): def __eq__(self, other):
if type(other) == type(self): if type(other) == type(self):
for e1, e2 in zip(self.content, other.content): for e1, e2 in zip_longest(self.content, other.content):
if e1 != e2: if e1 != e2:
return False return False
return True return True
return False return False
class ExplicitSemanticType(SemanticType):
"""
Explicit semantic type, the label is the only information we need.
"""
def to_markup(self):
return f'<{self.label} />'
class EmptySemanticType(SemanticType):
"""
Empty semantic type, the content is the only information we need.
"""
def to_markup(self):
if type(self.content) == str:
return f'{self.content}'
else:
return " ".join(map(str, self.content))
class UnimplementedToken(SemanticType): class UnimplementedToken(SemanticType):
label = "unimplemented" label = "unimplemented"
@ -46,27 +81,27 @@ class ImageToken(SemanticType):
label = "image" label = "image"
class TextToken(SemanticType): class TextToken(EmptySemanticType):
label = "text" label = "text"
class EnhancedToken(SemanticType): class EnhancedToken(ExplicitSemanticType):
label = "text:enhanced" label = "text:enhanced"
class EtEmToken(SemanticType): class EtEmToken(ExplicitSemanticType):
label = "text:em" label = "text:em"
class EtStrongToken(SemanticType): class EtStrongToken(ExplicitSemanticType):
label = "text:strong" label = "text:strong"
class EtUnderlineToken(SemanticType): class EtUnderlineToken(ExplicitSemanticType):
label = "text:underline" label = "text:underline"
class EtStrikethroughToken(SemanticType): class EtStrikethroughToken(ExplicitSemanticType):
label = "text:strikethrough" label = "text:strikethrough"
@ -141,25 +176,16 @@ def readable_markup(list_of_tokens):
:return: returns readable string. :return: returns readable string.
:rtype: string :rtype: string
""" """
readable_string = '' readable_list = []
for token in list_of_tokens: for token in list_of_tokens:
if type(token) == str: if type(token) == str:
readable_string += token readable_list.append(token)
else: else:
readable_string += _add_tag(token) try:
return readable_string readable_list.append(token.to_markup())
except Exception:
readable_list.append(str(token))
def _add_tag(token): return " ".join(readable_list)
"""
Function used for testing and readability purposes. Replaces matched markup with html-like tags.
:param token: matched markup token in analysed text.
:return: returns readable string.
"""
if token.label == 'text':
return token.content[0]
if token.content:
return "<{} = '{}' />".format(token.label, ",".join(token.content))
# Base elements # Base elements
@ -194,6 +220,10 @@ optional = (
pp.Opt(html_insert)("html_insert") + pp.Opt(var)("var") pp.Opt(html_insert)("html_insert") + pp.Opt(var)("var")
)("optional").add_parse_action(of_type(OptionalToken)) )("optional").add_parse_action(of_type(OptionalToken))
# Inline elements
il_link = pp.Regex(
r"""\[(?P<text>.+)\]\(['"](?P<url>[a-zA-Z-_:\/=@#!%\?\d\(\)\.]+)['"]\)"""
).add_parse_action(of_type(HyperlinkToken))
# Enhanced text elements # Enhanced text elements
et_em = pp.Literal('*')('em').add_parse_action(of_type(EtEmToken)) et_em = pp.Literal('*')('em').add_parse_action(of_type(EtEmToken))
@ -203,22 +233,15 @@ et_strikethrough = pp.Literal('~~')('strikethrough').add_parse_action(of_type(Et
et_custom_span = ( et_custom_span = (
pps('(#') + pp.Word(pp.nums)('span_id') + pps(')') pps('(#') + pp.Word(pp.nums)('span_id') + pps(')')
).set_name('custom_span').add_parse_action(of_type(EtCustomSpanToken)) ).set_name('custom_span').add_parse_action(of_type(EtCustomSpanToken))
markup = et_strong | et_em | et_strikethrough | et_underline | et_custom_span
# markup sums up all in-line elements
markup = il_link | et_strong | et_em | et_strikethrough | et_underline | et_custom_span
# Multiline elements # Multiline elements
se_start = (pps('<<') + structural_elements).add_parse_action(of_type(StructuralElementStartToken)) se_start = (pps('<<') + structural_elements).add_parse_action(of_type(StructuralElementStartToken))
se_end = (structural_elements + pps('>>')).add_parse_action(of_type(StructuralElementEndToken)) se_end = (structural_elements + pps('>>')).add_parse_action(of_type(StructuralElementEndToken))
se = se_end | se_start # Structural element se = se_end | se_start # Structural element
# Inline elements
# [link text]('http://test.icule')
il_link = pp.common.url
# il_link = (
# pp.Literal('[') + pp.SkipTo('](')('text') + '](' +
# quotes + url_characters('url') + pp.match_previous_literal(quotes)
# + ')'
# ).add_parse_action(of_type(HyperlinkToken))
# Oneline elements # Oneline elements
one_header = ( one_header = (
header_element + pp.SkipTo(pp.match_previous_literal(header_element)) header_element + pp.SkipTo(pp.match_previous_literal(header_element))
@ -234,10 +257,9 @@ one_ulist = pp.line_start + (
).add_parse_action(of_type(EtUlistToken)) ).add_parse_action(of_type(EtUlistToken))
# Final elements # Final elements
enhanced_text = pp.OneOrMore(markup + text) enhanced_text = pp.ZeroOrMore(
# pp.SkipTo(markup)('text').add_parse_action(of_type(TextToken)) + markup | markup | pp.SkipTo(markup)('text').add_parse_action(of_type(TextToken)) + markup
# pp.SkipTo(pp.line_end)("text").add_parse_action(of_type(TextToken)) ) + pp.Opt(pp.rest_of_line("text").add_parse_action(of_type(TextToken)))
# )
############################################################################## ##############################################################################
@ -253,7 +275,7 @@ line_to_replace = pp.OneOrMore(
pp.SkipTo(image ^ alias)('text').add_parse_action(of_type(TextToken)) pp.SkipTo(image ^ alias)('text').add_parse_action(of_type(TextToken))
^ image.add_parse_action(of_type(ImageToken)) ^ image.add_parse_action(of_type(ImageToken))
^ alias.add_parse_action(of_type(AliasToken)) ^ alias.add_parse_action(of_type(AliasToken))
) ^ pp.SkipTo(pp.lineEnd)('text').add_parse_action(of_type(TextToken)) ) ^ pp.rest_of_line('text').add_parse_action(of_type(TextToken))
############################################################################## ##############################################################################
# Temporary tests # Temporary tests
@ -261,12 +283,6 @@ line_to_replace = pp.OneOrMore(
if __name__ == '__main__': # pragma: no cover if __name__ == '__main__': # pragma: no cover
pp.autoname_elements() pp.autoname_elements()
list_strings = [
"'http://test.icule'",
]
rich.inspect(il_link.parse_string(list_strings[0]))
if not os.path.exists('../../../dev_outputs/'): if not os.path.exists('../../../dev_outputs/'):
os.mkdir('../../../dev_outputs/') os.mkdir('../../../dev_outputs/')
enhanced_text.create_diagram("../../../dev_outputs/diagram.html") enhanced_text.create_diagram("../../../dev_outputs/diagram.html")

View File

@ -58,7 +58,7 @@ expressions_to_match = {
# Testing the parser lexical elements # Testing the parser lexical elements
# test_advanced_expression_and_token_creation # test_advanced_expression_and_token_creation
dict_advanced_syntax_input_and_expected_output = { dict_advanced_syntax_input_and_expected_output = {
# Enhanced text # TODO : Test optional inline args # Enhanced text
"et_em": [ "et_em": [
# Single em element, should only match the first "*" # Single em element, should only match the first "*"
("*", (sy.EtEmToken(["*"]), )), ("*", (sy.EtEmToken(["*"]), )),
@ -80,9 +80,9 @@ dict_advanced_syntax_input_and_expected_output = {
("(#12345)", (sy.EtCustomSpanToken(["12345"]), )), ("(#12345)", (sy.EtCustomSpanToken(["12345"]), )),
], ],
"il_link": [ "il_link": [
# Single link element, should only match the first "[link_name](link)" # Single link element, should only match the first "[link_name]('link')"
("[text_link]('text://www.website.com/link.html')", ("[text_link]('text://www.website.com/link.html')",
sy.HyperlinkToken("[text_link]('text://www.website.com/link.html')")) [sy.HyperlinkToken(["[text_link]('text://www.website.com/link.html')"])])
], ],
# Enhanced Text # Enhanced Text
@ -107,10 +107,9 @@ dict_advanced_syntax_input_and_expected_output = {
sy.EtCustomSpanToken(["12"]), sy.TextToken(["test"]), sy.EtStrongToken(["**"]), sy.EtCustomSpanToken(["12"]), sy.TextToken(["test"]), sy.EtStrongToken(["**"]),
sy.TextToken(["test"]), sy.EtUnderlineToken(["__"]), sy.TextToken(["test"]), sy.EtUnderlineToken(["__"]),
sy.TextToken(["test"]), sy.EtStrikethroughToken(["~~"]), )), sy.TextToken(["test"]), sy.EtStrikethroughToken(["~~"]), )),
("Test text with a link [link_name](link)", (sy.TextToken(["Test text with a link "]), ("Test text with a link [link_name]('link')", (sy.TextToken(["Test text with a link"]), sy.HyperlinkToken(["[link_name]('link')"]), )), # noqa E501 (line too long)
sy.HyperlinkToken(["link_name", "link"]), )), ("Test text with a link [link_name](\"link\") and a *bold* text", (sy.TextToken(["Test text with a link"]),
("Test text with a link [link_name](link) and a *bold* text", (sy.TextToken(["Test text with a link "]), sy.HyperlinkToken(['[link_name]("link")']),
sy.HyperlinkToken(["link_name", "link"]),
sy.TextToken(["and a"]), sy.EtEmToken(["*"]), sy.TextToken(["and a"]), sy.EtEmToken(["*"]),
sy.TextToken(["bold"]), sy.EtEmToken(["*"]), )), sy.TextToken(["bold"]), sy.EtEmToken(["*"]), )),
], ],
@ -190,10 +189,10 @@ list_add_tag_input_and_expected_output = [
# Test cases for the add_tag function # Test cases for the add_tag function
# Input: # Input:
(sy.TextToken(["Text"]), "Text"), (sy.TextToken(["Text"]), "Text"),
(sy.EtEmToken(["*"]), "<text:em = '*' />"), (sy.EtEmToken(["*"]), "<text:em />"),
(sy.EtStrongToken(["**"]), "<text:strong = '**' />"), (sy.EtStrongToken(["**"]), "<text:strong />"),
(sy.EtUnderlineToken(["__"]), "<text:underline = '__' />"), (sy.EtUnderlineToken(["__"]), "<text:underline />"),
(sy.EtStrikethroughToken(["~~"]), "<text:strikethrough = '~~' />"), (sy.EtStrikethroughToken(["~~"]), "<text:strikethrough />"),
(sy.StructuralElementStartToken(["div"]), "<se:start = 'div' />"), (sy.StructuralElementStartToken(["div"]), "<se:start = 'div' />"),
(sy.StructuralElementEndToken(["div"]), "<se:end = 'div' />"), (sy.StructuralElementEndToken(["div"]), "<se:end = 'div' />"),
(sy.EtCustomSpanToken(["(#12)"]), "<text:custom_span = '(#12)' />"), (sy.EtCustomSpanToken(["(#12)"]), "<text:custom_span = '(#12)' />"),
@ -207,13 +206,12 @@ list_of_text_input_and_readable_output = [
("## Text2 ##", "one_header", "<header = '##,Text2 ' />"), ("## Text2 ##", "one_header", "<header = '##,Text2 ' />"),
# Text # Text
("*Italic*", "et_em", "<text:em = '*' />"), ("*Italic*", "et_em", "<text:em />"),
("__Underline__", "et_underline", "<text:underline = '__' />"), ("__Underline__", "et_underline", "<text:underline />"),
("**Bold**", "et_strong", "<text:strong = '**' />"), ("**Bold**", "et_strong", "<text:strong />"),
("~~Strikethrough~~", "et_strikethrough", "<text:strikethrough = '~~' />"), ("~~Strikethrough~~", "et_strikethrough", "<text:strikethrough />"),
("Text *bold __underline__ still bold*", "enhanced_text", "Text <text:em = '*' />bold <text:underline = '__' />underline<text:underline = '__' /> still bold<text:em = '*' />"), # noqa E501 (line too long) ("Text *bold __underline__ still bold*", "enhanced_text", "Text <text:em /> bold <text:underline /> underline <text:underline /> still bold <text:em /> "), # noqa E501 (line too long)
("[link]('http://www.google.com')", "il_link", "<hyperlink = '[link]('http://www.google.com')' />"),
("[link]('http://www.google.com')", "il_link", "<link: [link](http://www.google.com)>"),
# Structural elements # Structural elements
("<<div", "se", "<se:start = 'div' />"), ("<<div", "se", "<se:start = 'div' />"),
@ -291,6 +289,33 @@ def test_semantic_type():
assert strings_in_token(st, ["[1, 2, 3]", "test"]) assert strings_in_token(st, ["[1, 2, 3]", "test"])
def test_semantic_type_eq():
"""
Test that the semantic type is correctly created.
"""
st1 = sy.SemanticType("test")
st2 = sy.SemanticType("test")
assert st1 == st2
st1 = sy.SemanticType(["test"])
st2 = sy.SemanticType(["test2"])
assert st1 != st2
st1 = sy.SemanticType("test")
st2 = sy.SemanticType([1, 2, 3])
assert st1 != st2
st1 = sy.SemanticType([1, 2, 3])
st2 = sy.SemanticType([1, 2, 3])
assert st1 == st2
st1 = sy.SemanticType([1, 2, 3])
st2 = sy.SemanticType([1, 2, 3, 4])
assert st1 != st2
st1 = sy.SemanticType([1, 2, 3, 4])
st2 = sy.SemanticType([1, 2, 3])
assert st1 != st2
st1 = sy.SemanticType([1, 2, 3])
st2 = sy.SemanticType([1, 2, 3, 4])
assert st1 != st2
def find_expression_from_str(expression_str): def find_expression_from_str(expression_str):
""" """
Find the expression from its string representation. Find the expression from its string representation.
@ -354,7 +379,7 @@ def test__add_tag(token, expected_output):
:type token: sy.SemanticType :type token: sy.SemanticType
:type expected_output: str :type expected_output: str
""" """
assert sy._add_tag(token) == expected_output assert token.to_markup() == expected_output
# Test text functions # Test text functions
@ -374,3 +399,18 @@ def test_readable_markup(token_list, parsing_expression, expected):
result = expr.parse_string(token_list) result = expr.parse_string(token_list)
assert result is not None assert result is not None
assert sy.readable_markup(result) == expected assert sy.readable_markup(result) == expected
def test_weird_markup():
"""
Test that the readable markup is correctly generated.
"""
assert sy.readable_markup([sy.EmptySemanticType("weird")]) == "weird"
empty_token = sy.SemanticType(None)
empty_token.label = "weird"
assert sy.readable_markup([empty_token]) == "<[NOC] weird />"
class WeirdClass:
def __str__(self):
return "weird"
assert sy.readable_markup([sy.EmptySemanticType("weird"), empty_token, WeirdClass()]) == "weird <[NOC] weird /> weird"