Merged with origin, corrected some tests and reached 100% code coverage

This commit is contained in:
Maximilien Rigaut 2022-03-17 19:53:53 +01:00
commit 5bc833990b
2 changed files with 123 additions and 67 deletions

View File

@ -1,7 +1,9 @@
# Dedicated module for the syntax of all the parsing
import os
from itertools import zip_longest
import pyparsing as pp
import rich
# import rich
pps = pp.Suppress
@ -16,6 +18,20 @@ class SemanticType:
def __init__(self, content):
self.content = content
def to_markup(self):
"""
Function used for testing and readability purposes. Replaces matched markup with html-like tags.
"""
if self.content:
return_list = []
for elt in self.content:
try:
return_list.append(elt.to_markup())
except Exception:
return_list.append(str(elt))
return f"<{self.label} = '{','.join(return_list)}' />"
return f'<[NOC] {self.label} />'
def __str__(self):
if type(self.content) == str:
return f'{self.label}[{self.content}]'
@ -27,13 +43,32 @@ class SemanticType:
def __eq__(self, other):
if type(other) == type(self):
for e1, e2 in zip(self.content, other.content):
for e1, e2 in zip_longest(self.content, other.content):
if e1 != e2:
return False
return True
return False
class ExplicitSemanticType(SemanticType):
"""
Explicit semantic type, the label is the only information we need.
"""
def to_markup(self):
return f'<{self.label} />'
class EmptySemanticType(SemanticType):
"""
Empty semantic type, the content is the only information we need.
"""
def to_markup(self):
if type(self.content) == str:
return f'{self.content}'
else:
return " ".join(map(str, self.content))
class UnimplementedToken(SemanticType):
label = "unimplemented"
@ -46,27 +81,27 @@ class ImageToken(SemanticType):
label = "image"
class TextToken(SemanticType):
class TextToken(EmptySemanticType):
label = "text"
class EnhancedToken(SemanticType):
class EnhancedToken(ExplicitSemanticType):
label = "text:enhanced"
class EtEmToken(SemanticType):
class EtEmToken(ExplicitSemanticType):
label = "text:em"
class EtStrongToken(SemanticType):
class EtStrongToken(ExplicitSemanticType):
label = "text:strong"
class EtUnderlineToken(SemanticType):
class EtUnderlineToken(ExplicitSemanticType):
label = "text:underline"
class EtStrikethroughToken(SemanticType):
class EtStrikethroughToken(ExplicitSemanticType):
label = "text:strikethrough"
@ -141,25 +176,16 @@ def readable_markup(list_of_tokens):
:return: returns readable string.
:rtype: string
"""
readable_string = ''
readable_list = []
for token in list_of_tokens:
if type(token) == str:
readable_string += token
readable_list.append(token)
else:
readable_string += _add_tag(token)
return readable_string
def _add_tag(token):
"""
Function used for testing and readability purposes. Replaces matched markup with html-like tags.
:param token: matched markup token in analysed text.
:return: returns readable string.
"""
if token.label == 'text':
return token.content[0]
if token.content:
return "<{} = '{}' />".format(token.label, ",".join(token.content))
try:
readable_list.append(token.to_markup())
except Exception:
readable_list.append(str(token))
return " ".join(readable_list)
# Base elements
@ -194,6 +220,10 @@ optional = (
pp.Opt(html_insert)("html_insert") + pp.Opt(var)("var")
)("optional").add_parse_action(of_type(OptionalToken))
# Inline elements
il_link = pp.Regex(
r"""\[(?P<text>.+)\]\(['"](?P<url>[a-zA-Z-_:\/=@#!%\?\d\(\)\.]+)['"]\)"""
).add_parse_action(of_type(HyperlinkToken))
# Enhanced text elements
et_em = pp.Literal('*')('em').add_parse_action(of_type(EtEmToken))
@ -203,22 +233,15 @@ et_strikethrough = pp.Literal('~~')('strikethrough').add_parse_action(of_type(Et
et_custom_span = (
pps('(#') + pp.Word(pp.nums)('span_id') + pps(')')
).set_name('custom_span').add_parse_action(of_type(EtCustomSpanToken))
markup = et_strong | et_em | et_strikethrough | et_underline | et_custom_span
# markup sums up all in-line elements
markup = il_link | et_strong | et_em | et_strikethrough | et_underline | et_custom_span
# Multiline elements
se_start = (pps('<<') + structural_elements).add_parse_action(of_type(StructuralElementStartToken))
se_end = (structural_elements + pps('>>')).add_parse_action(of_type(StructuralElementEndToken))
se = se_end | se_start # Structural element
# Inline elements
# [link text]('http://test.icule')
il_link = pp.common.url
# il_link = (
# pp.Literal('[') + pp.SkipTo('](')('text') + '](' +
# quotes + url_characters('url') + pp.match_previous_literal(quotes)
# + ')'
# ).add_parse_action(of_type(HyperlinkToken))
# Oneline elements
one_header = (
header_element + pp.SkipTo(pp.match_previous_literal(header_element))
@ -234,10 +257,9 @@ one_ulist = pp.line_start + (
).add_parse_action(of_type(EtUlistToken))
# Final elements
enhanced_text = pp.OneOrMore(markup + text)
# pp.SkipTo(markup)('text').add_parse_action(of_type(TextToken)) + markup |
# pp.SkipTo(pp.line_end)("text").add_parse_action(of_type(TextToken))
# )
enhanced_text = pp.ZeroOrMore(
markup | pp.SkipTo(markup)('text').add_parse_action(of_type(TextToken)) + markup
) + pp.Opt(pp.rest_of_line("text").add_parse_action(of_type(TextToken)))
##############################################################################
@ -253,7 +275,7 @@ line_to_replace = pp.OneOrMore(
pp.SkipTo(image ^ alias)('text').add_parse_action(of_type(TextToken))
^ image.add_parse_action(of_type(ImageToken))
^ alias.add_parse_action(of_type(AliasToken))
) ^ pp.SkipTo(pp.lineEnd)('text').add_parse_action(of_type(TextToken))
) ^ pp.rest_of_line('text').add_parse_action(of_type(TextToken))
##############################################################################
# Temporary tests
@ -261,12 +283,6 @@ line_to_replace = pp.OneOrMore(
if __name__ == '__main__': # pragma: no cover
pp.autoname_elements()
list_strings = [
"'http://test.icule'",
]
rich.inspect(il_link.parse_string(list_strings[0]))
if not os.path.exists('../../../dev_outputs/'):
os.mkdir('../../../dev_outputs/')
enhanced_text.create_diagram("../../../dev_outputs/diagram.html")
enhanced_text.create_diagram("../../../dev_outputs/diagram.html")

View File

@ -58,7 +58,7 @@ expressions_to_match = {
# Testing the parser lexical elements
# test_advanced_expression_and_token_creation
dict_advanced_syntax_input_and_expected_output = {
# Enhanced text # TODO : Test optional inline args
# Enhanced text
"et_em": [
# Single em element, should only match the first "*"
("*", (sy.EtEmToken(["*"]), )),
@ -80,9 +80,9 @@ dict_advanced_syntax_input_and_expected_output = {
("(#12345)", (sy.EtCustomSpanToken(["12345"]), )),
],
"il_link": [
# Single link element, should only match the first "[link_name](link)"
# Single link element, should only match the first "[link_name]('link')"
("[text_link]('text://www.website.com/link.html')",
sy.HyperlinkToken("[text_link]('text://www.website.com/link.html')"))
[sy.HyperlinkToken(["[text_link]('text://www.website.com/link.html')"])])
],
# Enhanced Text
@ -107,12 +107,11 @@ dict_advanced_syntax_input_and_expected_output = {
sy.EtCustomSpanToken(["12"]), sy.TextToken(["test"]), sy.EtStrongToken(["**"]),
sy.TextToken(["test"]), sy.EtUnderlineToken(["__"]),
sy.TextToken(["test"]), sy.EtStrikethroughToken(["~~"]), )),
("Test text with a link [link_name](link)", (sy.TextToken(["Test text with a link "]),
sy.HyperlinkToken(["link_name", "link"]), )),
("Test text with a link [link_name](link) and a *bold* text", (sy.TextToken(["Test text with a link "]),
sy.HyperlinkToken(["link_name", "link"]),
sy.TextToken([" and a "]), sy.EtEmToken(["*"]),
sy.TextToken(["bold"]), sy.EtEmToken(["*"]), )),
("Test text with a link [link_name]('link')", (sy.TextToken(["Test text with a link"]), sy.HyperlinkToken(["[link_name]('link')"]), )), # noqa E501 (line too long)
("Test text with a link [link_name](\"link\") and a *bold* text", (sy.TextToken(["Test text with a link"]),
sy.HyperlinkToken(['[link_name]("link")']),
sy.TextToken(["and a"]), sy.EtEmToken(["*"]),
sy.TextToken(["bold"]), sy.EtEmToken(["*"]), )),
],
@ -190,10 +189,10 @@ list_add_tag_input_and_expected_output = [
# Test cases for the add_tag function
# Input:
(sy.TextToken(["Text"]), "Text"),
(sy.EtEmToken(["*"]), "<text:em = '*' />"),
(sy.EtStrongToken(["**"]), "<text:strong = '**' />"),
(sy.EtUnderlineToken(["__"]), "<text:underline = '__' />"),
(sy.EtStrikethroughToken(["~~"]), "<text:strikethrough = '~~' />"),
(sy.EtEmToken(["*"]), "<text:em />"),
(sy.EtStrongToken(["**"]), "<text:strong />"),
(sy.EtUnderlineToken(["__"]), "<text:underline />"),
(sy.EtStrikethroughToken(["~~"]), "<text:strikethrough />"),
(sy.StructuralElementStartToken(["div"]), "<se:start = 'div' />"),
(sy.StructuralElementEndToken(["div"]), "<se:end = 'div' />"),
(sy.EtCustomSpanToken(["(#12)"]), "<text:custom_span = '(#12)' />"),
@ -207,13 +206,12 @@ list_of_text_input_and_readable_output = [
("## Text2 ##", "one_header", "<header = '##,Text2 ' />"),
# Text
("*Italic*", "et_em", "<text:em = '*' />"),
("__Underline__", "et_underline", "<text:underline = '__' />"),
("**Bold**", "et_strong", "<text:strong = '**' />"),
("~~Strikethrough~~", "et_strikethrough", "<text:strikethrough = '~~' />"),
("Text *bold __underline__ still bold*", "enhanced_text", "Text <text:em = '*' />bold <text:underline = '__' />underline<text:underline = '__' /> still bold<text:em = '*' />"), # noqa E501 (line too long)
("[link]('http://www.google.com')", "il_link", "<link: [link](http://www.google.com)>"),
("*Italic*", "et_em", "<text:em />"),
("__Underline__", "et_underline", "<text:underline />"),
("**Bold**", "et_strong", "<text:strong />"),
("~~Strikethrough~~", "et_strikethrough", "<text:strikethrough />"),
("Text *bold __underline__ still bold*", "enhanced_text", "Text <text:em /> bold <text:underline /> underline <text:underline /> still bold <text:em /> "), # noqa E501 (line too long)
("[link]('http://www.google.com')", "il_link", "<hyperlink = '[link]('http://www.google.com')' />"),
# Structural elements
("<<div", "se", "<se:start = 'div' />"),
@ -291,6 +289,33 @@ def test_semantic_type():
assert strings_in_token(st, ["[1, 2, 3]", "test"])
def test_semantic_type_eq():
"""
Test that the semantic type is correctly created.
"""
st1 = sy.SemanticType("test")
st2 = sy.SemanticType("test")
assert st1 == st2
st1 = sy.SemanticType(["test"])
st2 = sy.SemanticType(["test2"])
assert st1 != st2
st1 = sy.SemanticType("test")
st2 = sy.SemanticType([1, 2, 3])
assert st1 != st2
st1 = sy.SemanticType([1, 2, 3])
st2 = sy.SemanticType([1, 2, 3])
assert st1 == st2
st1 = sy.SemanticType([1, 2, 3])
st2 = sy.SemanticType([1, 2, 3, 4])
assert st1 != st2
st1 = sy.SemanticType([1, 2, 3, 4])
st2 = sy.SemanticType([1, 2, 3])
assert st1 != st2
st1 = sy.SemanticType([1, 2, 3])
st2 = sy.SemanticType([1, 2, 3, 4])
assert st1 != st2
def find_expression_from_str(expression_str):
"""
Find the expression from its string representation.
@ -354,7 +379,7 @@ def test__add_tag(token, expected_output):
:type token: sy.SemanticType
:type expected_output: str
"""
assert sy._add_tag(token) == expected_output
assert token.to_markup() == expected_output
# Test text functions
@ -374,3 +399,18 @@ def test_readable_markup(token_list, parsing_expression, expected):
result = expr.parse_string(token_list)
assert result is not None
assert sy.readable_markup(result) == expected
def test_weird_markup():
"""
Test that the readable markup is correctly generated.
"""
assert sy.readable_markup([sy.EmptySemanticType("weird")]) == "weird"
empty_token = sy.SemanticType(None)
empty_token.label = "weird"
assert sy.readable_markup([empty_token]) == "<[NOC] weird />"
class WeirdClass:
def __str__(self):
return "weird"
assert sy.readable_markup([sy.EmptySemanticType("weird"), empty_token, WeirdClass()]) == "weird <[NOC] weird /> weird"