bootstraparse/src/bootstraparse/modules/syntax.py

298 lines
9.8 KiB
Python

# Dedicated module for the syntax of all the parsing
import os
import pyparsing as pp
import rich
pps = pp.Suppress
# Semantic group types
class SemanticType:
"""
Allows us to access basic operations and identify each token parsed.
"""
label = None
def __init__(self, content):
self.content = content
def __str__(self):
if type(self.content) == str:
return f'{self.label}[{self.content}]'
else:
return f'{self.label}{self.content}'
def __repr__(self):
return str(self)
def __eq__(self, other):
if type(other) == type(self):
for e1, e2 in zip(self.content, other.content):
if e1 != e2:
return False
return True
return False
class UnimplementedToken(SemanticType):
label = "unimplemented"
class AliasToken(SemanticType):
label = "alias"
class ImageToken(SemanticType):
label = "image"
class TextToken(SemanticType):
label = "text"
class EnhancedToken(SemanticType):
label = "text:enhanced"
class EtEmToken(SemanticType):
label = "text:em"
class EtStrongToken(SemanticType):
label = "text:strong"
class EtUnderlineToken(SemanticType):
label = "text:underline"
class EtStrikethroughToken(SemanticType):
label = "text:strikethrough"
class EtCustomSpanToken(SemanticType):
label = "text:custom_span"
class EtUlistToken(SemanticType):
label = "text:ulist"
class HeaderToken(SemanticType):
label = "header"
class StructuralElementToken(SemanticType):
label = "structural element"
class HyperlinkToken(SemanticType):
label = "hyperlink"
class TableToken(SemanticType):
label = "table"
class TableRowToken(SemanticType):
label = "table:row"
class TableCellToken(SemanticType):
label = "table:cell"
class TableSeparatorToken(SemanticType):
label = "table:separator"
def of_type(token_class):
"""
Function creating a custom function for generating the given Token type.
:param token_class: SemanticType class
:return: returns a function creating an instance of the given class
"""
def _of_type(_, __, content):
return token_class(content)
return _of_type
def readable_markup(list_of_tokens):
"""
Function used for testing and readability purposes. Replaces matched markup with html-like tags.
:param list_of_tokens: list of matched markup tokens in analysed text.
:return: returns readable string.
:rtype: string
"""
readable_string = ''
for token in list_of_tokens:
if type(token) == str:
readable_string += token
elif token.label == 'text:em':
readable_string += _add_tag('em', readable_markup(token.content))
elif token.label == 'text:strong':
readable_string += _add_tag('b', readable_markup(token.content))
elif token.label == 'text:underline':
readable_string += _add_tag('u', readable_markup(token.content))
elif token.label == 'text:strikethrough':
readable_string += _add_tag('st', readable_markup(token.content))
elif token.label == 'text:custom_span':
readable_string += _add_tag('#', readable_markup(token.content))
elif token.label == 'text':
readable_string += ' ' + ' '.join(token.content)
else:
raise Exception('Unknown token type' + token.label)
return readable_string[1:]
def _add_tag(tag, content):
if content:
return f' <{tag}>{content}</{tag}>'
return ''
# create_image_token = of_type(ImageToken)
# create_image_token()
# Base elements
quotes = pp.Word(r""""'""")
value = (quotes + pp.Word(pp.alphanums + r'.') + pp.match_previous_literal(quotes) ^
pp.common.fnumber)("value")
assignation = pp.Group(pp.common.identifier('var_name') + '=' + value('var_value'))("assignation")
text = pp.OneOrMore(pp.Word(pp.alphanums))('text').add_parse_action(of_type(TextToken))
http_characters = pp.Word(pp.alphanums + r'=+-_\/\\.:;!?%#@&*()[]{}~` ')
# Composite elements
var = '[' + pp.delimitedList(assignation ^ value)("list_vars").set_name("list_vars") + ']'
# Specific elements
image_element = ('@{' + pp.common.identifier('image_name') + '}')("image_element")
alias_element = ('@[' + pp.common.identifier('alias_name') + ']')("alias_element")
expression = pp.Word(pp.alphanums + r'=+-_\'",;:!<> ')
html_insert = '{' + expression('html_insert') + '}'
structural_elements = (pp.Word('div') ^ pp.Word('article') ^ pp.Word('aside') ^ pp.Word('section'))('structural_element')
# Optional elements
optional = (pp.Opt(html_insert)("html_insert") + pp.Opt(var)("var"))("optional")
# TODO : Everywhere : Avoid text at all costs since it is not a valid token, make a new token for it or use pp.SkipTo or (...)
# Enhanced text elements # TODO: Add all markups so that they can be parsed, they no longer need to be linked
# et_em = ('*' + enhanced_text + '*')('em').add_parse_action(of_type(EtEmToken))
# et_strong = ('**' + enhanced_text + '**')('strong').add_parse_action(of_type(EtStrongToken))
# et_underline = ('__' + enhanced_text + '__')('underline').add_parse_action(of_type(EtUnderlineToken))
# et_strikethrough = ('~~' + enhanced_text + '~~')('strikethrough').add_parse_action(of_type(EtStrikethroughToken))
# custom_span = ('(#' + pp.Word(pp.nums)('span_id') + ')').set_name('span_tag')
# et_custom_span = \
# (custom_span + enhanced_text + pp.match_previous_literal(custom_span))('custom_span')\
# .add_parse_action(of_type(EtCustomSpanToken))
# enhanced_text <== (text | et_strong | et_em | et_underline |
# et_strikethrough | et_custom_span) + pp.Opt(enhanced_text)
# Multiline elements
se_start = '~~' + structural_elements # TODO : It's not a text, rather a keyword and add ("name") and .add_parse_action(of_type(TextToken))
se_end = text + '~~' # TODO : Idem
# Inline elements
il_link = '[' + text + ']' + '(' + quotes + http_characters + pp.match_previous_literal(quotes) + ')'
# Todo: change the quotes to quoted_string and get rid of the text which was a placeholder
# Oneline elements
one_header = ('#' + text + pps('#')).add_parse_action(of_type(HeaderToken))
# 1 per hX or copy paste six times?
# TODO: add a header level to the token and get rid of the text which was a placeholder
one_olist = pp.line_start + (pp.Word(pp.nums) ^ pp.Word('#')) + '.' + text
# TODO: add a ("name") and add .add_parse_action(of_type(TextToken))
one_ulist = pp.line_start + ('-' + text).add_parse_action(of_type(EtUlistToken))
# TODO: add a ("name") and add .add_parse_action(of_type(TextToken)) and get rid of the text which was a placeholder
# Final elements
enhanced_text = text # TODO: It's actually text or et_em or et_strong etc..., either recursive or repeated
##############################################################################
# Pre_parser elements
##############################################################################
# Composite elements
image = image_element + optional
alias = alias_element + optional
# Syntax elements
line_to_replace = pp.OneOrMore(pp.SkipTo(image ^ alias)('text').add_parse_action(of_type(TextToken)) ^
image.add_parse_action(of_type(ImageToken)) ^
alias.add_parse_action(of_type(AliasToken)))\
^ pp.SkipTo(pp.lineEnd)('text').add_parse_action(of_type(TextToken))
##############################################################################
# Temporary tests
##############################################################################
if __name__ == '__main__': # pragma: no cover
pp.autoname_elements()
def find_color(new_time, old_time):
pct = percent(new_time, old_time)
if pct < 80:
return "red"
elif pct > 130:
return "green"
return "yellow"
def percent(new_time, old_time):
return round((old_time/new_time) * 100, 2)
list_strings = [
"Normal text",
"Normal text *em text*",
"Normal text **strong text**",
"Normal text __underline text__",
"Normal text ~~strikethrough text~~",
"Normal text (#123) custom span text(#123)",
"**Strong text __underline__** normal text",
"Normal text **strong text *em text*** *em text **strong text*** normal text",
"~~div",
"div~~",
"[link text]('http://test.icule')",
"# header #",
"1. ordered list",
"- unordered list",
]
# div_start.parse_string(list_strings[8])
# div_end.parse_string(list_strings[9])
# il_link.parse_string(list_strings[10])
# one_header.parse_string(list_strings[11])
# one_olist.parse_string(list_strings[12])
# one_ulist.parse_string(list_strings[13])
# enhanced_text.create_diagram("../../../dev_outputs/diagram.html")
# for string in list_strings:
# print('Input string:', string)
# output = enhanced_text.parseString(string)
# # rich.inspect(output)
# print('Output string:', readable_markup(output))
# print()
if not os.path.exists('../../../dev_outputs/'):
os.mkdir('../../../dev_outputs/')
enhanced_text.create_diagram("../../../dev_outputs/diagram.html")
import timeit
base_time = {}
for string in list_strings:
print('Input string:', string)
n_time = timeit.timeit("enhanced_text.parseString(string)", number=100, globals=globals())
base_time[string] = n_time
output = enhanced_text.parseString(string)
# rich.inspect(output)
rich.print(f'[] Output string: {readable_markup(output)} in {n_time:.2}s)')
print()
time = timeit.timeit("enhanced_text.parseString(mega_string)", number=5, globals=globals())
rich.print(f'[]Mega time: {time}')
base_time['mega_string'] = time
print('Base time:')
for string, time in base_time.items():
print(f' "{string}": {time},')