# Dedicated module for the syntax of all the parsing import os import pyparsing as pp import rich pps = pp.Suppress # Semantic group types class SemanticType: """ Allows us to access basic operations and identify each token parsed. """ label = None def __init__(self, content): self.content = content def __str__(self): if type(self.content) == str: return f'{self.label}[{self.content}]' else: return f'{self.label}{self.content}' def __repr__(self): return str(self) def __eq__(self, other): if type(other) == type(self): for e1, e2 in zip(self.content, other.content): if e1 != e2: return False return True return False class UnimplementedToken(SemanticType): label = "unimplemented" class AliasToken(SemanticType): label = "alias" class ImageToken(SemanticType): label = "image" class TextToken(SemanticType): label = "text" class EnhancedToken(SemanticType): label = "text:enhanced" class EtEmToken(SemanticType): label = "text:em" class EtStrongToken(SemanticType): label = "text:strong" class EtUnderlineToken(SemanticType): label = "text:underline" class EtStrikethroughToken(SemanticType): label = "text:strikethrough" class EtCustomSpanToken(SemanticType): label = "text:custom_span" class EtUlistToken(SemanticType): label = "text:ulist" class HeaderToken(SemanticType): label = "header" class StructuralElementToken(SemanticType): label = "structural element" class HyperlinkToken(SemanticType): label = "hyperlink" class TableToken(SemanticType): label = "table" class TableRowToken(SemanticType): label = "table:row" class TableCellToken(SemanticType): label = "table:cell" class TableSeparatorToken(SemanticType): label = "table:separator" def of_type(token_class): """ Function creating a custom function for generating the given Token type. :param token_class: SemanticType class :return: returns a function creating an instance of the given class """ def _of_type(_, __, content): return token_class(content) return _of_type def readable_markup(list_of_tokens): """ Function used for testing and readability purposes. Replaces matched markup with html-like tags. :param list_of_tokens: list of matched markup tokens in analysed text. :return: returns readable string. :rtype: string """ readable_string = '' for token in list_of_tokens: if type(token) == str: readable_string += token elif token.label == 'text:em': readable_string += _add_tag('em', readable_markup(token.content)) elif token.label == 'text:strong': readable_string += _add_tag('b', readable_markup(token.content)) elif token.label == 'text:underline': readable_string += _add_tag('u', readable_markup(token.content)) elif token.label == 'text:strikethrough': readable_string += _add_tag('st', readable_markup(token.content)) elif token.label == 'text:custom_span': readable_string += _add_tag('#', readable_markup(token.content)) elif token.label == 'text': readable_string += ' ' + ' '.join(token.content) else: raise Exception('Unknown token type' + token.label) return readable_string[1:] def _add_tag(tag, content): if content: return f' <{tag}>{content}' return '' # create_image_token = of_type(ImageToken) # create_image_token() # Base elements quotes = pp.Word(r""""'""") value = (quotes + pp.Word(pp.alphanums + r'.') + pp.match_previous_literal(quotes) ^ pp.common.fnumber)("value") assignation = pp.Group(pp.common.identifier('var_name') + '=' + value('var_value'))("assignation") text = pp.OneOrMore(pp.Word(pp.alphanums))('text').add_parse_action(of_type(TextToken)) http_characters = pp.Word(pp.alphanums + r'=+-_\/\\.:;!?%#@&*()[]{}~` ') # Composite elements var = '[' + pp.delimitedList(assignation ^ value)("list_vars").set_name("list_vars") + ']' # Specific elements image_element = ('@{' + pp.common.identifier('image_name') + '}')("image_element") alias_element = ('@[' + pp.common.identifier('alias_name') + ']')("alias_element") expression = pp.Word(pp.alphanums + r'=+-_\'",;:!<> ') html_insert = '{' + expression('html_insert') + '}' structural_elements = (pp.Word('div') ^ pp.Word('article') ^ pp.Word('aside') ^ pp.Word('section'))('structural_element') # Optional elements optional = (pp.Opt(html_insert)("html_insert") + pp.Opt(var)("var"))("optional") # TODO : Everywhere : Avoid text at all costs since it is not a valid token, make a new token for it or use pp.SkipTo or (...) # Enhanced text elements # TODO: Add all markups so that they can be parsed, they no longer need to be linked # et_em = ('*' + enhanced_text + '*')('em').add_parse_action(of_type(EtEmToken)) # et_strong = ('**' + enhanced_text + '**')('strong').add_parse_action(of_type(EtStrongToken)) # et_underline = ('__' + enhanced_text + '__')('underline').add_parse_action(of_type(EtUnderlineToken)) # et_strikethrough = ('~~' + enhanced_text + '~~')('strikethrough').add_parse_action(of_type(EtStrikethroughToken)) # custom_span = ('(#' + pp.Word(pp.nums)('span_id') + ')').set_name('span_tag') # et_custom_span = \ # (custom_span + enhanced_text + pp.match_previous_literal(custom_span))('custom_span')\ # .add_parse_action(of_type(EtCustomSpanToken)) # enhanced_text <== (text | et_strong | et_em | et_underline | # et_strikethrough | et_custom_span) + pp.Opt(enhanced_text) # Multiline elements se_start = '~~' + structural_elements # TODO : It's not a text, rather a keyword and add ("name") and .add_parse_action(of_type(TextToken)) se_end = text + '~~' # TODO : Idem # Inline elements il_link = '[' + text + ']' + '(' + quotes + http_characters + pp.match_previous_literal(quotes) + ')' # Todo: change the quotes to quoted_string and get rid of the text which was a placeholder # Oneline elements one_header = ('#' + text + pps('#')).add_parse_action(of_type(HeaderToken)) # 1 per hX or copy paste six times? # TODO: add a header level to the token and get rid of the text which was a placeholder one_olist = pp.line_start + (pp.Word(pp.nums) ^ pp.Word('#')) + '.' + text # TODO: add a ("name") and add .add_parse_action(of_type(TextToken)) one_ulist = pp.line_start + ('-' + text).add_parse_action(of_type(EtUlistToken)) # TODO: add a ("name") and add .add_parse_action(of_type(TextToken)) and get rid of the text which was a placeholder # Final elements enhanced_text = text # TODO: It's actually text or et_em or et_strong etc..., either recursive or repeated ############################################################################## # Pre_parser elements ############################################################################## # Composite elements image = image_element + optional alias = alias_element + optional # Syntax elements line_to_replace = pp.OneOrMore(pp.SkipTo(image ^ alias)('text').add_parse_action(of_type(TextToken)) ^ image.add_parse_action(of_type(ImageToken)) ^ alias.add_parse_action(of_type(AliasToken)))\ ^ pp.SkipTo(pp.lineEnd)('text').add_parse_action(of_type(TextToken)) ############################################################################## # Temporary tests ############################################################################## if __name__ == '__main__': # pragma: no cover pp.autoname_elements() def find_color(new_time, old_time): pct = percent(new_time, old_time) if pct < 80: return "red" elif pct > 130: return "green" return "yellow" def percent(new_time, old_time): return round((old_time/new_time) * 100, 2) list_strings = [ "Normal text", "Normal text *em text*", "Normal text **strong text**", "Normal text __underline text__", "Normal text ~~strikethrough text~~", "Normal text (#123) custom span text(#123)", "**Strong text __underline__** normal text", "Normal text **strong text *em text*** *em text **strong text*** normal text", "~~div", "div~~", "[link text]('http://test.icule')", "# header #", "1. ordered list", "- unordered list", ] # div_start.parse_string(list_strings[8]) # div_end.parse_string(list_strings[9]) # il_link.parse_string(list_strings[10]) # one_header.parse_string(list_strings[11]) # one_olist.parse_string(list_strings[12]) # one_ulist.parse_string(list_strings[13]) # enhanced_text.create_diagram("../../../dev_outputs/diagram.html") # for string in list_strings: # print('Input string:', string) # output = enhanced_text.parseString(string) # # rich.inspect(output) # print('Output string:', readable_markup(output)) # print() if not os.path.exists('../../../dev_outputs/'): os.mkdir('../../../dev_outputs/') enhanced_text.create_diagram("../../../dev_outputs/diagram.html") import timeit base_time = {} for string in list_strings: print('Input string:', string) n_time = timeit.timeit("enhanced_text.parseString(string)", number=100, globals=globals()) base_time[string] = n_time output = enhanced_text.parseString(string) # rich.inspect(output) rich.print(f'[] Output string: {readable_markup(output)} in {n_time:.2}s)') print() time = timeit.timeit("enhanced_text.parseString(mega_string)", number=5, globals=globals()) rich.print(f'[]Mega time: {time}') base_time['mega_string'] = time print('Base time:') for string, time in base_time.items(): print(f' "{string}": {time},')