From 06538d61028074a56e5612b7ccd2820dcef07eec Mon Sep 17 00:00:00 2001 From: Maximilien Rigaut Date: Sun, 20 Mar 2022 07:51:29 +0100 Subject: [PATCH 01/13] Added tools for test verbosity --- tests/modules/test_syntax.py | 35 ++++++++--- tests/modules/tools.py | 113 +++++++++++++++++++++++++++++++++++ 2 files changed, 141 insertions(+), 7 deletions(-) create mode 100644 tests/modules/tools.py diff --git a/tests/modules/test_syntax.py b/tests/modules/test_syntax.py index c8471c7..bbbded5 100644 --- a/tests/modules/test_syntax.py +++ b/tests/modules/test_syntax.py @@ -2,19 +2,33 @@ from itertools import zip_longest import pyparsing import pytest -import inspect - import bootstraparse.modules.syntax as sy - +from tools import __GL, __module_path, find_variables_in_file # , find_functions_in_file, find_classes_in_file ptp = pytest.param __XF = pytest.mark.xfail -# Cursed frame inspection -def __GL(): - return inspect.getframeinfo(inspect.currentframe().f_back).lineno +############################################################################################################## +# Utility functions +############################################################################################################## +def likely_definition(list_expression, file): + """ + Returns a dict for each expresion and the associated line where this expression is likely to be defined. + :param list_expression: list of expressions + :param file: file to search in + :return: dict of expression and line + """ + likely_definition_dict = {} + loop_list = [e for e in list_expression] + with open(file) as f: + for line_number, line in enumerate(f): + for expression_id, expression in enumerate(loop_list): + if expression in line: + likely_definition_dict[expression] = line_number+1 + loop_list.remove(expression) + return likely_definition_dict, file ############################################################################################################## @@ -368,6 +382,9 @@ zipped_dict_advanced_syntax_input_and_expected_output = [ ] for item in sublist ] +__syntax_file = __module_path("syntax.py") +__definition_of_syntax_elements = find_variables_in_file(__syntax_file, dict_advanced_syntax_input_and_expected_output.keys()) + # test__add_tag list_add_tag_input_and_expected_output = [ # Test cases for the add_tag function @@ -551,8 +568,12 @@ def test_advanced_expression_and_token_creation(markup_element, to_parse, expect assert isinstance(expr, pyparsing.ParserElement) result = expr.parse_string(to_parse) print() - print(f"Parsing expression: {to_parse} with {markup_element}") + print(f"Parsing string: '{to_parse}'") print("Defined at %(filename)s:%(lineno)d" % {'filename': __file__, 'lineno': line_test}) + print(f"With expression: '{markup_element}'") + # print(likely_definition_of_advanced_syntax) + print("Defined at %(filename)s:%(lineno)d" % {'filename': __syntax_file, + 'lineno': __definition_of_syntax_elements[markup_element]}) print(f"Found: {result} (len:{len(result)}).") print(f"Expected: {expected} (len:{len(expected)})") assert result is not None diff --git a/tests/modules/tools.py b/tests/modules/tools.py new file mode 100644 index 0000000..8b308ec --- /dev/null +++ b/tests/modules/tools.py @@ -0,0 +1,113 @@ +import inspect +import os +import re + + +############################################################################### +# List of tools for testing +############################################################################### +############################################################################### +# Paths +######################################## + + +def __base(): + """ + Return the path of the main module (relative to the tests) + :return: Path of the main module + :rtype: str + """ + return "../../src/bootstraparse/" + + +def __module_path(module_name): + """ + Return the path of the module module_name + :param module_name: Name of the module + :type module_name: str + :return: Path of the module + :rtype: str + """ + return os.path.normpath(os.path.join(os.path.dirname(__file__), __base(), "modules", module_name)) + + +############################################################################### +# Frame inspection +######################################## +# Cursed frame inspection +def __GL(): + return inspect.getframeinfo(inspect.currentframe().f_back).lineno + + +############################################################################### +# Code inspection +######################################## +def find_string_in_file(file_path, list_regex_to_find): + """ + Find the string in the file stream + :param file_path: Path of the file + :type file_path: str + :param list_regex_to_find: List of string to find + :type list_regex_to_find: list + :return: Dict of string found and line number + :rtype: dict + """ + dict_string_found = {} + list_regex_to_find_left = [string for string in list_regex_to_find] + with open(file_path, "r") as file: + for line_number, line in enumerate(file): + for string_number, regex_to_find in enumerate(list_regex_to_find_left): + match = regex_to_find.search(line) + if match: + dict_string_found[match.group(1)] = line_number + 1 + list_regex_to_find_left.pop(string_number) + break + if not list_regex_to_find_left: + break + if list_regex_to_find_left: + print("String not found: " + str(list_regex_to_find_left)) + return dict_string_found + + +def find_functions_in_file(file_path, list_function_name): + """ + Find the function in the file stream + :param file_path: Path of the file + :type file_path: str + :param list_function_name: List of function name to find + :type list_function_name: iterable + :return: Dict of function found and line number + :rtype: dict + """ + return find_string_in_file(file_path, [re.compile(f"$def ({function_name})") for function_name in list_function_name]) + + +def find_classes_in_file(file_path, list_class_name): + """ + Find the class in the file stream + :param file_path: Path of the file + :type file_path: str + :param list_class_name: List of class name to find + :type list_class_name: iterable + :return: Dict of class found and line number + :rtype: dict + """ + return find_string_in_file(file_path, [re.compile(f"^class ({class_name})") for class_name in list_class_name]) + + +def find_variables_in_file(file_path, list_variable_name): + """ + Find the variable in the file stream + :param file_path: Path of the file + :type file_path: str + :param list_variable_name: List of variable name to find + :type list_variable_name: iterable + :return: Dict of variable found and line number + :rtype: dict + """ + return find_string_in_file(file_path, [re.compile(f"^({variable_name}) ?=") for variable_name in list_variable_name]) + + +if __name__ == "__main__": + # print(find_string_in_file(__module_path("syntax.py"), [re.compile(r"^(html_insert) ?=")])) + pass From a5564669990b0564419e6d36380b4ab499b78e35 Mon Sep 17 00:00:00 2001 From: Maximilien Rigaut Date: Sun, 20 Mar 2022 18:07:35 +0100 Subject: [PATCH 02/13] Added all syntax elements to tests, still to reach 100% --- src/bootstraparse/modules/syntax.py | 21 +++-- tests/modules/test_syntax.py | 127 +++++++++++++++++++++++++++- 2 files changed, 136 insertions(+), 12 deletions(-) diff --git a/src/bootstraparse/modules/syntax.py b/src/bootstraparse/modules/syntax.py index f4e561a..775dbba 100644 --- a/src/bootstraparse/modules/syntax.py +++ b/src/bootstraparse/modules/syntax.py @@ -235,7 +235,7 @@ value = (pps(quotes) + pp.Word(pp.alphanums + r'.') + pps(pp.match_previous_lite pp.common.fnumber)("value") assignation = pp.Group( pp.common.identifier('var_name') + pps('=') + value('var_value') -)("assignation") # TODO: More extensive testing +)("assignation") text = pp.OneOrMore(pp.Word(pp.alphanums))('text').add_parse_action(of_type(TextToken)) url_characters = pp.common.url @@ -243,20 +243,25 @@ url_characters = pp.common.url var = pps('[') + pp.delimitedList( assignation.add_parse_action(of_type(BeAssignToken)) ^ value.add_parse_action(of_type(BeValueToken)) -)("list_vars").set_name("list_vars") + pps(']') # TODO: More extensive testing +)("list_vars").set_name("list_vars") + pps(']') # Specific elements image_element = ('@{' + pp.common.identifier('image_name') + '}')("image_element") alias_element = ('@[' + pp.common.identifier('alias_name') + ']')("alias_element") expression = pp.Word(pp.alphanums + r'=+-_\'",;:!\/\\ ') -html_insert = pps('{') + expression('html_insert') + pps('}') # TODO: More extensive testing -class_insert = pps('{{') + expression('class_insert') + pps('}}') # TODO: test dis +html_insert = pps('{') + expression('html_insert') + pps('}') +class_insert = pps('{{') + expression('class_insert') + pps('}}') # Optional elements -optional = ( - pp.Opt(class_insert)("class_insert").add_parse_action(of_type(OptionalClassToken)) & - pp.Opt(html_insert)("html_insert").add_parse_action(of_type(OptionalInsertToken)) & - pp.Opt(var)("var").add_parse_action(of_type(OptionalVarToken)) # noqa TODO: add class_insert to tests +# optional = ( +# pp.Opt(class_insert)("class_insert").add_parse_action(of_type(OptionalClassToken)) & +# pp.Opt(html_insert)("html_insert").add_parse_action(of_type(OptionalInsertToken)) & +# pp.Opt(var)("var").add_parse_action(of_type(OptionalVarToken)) +# )("optional").add_parse_action(of_type(OptionalToken)) # Macro OptionalToken +optional = pp.ZeroOrMore( + class_insert("class_insert").add_parse_action(of_type(OptionalClassToken)) | + html_insert("html_insert").add_parse_action(of_type(OptionalInsertToken)) | + var("var").add_parse_action(of_type(OptionalVarToken)) )("optional").add_parse_action(of_type(OptionalToken)) # Macro OptionalToken # Structural elements diff --git a/tests/modules/test_syntax.py b/tests/modules/test_syntax.py index bbbded5..adea19e 100644 --- a/tests/modules/test_syntax.py +++ b/tests/modules/test_syntax.py @@ -124,8 +124,117 @@ dict_advanced_syntax_input_and_expected_output = { ("[text_link]('text://www.website.com/link.html')", [sy.HyperlinkToken(["[text_link]('text://www.website.com/link.html')"])], __GL()) ], - - # il_link | et_strong | et_em | et_strikethrough | et_underline | et_custom_span + "assignation": [ + # Single assignation element, should only match the first "a=1" + ("a=1", [sy.BeAssignToken([["a", 1]]), ], __GL()), + ("a='str'", [sy.BeAssignToken([["a", 'str']]), ], __GL()), + ], + "var": [ + # List of variables or assignations, should match "[a=12, b=13, 'c']" + ("[a=12, b=13, 'c']", [ + sy.BeAssignToken([["a", 12]]), + sy.BeAssignToken([["b", 13]]), + sy.BeValueToken(["c"]), + ], __GL()), + ("[a=12, b=13, 'c', d=14]", [ + sy.BeAssignToken([["a", 12]]), + sy.BeAssignToken([["b", 13]]), + sy.BeValueToken(["c"]), + sy.BeAssignToken([["d", 14]]), + ], __GL()), + ], + "optional": [ + # Optional element, should match any optional" + ("[a=12, b=13, 'c']", [ + sy.OptionalToken([ + sy.OptionalVarToken([ + sy.BeAssignToken([["a", 12]]), + sy.BeAssignToken([["b", 13]]), + sy.BeValueToken(["c"]), + ]), + ]), + ], __GL()), + ("[a=12, b=13, 'c', d=14]", [ + sy.OptionalToken([ + sy.OptionalVarToken([ + sy.BeAssignToken([["a", 12]]), + sy.BeAssignToken([["b", 13]]), + sy.BeValueToken(["c"]), + sy.BeAssignToken([["d", 14]]), + ]), + ]), + ], __GL()), + ("[a=12]{aze,/.}", ( + sy.OptionalToken([ + sy.OptionalVarToken([ + sy.BeAssignToken([["a", 12]]), + ]), + sy.OptionalInsertToken([ + "aze,/.", + ]), + ]), + ), __GL()), + ("{aze,/.}[a=12]", ( + sy.OptionalToken([ + sy.OptionalInsertToken([ + "aze,/.", + ]), + sy.OptionalVarToken([ + sy.BeAssignToken([["a", 12]]), + ]), + ]), + ), __GL()), + ("{aze!=}", ( + sy.OptionalToken([ + sy.OptionalInsertToken([ + "aze!=", + ]), + ]), + ), __GL()), + ("{{aze!=}}", ( + sy.OptionalToken([ + sy.OptionalClassToken([ + "aze!=", + ]), + ]), + ), __GL()), + ("{{aze!=}}{azett!=}", ( + sy.OptionalToken([ + sy.OptionalClassToken([ + "aze!=", + ]), + sy.OptionalInsertToken([ + "azett!=", + ]), + ]), + ), __GL()), + ("{{aze!=}}{azett!=}[a=12]", ( + sy.OptionalToken([ + sy.OptionalClassToken([ + "aze!=", + ]), + sy.OptionalInsertToken([ + "azett!=", + ]), + sy.OptionalVarToken([ + sy.BeAssignToken([["a", 12]]), + ]), + ]), + ), __GL()), + ("[a=12]{aze,/.}{{azer!=}}", ( + sy.OptionalToken([ + sy.OptionalVarToken([ + sy.BeAssignToken([["a", 12]]), + ]), + sy.OptionalInsertToken([ + "aze,/.", + ]), + sy.OptionalClassToken([ + "azer!=", + ]), + ]), + ), __GL()), + ], # Enhanced Text "enhanced_text": [ # Matches a line of text, with or without inline elements @@ -169,7 +278,18 @@ dict_advanced_syntax_input_and_expected_output = { sy.EtStrikethroughToken(["~~"]), sy.TextToken(["Strikethrough"]), ), __GL()), ], - + "html_insert": [ + # Match a HTML insert + ("{azer='\"}", ( + "azer='\"", + ), __GL()), + ], + "class_insert": [ + # Match a class insert + ("{{azer='\"}}", ( + "azer='\"", + ), __GL()), + ], # Structural Elements "se_start": [ # Matches a start of a structural element @@ -571,7 +691,6 @@ def test_advanced_expression_and_token_creation(markup_element, to_parse, expect print(f"Parsing string: '{to_parse}'") print("Defined at %(filename)s:%(lineno)d" % {'filename': __file__, 'lineno': line_test}) print(f"With expression: '{markup_element}'") - # print(likely_definition_of_advanced_syntax) print("Defined at %(filename)s:%(lineno)d" % {'filename': __syntax_file, 'lineno': __definition_of_syntax_elements[markup_element]}) print(f"Found: {result} (len:{len(result)}).") From ef060185597a5429251943fbc7fe47e1586c0b8e Mon Sep 17 00:00:00 2001 From: Maximilien Rigaut Date: Mon, 21 Mar 2022 08:48:28 +0100 Subject: [PATCH 03/13] Reached 100% coverage and added tests for reparse. Still some errors on this optionnal parameter. --- src/bootstraparse/modules/syntax.py | 2 +- tests/modules/test_syntax.py | 31 ++++++++++++++++++++++++++--- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/bootstraparse/modules/syntax.py b/src/bootstraparse/modules/syntax.py index 775dbba..614b3b9 100644 --- a/src/bootstraparse/modules/syntax.py +++ b/src/bootstraparse/modules/syntax.py @@ -205,7 +205,7 @@ def reparse(parse_element): :return: returns a function parsing given match with specified parsing element """ def _reparse(original_string, _, __): - parse_element.parse_string(original_string) + return parse_element.parse_string(original_string) return _reparse diff --git a/tests/modules/test_syntax.py b/tests/modules/test_syntax.py index adea19e..bf36a4d 100644 --- a/tests/modules/test_syntax.py +++ b/tests/modules/test_syntax.py @@ -173,7 +173,7 @@ dict_advanced_syntax_input_and_expected_output = { "aze,/.", ]), ]), - ), __GL()), + ), __GL(), __XF), ("{aze,/.}[a=12]", ( sy.OptionalToken([ sy.OptionalInsertToken([ @@ -183,7 +183,7 @@ dict_advanced_syntax_input_and_expected_output = { sy.BeAssignToken([["a", 12]]), ]), ]), - ), __GL()), + ), __GL(), __XF), ("{aze!=}", ( sy.OptionalToken([ sy.OptionalInsertToken([ @@ -233,7 +233,7 @@ dict_advanced_syntax_input_and_expected_output = { "azer!=", ]), ]), - ), __GL()), + ), __GL(), __XF), ], # Enhanced Text "enhanced_text": [ @@ -558,6 +558,15 @@ list_of_text_input_and_readable_output = [ ] +list_of_reparsing_input_and_expected_output = [ + ("reparse_test", "enhanced_text", sy.TextToken(["reparse_test"]), __GL()) # TODO : add more test cases +] +final_list_of_reparsing_input_and_expected_output = [ + ptp(item[0], item[1], item[2], item[3], id=f'{item[0]} (line:{item[3]})', marks=item[4:]) + for item in list_of_reparsing_input_and_expected_output +] + + @pytest.mark.parametrize("token_name,token_class", list_of_token_types.items()) def test_token_type_exists(token_name, token_class): """ @@ -640,6 +649,9 @@ def test_semantic_type_eq(): st1 = sy.SemanticType([1, 2, 3]) st2 = sy.SemanticType([1, 2, 3, 4]) assert st1 != st2 + st1 = sy.SemanticType([[1, 2, 4]]) + st2 = sy.SemanticType([[1, 2, 3]]) + assert st1 != st2 def find_expression_from_str(expression_str): @@ -754,3 +766,16 @@ def test_weird_markup(): assert sy.readable_markup( [sy.EmptySemanticType("weird"), empty_token, WeirdClass()]) == "weird <[NOC] weird /> weird" + + +@pytest.mark.parametrize("original_string, reparse_with, expected_output, line_test", + final_list_of_reparsing_input_and_expected_output) +def test_reparse(original_string, reparse_with, expected_output, line_test): + """ + Test the reparse function + """ + expression = find_expression_from_str(reparse_with) + reparse_action = pyparsing.Word(pyparsing.alphanums).add_parse_action(sy.reparse(expression)) + reparse_result = reparse_action.parseString(original_string) + assert reparse_result is not None + assert reparse_result[0] == expected_output From af73e3c3e58a129a2988ac6935c55b09bfc3eeeb Mon Sep 17 00:00:00 2001 From: vky Date: Tue, 22 Mar 2022 15:22:18 +0100 Subject: [PATCH 04/13] made hyperlinks markdown-compatible --- src/bootstraparse/modules/syntax.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstraparse/modules/syntax.py b/src/bootstraparse/modules/syntax.py index 614b3b9..d83fd65 100644 --- a/src/bootstraparse/modules/syntax.py +++ b/src/bootstraparse/modules/syntax.py @@ -276,7 +276,7 @@ display_element = pp.Word('!') # Inline elements il_link = pp.Regex( - r"""\[(?P.+)\]\(['"](?P[a-zA-Z-_:\/=@#!%\?\d\(\)\.]+)['"]\)""" + r"""\[(?P.+)\]\(['"]?(?P[a-zA-Z-_:\/=@#!%\?\d\(\)\.]+)['"]?\)""" ).add_parse_action(of_type(HyperlinkToken)) # Enhanced text elements From c07504ba6efe33dff18398b53f87580468eb6e04 Mon Sep 17 00:00:00 2001 From: vky Date: Tue, 22 Mar 2022 15:22:24 +0100 Subject: [PATCH 05/13] added specs --- workflow-documents/specs.yaml | 110 ++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 workflow-documents/specs.yaml diff --git a/workflow-documents/specs.yaml b/workflow-documents/specs.yaml new file mode 100644 index 0000000..c58a579 --- /dev/null +++ b/workflow-documents/specs.yaml @@ -0,0 +1,110 @@ +# Enhanced text +italics: + bpr: '*italics*' + markdown compatible: yes + additional unsupported markdown syntax: '_italics_' + accepts arguments: no + type: enhanced text + +bold: + bpr: '**bold**' + markdown compatible: yes + accepts arguments: no + type: enhanced text + +strikethrough: + bpr: '~~strikethrough~~' + markdown compatible: yes + accepts arguments: no + type: enhanced text + +underline: + bpr: '__underline__' + markdown compatible: - + accepts arguments: no + type: enhanced text + + +# In-line elements +hyperlinks: + bpr: '[text]("hyperlink")' + markdown compatible: yes + accepts arguments: no + type: in-line + +# Pre-parsing elements +aliases: + bpr: '@[alias]' + markdown compatible: - + accepts arguments: yes + type: pre-parsing + +images: + bpr: '@{image}' + markdown compatible: no + accepts arguments: yes + type: pre-parsing + +# One-line elements +headers: + bpr: '# header #' + markdown compatible: no + accepts arguments: yes + accepts enhanced text: no + type: one-line + +display: + bpr: '! header !' + markdown compatible: - + accepts arguments: yes + accepts enhanced text: no + type: one-line + +# Not sure about sublists for the moment, add that later +numbered lists: + bpr: '#. nbd list item' + markdown compatible: no + accepts arguments: no + accepts enhanced text: yes + type: one-line + +bullet lists: + bpr: '- list item' + markdown compatible: yes + accepts arguments: no + accepts enhanced text: yes + type: one-line + + +# Multi-line elements +structural elements: + bpr: '<>' + accepted elements: div, article, aside, section + markdown compatible: - + accepts arguments: yes + type: multi-line + +tables: + bpr: '| table cell |' + markdown compatible: - + accepts arguments: yes + accepts enhanced text: yes + type: multi-line + +table-separator: + bpr: '|-----------|' + # accepts ':' either at the start, middle or end of the line to align text for the column + # putting a separator after a row will change said row to table-header + # putting a separator after a column will change said column to table-header + markdown compatible: - + accepts arguments: yes + accepts enhanced text: yes + type: multi-line + + +# # Things to add +# blockquotes: +# code: +# code-blocks: +# horizontal rule: +# reference-style link: From 755416cbc599df9c96145a2d258da42fbea4696a Mon Sep 17 00:00:00 2001 From: vky Date: Wed, 23 Mar 2022 15:49:42 +0100 Subject: [PATCH 06/13] added todos --- workflow-documents/specs.yaml | 60 ++++++++++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 5 deletions(-) diff --git a/workflow-documents/specs.yaml b/workflow-documents/specs.yaml index c58a579..b55cc92 100644 --- a/workflow-documents/specs.yaml +++ b/workflow-documents/specs.yaml @@ -5,24 +5,28 @@ italics: additional unsupported markdown syntax: '_italics_' accepts arguments: no type: enhanced text + scope: first release bold: bpr: '**bold**' markdown compatible: yes accepts arguments: no type: enhanced text + scope: first release strikethrough: bpr: '~~strikethrough~~' markdown compatible: yes accepts arguments: no type: enhanced text + scope: first release underline: bpr: '__underline__' markdown compatible: - accepts arguments: no type: enhanced text + scope: first release # In-line elements @@ -31,6 +35,7 @@ hyperlinks: markdown compatible: yes accepts arguments: no type: in-line + scope: first release # Pre-parsing elements aliases: @@ -38,12 +43,14 @@ aliases: markdown compatible: - accepts arguments: yes type: pre-parsing + scope: first release images: bpr: '@{image}' markdown compatible: no accepts arguments: yes type: pre-parsing + scope: first release # One-line elements headers: @@ -52,6 +59,7 @@ headers: accepts arguments: yes accepts enhanced text: no type: one-line + scope: first release display: bpr: '! header !' @@ -59,6 +67,7 @@ display: accepts arguments: yes accepts enhanced text: no type: one-line + scope: first release # Not sure about sublists for the moment, add that later numbered lists: @@ -67,6 +76,7 @@ numbered lists: accepts arguments: no accepts enhanced text: yes type: one-line + scope: first release bullet lists: bpr: '- list item' @@ -74,6 +84,7 @@ bullet lists: accepts arguments: no accepts enhanced text: yes type: one-line + scope: first release # Multi-line elements @@ -83,6 +94,7 @@ structural elements: markdown compatible: - accepts arguments: yes type: multi-line + scope: first release tables: bpr: '| table cell |' @@ -90,6 +102,7 @@ tables: accepts arguments: yes accepts enhanced text: yes type: multi-line + scope: first release table-separator: bpr: '|-----------|' @@ -100,11 +113,48 @@ table-separator: accepts arguments: yes accepts enhanced text: yes type: multi-line + scope: first release # # Things to add -# blockquotes: -# code: -# code-blocks: -# horizontal rule: -# reference-style link: +blockquotes: + bpr: '> quote' + markdown compatible: yes + accepts arguments: yes + accepts enhanced text: yes + type: one-line + scope: first release + +code: + bpr: '`quote`' + markdown compatible: yes + accepts arguments: no + accepts enhanced text: no + type: in-line + scope: post-release + +code-blocks: + bpr: '```quote```' + markdown compatible: yes + accepts arguments: yes + accepts enhanced text: no + type: multi-line + scope: post-release + +horizontal rule: + bpr: '---' + markdown compatible: yes + accepts arguments: yes + accepts enhanced text: no + type: one-line + scope: first release + +reference-style link: + bpr: '[digit]("hyperlink")' + markdown syntax: '[link][digit]' # needs testing, unclear + accepts arguments: yes + accepts enhanced text: no + type: one-line + scope: first release + +# Inline-style link with title \ No newline at end of file From 67cbad1daebe87a36e9db397c42fc608fd699ce2 Mon Sep 17 00:00:00 2001 From: Maximilien Rigaut Date: Thu, 24 Mar 2022 09:32:57 +0100 Subject: [PATCH 07/13] Upgraded requirements. Let's not be outdated before release. --- requirements.txt | 6 +++--- requirements_dev.txt | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/requirements.txt b/requirements.txt index 5dd2765..610719e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ pyyaml==6.0 -rich~=11.0.0 -regex~=2022.1.18 +rich==12.0.1 +regex==2022.3.15 pyparse~=1.1.7 pyparsing[diagrams]~=3.0.7 -pytest~=6.2.5 \ No newline at end of file +pytest==7.1.1 \ No newline at end of file diff --git a/requirements_dev.txt b/requirements_dev.txt index aa8db7f..156e401 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,8 +1,8 @@ flake8==4.0.1 -tox==3.24.3 -pytest==6.2.5 -pytest-cov==2.12.1 -mypy===0.910 +tox==3.24.5 +pytest==7.1.1 +pytest-cov==3.0.0 +mypy==0.941 pyyaml==6.0 -rich~=11.0.0 -regex~=2022.1.18 +rich==12.0.1 +regex==2022.3.15 From 45439b55b9f06f8d53202cdd3aae98f5d04b2bcd Mon Sep 17 00:00:00 2001 From: Maximilien Rigaut Date: Thu, 24 Mar 2022 11:19:47 +0100 Subject: [PATCH 08/13] PEP and test verbosity --- tests/modules/test_syntax.py | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/tests/modules/test_syntax.py b/tests/modules/test_syntax.py index bf36a4d..a06a8c5 100644 --- a/tests/modules/test_syntax.py +++ b/tests/modules/test_syntax.py @@ -15,7 +15,7 @@ __XF = pytest.mark.xfail ############################################################################################################## def likely_definition(list_expression, file): """ - Returns a dict for each expresion and the associated line where this expression is likely to be defined. + Returns a dict for each expression and the associated line where this expression is likely to be defined. :param list_expression: list of expressions :param file: file to search in :return: dict of expression and line @@ -144,7 +144,7 @@ dict_advanced_syntax_input_and_expected_output = { ], __GL()), ], "optional": [ - # Optional element, should match any optional" + # Optional element, should match any optional element ("[a=12, b=13, 'c']", [ sy.OptionalToken([ sy.OptionalVarToken([ @@ -279,7 +279,7 @@ dict_advanced_syntax_input_and_expected_output = { ), __GL()), ], "html_insert": [ - # Match a HTML insert + # Match an HTML insert ("{azer='\"}", ( "azer='\"", ), __GL()), @@ -559,7 +559,19 @@ list_of_text_input_and_readable_output = [ list_of_reparsing_input_and_expected_output = [ - ("reparse_test", "enhanced_text", sy.TextToken(["reparse_test"]), __GL()) # TODO : add more test cases + ("reparse_test", "enhanced_text", [sy.TextToken(["reparse_test"])], __GL()), + ("reparse text *bold*, __underlined__ (#123) Custom", "enhanced_text", [ + sy.TextToken(["reparse text"]), + sy.EtEmToken(["*"]), + sy.TextToken(["bold"]), + sy.EtEmToken(["*"]), + sy.TextToken([","]), + sy.EtUnderlineToken(["__"]), + sy.TextToken(["underlined"]), + sy.EtUnderlineToken(["__"]), + sy.EtCustomSpanToken(["123"]), + sy.TextToken(["Custom"]) + ], __GL()) ] final_list_of_reparsing_input_and_expected_output = [ ptp(item[0], item[1], item[2], item[3], id=f'{item[0]} (line:{item[3]})', marks=item[4:]) @@ -774,8 +786,17 @@ def test_reparse(original_string, reparse_with, expected_output, line_test): """ Test the reparse function """ - expression = find_expression_from_str(reparse_with) + print() + print(f"Reparsing string: '{original_string}'") + print("Defined at %(filename)s:%(lineno)d" % {'filename': __file__, 'lineno': int(line_test)}) + print(f"With expression: '{reparse_with}'") + print("Defined at %(filename)s:%(lineno)d" % {'filename': __syntax_file, + 'lineno': __definition_of_syntax_elements[reparse_with]}) + print(f"Expected: {expected_output}") + + expression = find_expression_from_str(str(reparse_with)) reparse_action = pyparsing.Word(pyparsing.alphanums).add_parse_action(sy.reparse(expression)) - reparse_result = reparse_action.parseString(original_string) + reparse_result = reparse_action.parseString(str(original_string)) assert reparse_result is not None - assert reparse_result[0] == expected_output + for expected, result in zip_longest(expected_output, reparse_result): + assert expected == result From c8c863d8921a7a515b9d0409aaeebcd8b6e9daf0 Mon Sep 17 00:00:00 2001 From: Maximilien Rigaut Date: Thu, 24 Mar 2022 11:45:30 +0100 Subject: [PATCH 09/13] Optional fix, the solution was to make optional non-optional --- src/bootstraparse/modules/syntax.py | 37 +++++++++++++++-------------- tests/modules/test_syntax.py | 12 +++++++++- 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/src/bootstraparse/modules/syntax.py b/src/bootstraparse/modules/syntax.py index d83fd65..5aa1de7 100644 --- a/src/bootstraparse/modules/syntax.py +++ b/src/bootstraparse/modules/syntax.py @@ -248,19 +248,14 @@ var = pps('[') + pp.delimitedList( # Specific elements image_element = ('@{' + pp.common.identifier('image_name') + '}')("image_element") alias_element = ('@[' + pp.common.identifier('alias_name') + ']')("alias_element") -expression = pp.Word(pp.alphanums + r'=+-_\'",;:!\/\\ ') +expression = pp.Word(pp.alphanums + r'=+-_\'",;:!\/\\. ') html_insert = pps('{') + expression('html_insert') + pps('}') class_insert = pps('{{') + expression('class_insert') + pps('}}') # Optional elements -# optional = ( -# pp.Opt(class_insert)("class_insert").add_parse_action(of_type(OptionalClassToken)) & -# pp.Opt(html_insert)("html_insert").add_parse_action(of_type(OptionalInsertToken)) & -# pp.Opt(var)("var").add_parse_action(of_type(OptionalVarToken)) -# )("optional").add_parse_action(of_type(OptionalToken)) # Macro OptionalToken -optional = pp.ZeroOrMore( - class_insert("class_insert").add_parse_action(of_type(OptionalClassToken)) | - html_insert("html_insert").add_parse_action(of_type(OptionalInsertToken)) | +optional = pp.OneOrMore( + class_insert("class_insert").add_parse_action(of_type(OptionalClassToken)) ^ + html_insert("html_insert").add_parse_action(of_type(OptionalInsertToken)) ^ var("var").add_parse_action(of_type(OptionalVarToken)) )("optional").add_parse_action(of_type(OptionalToken)) # Macro OptionalToken @@ -296,7 +291,7 @@ enhanced_text = pp.ZeroOrMore( # Multiline elements se_start = (pps('<<') + structural_elements).add_parse_action(of_type(StructuralElementStartToken)) -se_end = (structural_elements + pps('>>')).add_parse_action(of_type(StructuralElementEndToken)) + optional +se_end = (structural_elements + pps('>>')).add_parse_action(of_type(StructuralElementEndToken)) + pp.Opt(optional) se = se_end | se_start # Structural element table_row = pp.OneOrMore( # pp.Regex(r'\|(\d)?')('table_colspan') + @@ -314,24 +309,30 @@ multi_line = se | table # Oneline elements one_header = ( - header_element + pp.SkipTo(pp.match_previous_literal(header_element)) + header_element + + pp.SkipTo(pp.match_previous_literal(header_element)) + + pps(pp.match_previous_literal(header_element)) + + pp.Opt(optional) ).add_parse_action(of_type(HeaderToken)) one_display = ( - display_element + pp.SkipTo(pp.match_previous_literal(display_element)) + display_element + + pp.SkipTo(pp.match_previous_literal(display_element)) + + pps(pp.match_previous_literal(display_element)) + + pp.Opt(optional) ).add_parse_action(of_type(DisplayToken)) one_olist = pp.line_start + ( pps(pp.Literal('#.')) + ( - (pp.SkipTo(optional)('text').add_parse_action(of_type(TextToken)) + optional) ^ # TODO: add reparse + (pp.SkipTo(optional)('text').add_parse_action(of_type(TextToken)) + optional) | # TODO: add reparse enhanced_text) ).add_parse_action(of_type(EtOlistToken)) one_ulist = pp.line_start + ( pps(pp.Literal('-')) + ( - (pp.SkipTo(optional)('text').add_parse_action(of_type(TextToken)) + optional) ^ # TODO: add reparse + (pp.SkipTo(optional)('text').add_parse_action(of_type(TextToken)) + optional) | # TODO: add reparse enhanced_text) ).add_parse_action(of_type(EtUlistToken)) # one_line sums up all one-line elements -one_line = (one_header | one_display | one_olist | one_ulist) + pp.Opt(optional) +one_line = (one_header | one_display | one_olist | one_ulist) # Final elements line = one_line | multi_line | enhanced_text @@ -342,8 +343,8 @@ line = one_line | multi_line | enhanced_text ############################################################################## # Composite elements -image = image_element + optional -alias = alias_element + optional +image = image_element + pp.Opt(optional) +alias = alias_element + pp.Opt(optional) # Syntax elements line_to_replace = pp.OneOrMore( @@ -363,4 +364,4 @@ if __name__ == '__main__': # pragma: no cover line.create_diagram("../../../dev_outputs/diagram_line.html") multi_line.create_diagram("../../../dev_outputs/diagram_multi_line.html") one_line.create_diagram("../../../dev_outputs/diagram_one_line.html") - enhanced_text.create_diagram("../../../dev_outputs/diagram_enchanced_text.html") + enhanced_text.create_diagram("../../../dev_outputs/diagram_enhanced_text.html") diff --git a/tests/modules/test_syntax.py b/tests/modules/test_syntax.py index a06a8c5..a1b3eff 100644 --- a/tests/modules/test_syntax.py +++ b/tests/modules/test_syntax.py @@ -53,6 +53,7 @@ list_of_token_types = { } # Dictionary of all lexical elements and a list of matching strings for the Pre_parser +# test_expression_matching expressions_to_match = { # Base elements "quotes": ["'hi, there'", "'hi, there'"], @@ -73,7 +74,16 @@ expressions_to_match = { "html_insert": ['{test=12, 22=3, "=5}', "{testing=12, 22=3, 4=5, 6='7'}"], # Optional elements - "optional": ["a=1", "a=1.33", "tr2='hu'"], + "optional": ["[a=1]", "[a=1, b=2]", + "{a=1'\";}", + "[a=1, b='2', 43, 'eee']", + "[a=1, b='2', 43, 'eee'] {azer=,;}", + "{{a=1, b='2', 43, 'eee'}}", + "{{a=1, b='2', 43, 'eee'}} {azer=,;}", + "{{a=1, b='2', 43, 'eee'}} {azer=,;} [azer=,;]", + "[a=1, b='2', 43, 'eee'] {azer=,;} {{azer=,;}}", + ], + # Pre_parser elements "image": ["@{image}{test=22}[a=12,22,c,d,ERE,r,3]", "@{image123_456}{a=12,22,c,d,ERE,r,3}", From 8d6ccd1e4403a8a84e1c4d9e7077500ab2b19d88 Mon Sep 17 00:00:00 2001 From: Maximilien Rigaut Date: Thu, 24 Mar 2022 11:50:56 +0100 Subject: [PATCH 10/13] Fixing old tests --- tests/modules/test_syntax.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/modules/test_syntax.py b/tests/modules/test_syntax.py index a1b3eff..106b07f 100644 --- a/tests/modules/test_syntax.py +++ b/tests/modules/test_syntax.py @@ -183,7 +183,7 @@ dict_advanced_syntax_input_and_expected_output = { "aze,/.", ]), ]), - ), __GL(), __XF), + ), __GL()), ("{aze,/.}[a=12]", ( sy.OptionalToken([ sy.OptionalInsertToken([ @@ -193,7 +193,7 @@ dict_advanced_syntax_input_and_expected_output = { sy.BeAssignToken([["a", 12]]), ]), ]), - ), __GL(), __XF), + ), __GL()), ("{aze!=}", ( sy.OptionalToken([ sy.OptionalInsertToken([ @@ -243,7 +243,7 @@ dict_advanced_syntax_input_and_expected_output = { "azer!=", ]), ]), - ), __GL(), __XF), + ), __GL()), ], # Enhanced Text "enhanced_text": [ @@ -389,7 +389,7 @@ dict_advanced_syntax_input_and_expected_output = { sy.OptionalInsertToken(["var='test', number=11"]), ]), ]), - ), __GL(), __XF), + ), __GL()), ("#. Text8 [class='blue', 123]{var='test', number=11}", ( sy.EtOlistToken([ sy.TextToken(["Text8"]), @@ -401,11 +401,11 @@ dict_advanced_syntax_input_and_expected_output = { sy.OptionalInsertToken(["var='test', number=11"]), ]), ]), - ), __GL(), __XF), + ), __GL()), ("! Display9 ! [class='blue', 123]{var='test', number=11}", ( sy.DisplayToken([ "!", - "Display9", + "Display9 ", sy.OptionalToken([ sy.OptionalVarToken([ sy.BeAssignToken([["class", "blue"]]), @@ -414,11 +414,11 @@ dict_advanced_syntax_input_and_expected_output = { sy.OptionalInsertToken(["var='test', number=11"]), ]), ]), - ), __GL(), __XF), # Bad optional implementation + ), __GL()), # Bad optional implementation ("## Text10 ## [class='blue', 123]{var='test', number=11}", ( sy.HeaderToken([ "##", - "Text10", + "Text10 ", sy.OptionalToken([ sy.OptionalVarToken([ sy.BeAssignToken([["class", "blue"]]), @@ -427,7 +427,7 @@ dict_advanced_syntax_input_and_expected_output = { sy.OptionalInsertToken(["var='test', number=11"]), ]), ]), - ), __GL(), __XF), # Bad optional implementation + ), __GL()), # Bad optional implementation ], # Tables @@ -472,7 +472,7 @@ dict_advanced_syntax_input_and_expected_output = { "var='test', number=11", ]), ]), - ), __GL(), __XF), # Bad optional implementation + ), __GL()), ], "table_separator": [ From 961bfafc623638c02a67a4b7256e98eb58ca63fa Mon Sep 17 00:00:00 2001 From: Maximilien Rigaut Date: Thu, 24 Mar 2022 13:35:07 +0100 Subject: [PATCH 11/13] Quickfix of reparse and ajusted coverage and tests. --- src/bootstraparse/modules/syntax.py | 12 ++++----- tests/modules/test_syntax.py | 39 ++++++++++++++++++++++++++--- 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/src/bootstraparse/modules/syntax.py b/src/bootstraparse/modules/syntax.py index 5aa1de7..a1a7684 100644 --- a/src/bootstraparse/modules/syntax.py +++ b/src/bootstraparse/modules/syntax.py @@ -27,7 +27,7 @@ class SemanticType: for elt in self.content: try: return_list.append(elt.to_markup()) - except Exception: + except Exception: # noqa E722 (This function is used for testing only) return_list.append(str(elt)) return f"<{self.label} = '{','.join(return_list)}' />" return f'<[NOC] {self.label} />' @@ -204,8 +204,8 @@ def reparse(parse_element): :param parse_element: pp object defining desired match :return: returns a function parsing given match with specified parsing element """ - def _reparse(original_string, _, __): - return parse_element.parse_string(original_string) + def _reparse(__, _, tokens): + return parse_element.parseString(tokens[0]) # TODO: fix this with a proper solution return _reparse @@ -224,7 +224,7 @@ def readable_markup(list_of_tokens): else: try: readable_list.append(token.to_markup()) - except Exception: + except Exception: # noqa E722 (This function is used for testing and readability purposes) readable_list.append(str(token)) return " ".join(readable_list) @@ -322,12 +322,12 @@ one_display = ( ).add_parse_action(of_type(DisplayToken)) one_olist = pp.line_start + ( pps(pp.Literal('#.')) + ( - (pp.SkipTo(optional)('text').add_parse_action(of_type(TextToken)) + optional) | # TODO: add reparse + (pp.SkipTo(optional)('text').add_parse_action(reparse(enhanced_text)) + optional) | enhanced_text) ).add_parse_action(of_type(EtOlistToken)) one_ulist = pp.line_start + ( pps(pp.Literal('-')) + ( - (pp.SkipTo(optional)('text').add_parse_action(of_type(TextToken)) + optional) | # TODO: add reparse + (pp.SkipTo(optional)('text').add_parse_action(reparse(enhanced_text)) + optional) | enhanced_text) ).add_parse_action(of_type(EtUlistToken)) diff --git a/tests/modules/test_syntax.py b/tests/modules/test_syntax.py index 106b07f..8e14adf 100644 --- a/tests/modules/test_syntax.py +++ b/tests/modules/test_syntax.py @@ -501,6 +501,31 @@ dict_advanced_syntax_input_and_expected_output = { ("|---|---|", (sy.TableSeparatorToken(["---", "---"]),), __GL()), ("- Text", (sy.EtUlistToken([sy.TextToken(["Text"])]),), __GL()), ("div>>", (sy.StructuralElementEndToken(["div"]),), __GL()), + ("#. Text with *em* and __underline__ and {arguments}{{super-arguments}}[a=1,'ui']", ( + sy.EtOlistToken([ + sy.TextToken(["Text with"]), + sy.EtEmToken(["*"]), + sy.TextToken(["em"]), + sy.EtEmToken(["*"]), + sy.TextToken(["and"]), + sy.EtUnderlineToken(["__"]), + sy.TextToken(["underline"]), + sy.EtUnderlineToken(["__"]), + sy.TextToken(["and"]), + sy.OptionalToken([ + sy.OptionalInsertToken([ + "arguments", + ]), + sy.OptionalClassToken([ + "super-arguments", + ]), + sy.OptionalVarToken([ + sy.BeAssignToken([["a", 1]]), + sy.BeValueToken(["ui"]), + ]), + ]), + ]), + ), __GL(),), ], } @@ -802,11 +827,19 @@ def test_reparse(original_string, reparse_with, expected_output, line_test): print(f"With expression: '{reparse_with}'") print("Defined at %(filename)s:%(lineno)d" % {'filename': __syntax_file, 'lineno': __definition_of_syntax_elements[reparse_with]}) - print(f"Expected: {expected_output}") + print(f"Expected: {expected_output} (len:{len(expected_output)})") expression = find_expression_from_str(str(reparse_with)) - reparse_action = pyparsing.Word(pyparsing.alphanums).add_parse_action(sy.reparse(expression)) + reparse_action = pyparsing.Word(pyparsing.printables+" ").add_parse_action(sy.reparse(expression)) reparse_result = reparse_action.parseString(str(original_string)) + print(f"Found: {reparse_result} (len:{len(reparse_result)}).") + assert reparse_result is not None for expected, result in zip_longest(expected_output, reparse_result): - assert expected == result + assert result == expected + + +def test_empty_token(): + """Test that empty tokens are correctly dropped.""" + + assert sy.of_type(sy.EmptySemanticType)("", [], []) is None From 00545b5183fba40ea2a9f8c4d2f8b0e0c3aca9e9 Mon Sep 17 00:00:00 2001 From: Maximilien Rigaut Date: Thu, 24 Mar 2022 14:35:45 +0100 Subject: [PATCH 12/13] Removed "-" in the specs. --- workflow-documents/specs.yaml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/workflow-documents/specs.yaml b/workflow-documents/specs.yaml index b55cc92..d85b691 100644 --- a/workflow-documents/specs.yaml +++ b/workflow-documents/specs.yaml @@ -23,7 +23,7 @@ strikethrough: underline: bpr: '__underline__' - markdown compatible: - + markdown compatible: no accepts arguments: no type: enhanced text scope: first release @@ -40,7 +40,7 @@ hyperlinks: # Pre-parsing elements aliases: bpr: '@[alias]' - markdown compatible: - + markdown compatible: no accepts arguments: yes type: pre-parsing scope: first release @@ -63,7 +63,7 @@ headers: display: bpr: '! header !' - markdown compatible: - + markdown compatible: no accepts arguments: yes accepts enhanced text: no type: one-line @@ -75,6 +75,7 @@ numbered lists: markdown compatible: no accepts arguments: no accepts enhanced text: yes + additional unsupported features: sub-lists type: one-line scope: first release @@ -91,14 +92,14 @@ bullet lists: structural elements: bpr: '<>' accepted elements: div, article, aside, section - markdown compatible: - + markdown compatible: no accepts arguments: yes type: multi-line scope: first release tables: bpr: '| table cell |' - markdown compatible: - + markdown compatible: no accepts arguments: yes accepts enhanced text: yes type: multi-line @@ -109,7 +110,7 @@ table-separator: # accepts ':' either at the start, middle or end of the line to align text for the column # putting a separator after a row will change said row to table-header # putting a separator after a column will change said column to table-header - markdown compatible: - + markdown compatible: no accepts arguments: yes accepts enhanced text: yes type: multi-line From f68e177e0266781deb34f306cfd3f8c189c4def2 Mon Sep 17 00:00:00 2001 From: Maximilien Rigaut Date: Thu, 24 Mar 2022 14:47:20 +0100 Subject: [PATCH 13/13] Bare specs of the context_mngr.py --- src/bootstraparse/modules/context_mngr.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/bootstraparse/modules/context_mngr.py b/src/bootstraparse/modules/context_mngr.py index c30b385..e3fb1ee 100644 --- a/src/bootstraparse/modules/context_mngr.py +++ b/src/bootstraparse/modules/context_mngr.py @@ -1 +1,18 @@ # Interprets where the parser is located and resolves errors +class BaseContainer: + pass + # Must define __init__, __str__, __repr__ and a print_all method + # will contain all sub-tokens and/or sub-containers + # Defines a method to get all token elements (__iter__) and a method to get a specific element (__getitem__) + # Defines a method to get the number of elements (__len__) + # Defines a method to get the name of the container (__name__) + # Defines a method to the optionals inside the container ( ? .optional()) + +# Define containers all the Enhanced text elements, divs, headers, list and any element that can be a container + + +class ContextManager: + pass +# Defines all methods to manage the context of the parser as a file is being parsed +# Builds all containers as the parser is parsing the file +