From 129b540d36fa8a99563662691cb3730fd8c94fc5 Mon Sep 17 00:00:00 2001 From: Maximilien Rigaut Date: Wed, 15 Jun 2022 17:38:07 +0200 Subject: [PATCH 01/31] Applied an emergency test update --- .github/workflows/python-app.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index f0b2e45..52af6c2 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -5,9 +5,9 @@ name: Python application on: push: - branches: [ main, test ] + branches: [ main, release ] pull_request: - branches: [ main, test ] + branches: [ main, release ] jobs: From 19ddd401343242697c4871e77995145a73655083 Mon Sep 17 00:00:00 2001 From: Maximilien Rigaut Date: Wed, 15 Jun 2022 17:41:17 +0200 Subject: [PATCH 02/31] Another emergency push to release --- .github/workflows/deploy.yml | 31 +++++++++++++++++++++++++++++++ .github/workflows/python-app.yml | 20 -------------------- README.md | 5 ++++- 3 files changed, 35 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/deploy.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..7391ff3 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,31 @@ +# This workflow will install Python dependencies, and deploy the application to GitHub Pages. + +name: Python application + +on: + push: + branches: [ main, release ] + pull_request: + branches: [ main, release ] + +jobs: + deployment: + runs-on: ubuntu-latest + environment: production + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.10 + uses: actions/setup-python@v2 + with: + python-version: "3.10" + - name: Build the application + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -e . + python -m bootstraparse "example_userfiles" "docs" + - name: deploy + uses: JamesIves/github-pages-deploy-action@v4.3.3 + with: + branch: gh-pages # The branch the action should deploy to. + folder: docs \ No newline at end of file diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 52af6c2..1dd3356 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -40,23 +40,3 @@ jobs: - name: Test with pytest run: | pytest --cov-fail-under=100 -o xfail_strict=true --cov-report=term-missing . - deployment: - runs-on: ubuntu-latest - environment: production - steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.10 - uses: actions/setup-python@v2 - with: - python-version: "3.10" - - name: Build the application - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install -e . - python -m bootstraparse "example_userfiles" "docs" - - name: deploy - uses: JamesIves/github-pages-deploy-action@v4.3.3 - with: - branch: gh-pages # The branch the action should deploy to. - folder: docs diff --git a/README.md b/README.md index da087b2..45931c0 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,10 @@ bootstraparse aims to be customisable in that regard, but its first iteration an You can refer to the `example_userfiles` for an idea of what the parser expects you to feed it and find an example of our automated generation at https://idle-org.github.io/bootstraparse/ for this particular folder. Or you can run the program yourself for the same result! -[![Python application](https://github.com/idle-org/bootstraparse/actions/workflows/python-app.yml/badge.svg?branch=main)](https://github.com/idle-org/bootstraparse/actions/workflows/python-app.yml) +- Main + - [![Python Main](https://github.com/idle-org/bootstraparse/actions/workflows/python-app.yml/badge.svg?branch=main)](https://github.com/idle-org/bootstraparse/actions/workflows/python-app.yml) +- Releases + - [![Python Release](https://github.com/idle-org/bootstraparse/actions/workflows/python-app.yml/badge.svg?branch=release)](https://github.com/idle-org/bootstraparse/actions/workflows/python-app.yml) --- ## Roadmap From f4e16a0f3b8cac19321fe3caf16fd5e85d8a9c5d Mon Sep 17 00:00:00 2001 From: Maximilien Rigaut Date: Wed, 15 Jun 2022 18:01:45 +0200 Subject: [PATCH 03/31] Renamed actions --- .github/workflows/deploy.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 7391ff3..822ba79 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,12 +1,12 @@ # This workflow will install Python dependencies, and deploy the application to GitHub Pages. -name: Python application +name: Deployment on: push: - branches: [ main, release ] + branches: [ main ] pull_request: - branches: [ main, release ] + branches: [ main ] jobs: deployment: From 3c166d07e1abd5c3bcc0d8b645d408b140ee233f Mon Sep 17 00:00:00 2001 From: vicky Date: Thu, 9 Jun 2022 18:58:00 +0200 Subject: [PATCH 04/31] minor code refactor --- src/bootstraparse/modules/preparser.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/bootstraparse/modules/preparser.py b/src/bootstraparse/modules/preparser.py index 47392ec..4e0e85e 100644 --- a/src/bootstraparse/modules/preparser.py +++ b/src/bootstraparse/modules/preparser.py @@ -144,8 +144,7 @@ class PreParser: RecursionError(f"Error: {e} was imported earlier in {self.list_of_paths}"), level='CRITICAL' ) if e in self.global_dict_of_imports: - pp = self.global_dict_of_imports[e] - self.local_dict_of_imports[e] = pp + self.local_dict_of_imports[e] = self.global_dict_of_imports[e] else: try: pp = PreParser(e, self._env, self.list_of_paths.copy(), self.global_dict_of_imports) @@ -202,9 +201,9 @@ class PreParser: import_list = self.parse_import_list() source_lines = self.readlines() for import_path, import_line in import_list: - source_lines[import_line] = "" - temp_file.writelines(source_lines[source_line_count:import_line]) - source_line_count = import_line + source_lines[import_line] = "" # remove the line where the import was + temp_file.writelines(source_lines[source_line_count:import_line]) # copy origin to destination + source_line_count = import_line # update origin for next import import_file = self.global_dict_of_imports[import_path].export_with_imports() temp_file.writelines(import_file.readlines()) temp_file.writelines(source_lines[source_line_count:]) From ed330e66a484dfb5d26f91a9247c27fbb50ede6e Mon Sep 17 00:00:00 2001 From: Maximilien Rigaut Date: Wed, 8 Jun 2022 19:46:07 +0200 Subject: [PATCH 05/31] config: rtype and function doc --- src/bootstraparse/modules/config.py | 39 +++++++++++------------------ 1 file changed, 15 insertions(+), 24 deletions(-) diff --git a/src/bootstraparse/modules/config.py b/src/bootstraparse/modules/config.py index f377727..49ffaaa 100644 --- a/src/bootstraparse/modules/config.py +++ b/src/bootstraparse/modules/config.py @@ -17,16 +17,16 @@ class ConfigLoader: """ Reads all config files in a folder Behaves like a dictionary of all config files - return FileNotFound Error on Error - :param config_folder: The folder to read """ def __init__(self, config_folder=None, extensions=("yaml", "yml")): """ Defines the config folder and loads all configs :param config_folder: path to config file + :type config_folder: (str | list[str]) :param extensions: file extensions to load - :return: None + :type extensions: list[str] + :raise TypeError: if config_folder is not a string or a list of strings """ if config_folder is None: self.config_folders = [] @@ -54,6 +54,7 @@ class ConfigLoader: """ Add a Folder to the list of configs :param folder: path to folder + :type folder: str :return: None """ self.config_folders.append(folder) @@ -63,6 +64,7 @@ class ConfigLoader: """ Loads a config file :param filepath: path to config file + :type filepath: str :return: None """ basename = os.path.basename(filepath) @@ -81,6 +83,8 @@ class ConfigLoader: def load_from_folder(self, folder): """ Loads all configs in the config folder + :param folder: path to folder + :type folder: str :return: None """ for filename in os.listdir(folder): @@ -92,6 +96,8 @@ class ConfigLoader: """ Returns a yaml config object if in self.loaded_conf :param item: config key + :type item: str + :raise KeyError: if item is not in self.loaded_conf :return: config value """ try: @@ -102,36 +108,21 @@ class ConfigLoader: def __contains__(self, item): """ Same as a dictionary. + :param item: config key + :type item: str + :return: True if item is in self.loaded_conf """ return item in self.loaded_conf def __repr__(self): """ Returns config as string - :return: config as string + :return: the dict of configs and their values + :rtype: str """ - return self.loaded_conf.__repr__() + return str(self.loaded_conf.__repr__()) -# class UserConfig(ConfigLoader): -# """ -# Reads user config file. -# :param config_folder: path to config file -# :return: None -# """ -# def __init__(self, config_folder): -# super().__init__(config_folder) -# -# -# class GlobalConfig(ConfigLoader): -# """ -# Reads global config file. -# :param config_folder: path to config file -# :return: None -# """ -# def __init__(self, config_folder): -# super().__init__(config_folder) - if __name__ == '__main__': # pragma: no cover conf = ConfigLoader("../configs/") conf.add_folder("../../../example_userfiles/config/") From 3e1c8b8f194030a7a89aa63549fcc9f899e5a73a Mon Sep 17 00:00:00 2001 From: Maximilien Rigaut Date: Wed, 8 Jun 2022 19:46:37 +0200 Subject: [PATCH 06/31] PEP --- tests/modules/test_syntax_to_original.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/modules/test_syntax_to_original.py b/tests/modules/test_syntax_to_original.py index 490fda8..d559faa 100644 --- a/tests/modules/test_syntax_to_original.py +++ b/tests/modules/test_syntax_to_original.py @@ -1,7 +1,7 @@ import pytest -from bootstraparse.modules import syntax, error_mngr, tools, context_mngr -from bootstraparse.modules.tools import __GL, __module_path, find_variables_in_file +from bootstraparse.modules import syntax # , error_mngr, tools, context_mngr +# from bootstraparse.modules.tools import __GL, __module_path, find_variables_in_file __XF = pytest.mark.xfail From e420ff8e5c5eea10b5c9712ebe222d76a026cad6 Mon Sep 17 00:00:00 2001 From: Maximilien Rigaut Date: Wed, 8 Jun 2022 19:48:03 +0200 Subject: [PATCH 07/31] context_mngr.py : rtype and function doc (part) --- src/bootstraparse/modules/context_mngr.py | 168 ++++++++++++++++++++-- src/bootstraparse/modules/syntax.py | 2 +- 2 files changed, 159 insertions(+), 11 deletions(-) diff --git a/src/bootstraparse/modules/context_mngr.py b/src/bootstraparse/modules/context_mngr.py index a64d3fb..2cd6d5b 100644 --- a/src/bootstraparse/modules/context_mngr.py +++ b/src/bootstraparse/modules/context_mngr.py @@ -7,7 +7,6 @@ # Usage: # from bootstraparse.modules.context_mngr import ContextManager # ctx = ContextManager() -# ctx.??? # container = BaseContainer() # container[number] -> The number element in the content # "class_insert" >> container[number] -> Get an element from one of the mapped methods @@ -24,6 +23,14 @@ class BaseContainer: subtype = None def __init__(self, content=None, optionals=None, others=None): + """ + :param content: List of tokens to be added to the container + :param optionals: List of optional tokens to be added to the container (from the parser) + :param others: List of values to add to the .format() method + :type content: (list[(syntax.SemanticType | BaseContainer)]) + :type optionals: (syntax.SemanticType) + :type others: (dict[str, str]) + """ if content is None: content = [] if others is None: @@ -34,24 +41,41 @@ class BaseContainer: self.indentation_level = 0 # future: add an indentation level to every token for human readability of the final output def get_content(self, exm, arbitrary_list=None): + """ + Get the content of any BaseContainer object (default self.content). + :param exm: ExportManager to use for exporting + :param arbitrary_list: List to get the content from + :type exm: export.ExportManager + :type arbitrary_list: list[BaseContainer] + :rtype : str + """ output = "" if not arbitrary_list: arbitrary_list = self.content - # rich.inspect(arbitrary_list) for element in arbitrary_list: if isinstance(element, BaseContainer): output += element.export(exm) + " " - # rich.inspect(output[:-1]) return output[:-1] def get_optionals(self): + """ + Get the optionals of the container. + :rtype : (syntax.SemanticType | BaseContainer) + :return: The optionals of the container + """ return self.optionals def get_others(self): + """ + Get the other values of the container. + :rtype : dict[str, str] + :return: The other values of the container + """ return self.others def export(self, exm): """ + Export the container to a string. :type exm: export.ExportManager :rtype : str """ @@ -62,15 +86,27 @@ class BaseContainer: return output def add(self, other): + """ + Adds a list of elements to the container. + :param other: List of elements to add + :type other: syntax.SemanticType + :return: None + """ self.content.append(other) def class_name(self): + """ + A string representation of the class name. + :rtype : str + """ return self.__class__.__name__ def validate(self, other): """ Takes a list as input and checks if every element is in map. - :return: Bool + :param other: List of elements to check + :type other: list[(syntax.SemanticType | BaseContainer)] + :return: True if every element is in map, False otherwise """ for o in other: if o not in self.map: @@ -78,30 +114,68 @@ class BaseContainer: return True def debug_map(self): + """ + Prints the map of the container. + :return: None + """ print(f'Debug for {self.class_name()} <{id(self)}>') for k, v in self.map.items(): print(f'{k} = {v}') def __len__(self): + """ + :rtype : int + """ return len(self.content) def __iter__(self): + """ + :rtype : iter[syntax.SemanticType] + """ for content in self.content: yield content def __getitem__(self, item): + """ + :param item: Index of the element to get + :type item: int + :rtype : (syntax.SemanticType | BaseContainer) + """ return self.content[item] def __setitem__(self, key, value): + """ + :param key: Index of the element to set + :param value: Value to set + :type key: int + :type value: (syntax.SemanticType | BaseContainer) + """ self.content[key] = value def __getslice__(self, start, end): + """ + :param start: Start index of the slice + :param end: End index of the slice + :type start: int + :type end: int + :rtype : list[(syntax.SemanticType | BaseContainer)] + """ return self.content[start:end] def __rshift__(self, other): + """ + :param other: The name of the map element to get + :type other: str + :rtype : (syntax.SemanticType | BaseContainer) + """ return self.map[other]() def __eq__(self, other): + """ + :param other: The other object to compare to + :type other: BaseContainer | list[(syntax.SemanticType | BaseContainer)] | syntax.SemanticType + :rtype : bool + """ if not hasattr(other, "__len__"): return False if len(self) != len(other): @@ -123,13 +197,23 @@ class BaseContainer: return True def __invert__(self): + """ + Returns the inverted optionals of the container. + :rtype : BaseContainer + """ return self.get_optionals() def __str__(self): + """ + :rtype : str + """ return '{} ({})'.format(", ".join(map(str, self.content)), ", ".join(map(str, self.get_optionals())) if self.get_optionals() else "") def __repr__(self): + """ + :rtype : str + """ representation = {} for content in self.content: if type(content) in representation: @@ -146,6 +230,9 @@ class BaseContainer: ) def print_all(self, ident=''): + """ + Prints the container and all its content. + """ print(ident + self.class_name()) for e in self: if isinstance(e, BaseContainer): @@ -154,8 +241,12 @@ class BaseContainer: print(ident + "> " + str(e)) print("") - def to_container(self, filter_func=None): # idea: additional parameters for encapsulation - return self + def to_container(self, filter_func=None): # noqa : C901 + """ + Returns a container with the same content as the current container. + Used for compatibility with the syntax for encapsulation. + """ + return self # idea: additional parameters for encapsulation # Define containers all the Enhanced text elements, divs, headers, list and any element that can be a container @@ -194,7 +285,7 @@ class EtCustomSpanContainer(BaseContainer): type = "inline_elements" def export(self, exm): - self.subtype = "custom_" + self.content[0].content[0] + self.subtype = "custom_" + self.content[0].content[0] # noqa F821 (self.content[0] is a token, by definition return super().export(exm) @@ -253,7 +344,7 @@ class HeaderContainer(BaseContainer): subtype = "header" def export(self, exm): - self.others = {} + self.others = {} # noqa F821 self.others["header_level"] = len(self.content[0].content[0]) return super().export(exm) @@ -349,6 +440,10 @@ class ContextManager: ---------- parsed_list : list[syntax.SemanticType] List of parsed tokens output by our parser. + name : str + Name of the file being parsed. + ident : int + Number of spaces to indent the output. """ self.output = [] self.parsed_list = parsed_list @@ -423,11 +518,30 @@ class ContextManager: # print(" <<< Pile:", self.pile) def _add_matched(self, label, index): + """ + Method to add a matched element to the dictionary of matched elements. + Parameters + ---------- + label : str + Label of the matched element. + index : int + Index of the matched element in the pile. + """ if label not in self.matched_elements: self.matched_elements[label] = [] self.matched_elements[label] += [index] def _get_matched(self, label): + """ + Method to get the index of the last matched element of a given label. + Parameters + ---------- + label : str + Label of the matched element. + Returns + ------- + int : Index of the last matched element of the given label. + """ return self.matched_elements[label].pop() def __call__(self): @@ -438,6 +552,10 @@ class ContextManager: ------- list[BaseContainer] Returns the pile entirely processed as a list of containers. + Raise + ----- + MismatchedContainerError + Raised when a container is not matched to its corresponding token. """ # self.pile = self.parsed_list.copy() if self.contextualised: @@ -497,6 +615,10 @@ class ContextManager: return self.finalize_pile() def __iter__(self): + """ + Iterator over the pile. + :rtype: BaseContainer + """ for e in self.pile: if e: yield e @@ -505,6 +627,7 @@ class ContextManager: """ Function for cleaning up of the pile after full contextualisation. Removes Nones and checks for any errors or illogical containers. + :return: list[BaseContainer] """ final_pile = [] for p in self.pile: @@ -532,6 +655,15 @@ class ContextManager: def lookahead(self, token, index): """ Iterates the pile beginning from index and looks for all tokens matching labels with token. + Parameters + ---------- + token : syntax.BaseContainer + Token to look for. + index : int + Index of the token in the pile. + Returns + ------- + (int, int) : (number of tokens matched, number of linebreaks) """ range_to_encapsulate = 0 line_skipped = 0 @@ -557,18 +689,31 @@ class ContextManager: return range_to_encapsulate, line_skipped def recontext(self, token): + """ + Function to recontextualise the content of a token. + Parameters + ---------- + token : syntax.SemanticType + Token to recontextualise. + """ token.content = ContextManager(token.content, name=self.name)() def get_last_container_in_pile(self, index): """ Returns the last element in the pile if it is a Container, raises an error otherwise. - Return - ------ + Parameters + ---------- + index : int + Index of the element to check. + + Returns + ------- BaseContainer Raises ------ error_mngr.LonelyOptionalError + Raised when an element is found in the pile that is not encapsulated in a container. """ i = index-1 # skip last token as it is self while i >= 0: @@ -581,6 +726,9 @@ class ContextManager: log_exception(LonelyOptionalError(self.pile[index], None), level="CRITICAL") def print_all(self): + """ + Prints the pile. + """ for e in self: if isinstance(e, BaseContainer): e.print_all() diff --git a/src/bootstraparse/modules/syntax.py b/src/bootstraparse/modules/syntax.py index 40654d8..af7ba63 100644 --- a/src/bootstraparse/modules/syntax.py +++ b/src/bootstraparse/modules/syntax.py @@ -102,7 +102,7 @@ class SemanticType: raise MismatchedContainerError(self) -class TokensToMatch: +class TokensToMatch(SemanticType): """Class used for verification in the context manager. Classes inheriting this should be added to matched_elements.""" pass From 79d22b18ad8933309bc19e45cf950b53e7302205 Mon Sep 17 00:00:00 2001 From: Vicky Date: Wed, 8 Jun 2022 21:33:41 +0200 Subject: [PATCH 08/31] Update index.bpr Corrected [] and {} --- example_userfiles/index.bpr | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/example_userfiles/index.bpr b/example_userfiles/index.bpr index b2acd7a..a96516a 100644 --- a/example_userfiles/index.bpr +++ b/example_userfiles/index.bpr @@ -3,27 +3,27 @@ Structural Elements: <
>[class='blue', 123]{var='test', number=11}{{cinsertion}} +div>>{class='blue', 123}{{cinsertion}}[var='test', number=11] <
>[class='blue', 123]{var='test', number=11}{{cinsertion}} +article>>{class='blue', 123}{{cinsertion}}[var='test', number=11] <