From 5c104b155f074a143f17c837817e79d1099ac2d7 Mon Sep 17 00:00:00 2001 From: Maximilien Rigaut Date: Wed, 15 Jun 2022 20:53:45 +0200 Subject: [PATCH 1/2] Hotfix:deploy --- .gitignore | 3 ++- README.md | 7 ++++--- src/bootstraparse/__main__.py | 3 ++- src/bootstraparse/configs/parser_config.yml | 2 +- src/bootstraparse/modules/config.py | 11 ++++++++--- tests/modules/test_config.py | 12 ++++++++---- 6 files changed, 25 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index fbe5e12..1f7e2ed 100644 --- a/.gitignore +++ b/.gitignore @@ -156,4 +156,5 @@ cython_debug/ example_userfiles/output/ example_userfiles/output/* dev_outputs/* -/example_output/* +example_output/* +dics/* diff --git a/README.md b/README.md index 86a2570..3914c63 100644 --- a/README.md +++ b/README.md @@ -6,10 +6,11 @@ bootstraparse aims to be customisable in that regard, but its first iteration an You can refer to the `example_userfiles` for an idea of what the parser expects you to feed it and find an example of our automated generation at https://idle-org.github.io/bootstraparse/ for this particular folder. Or you can run the program yourself for the same result! - Main - - [![Python Main](https://github.com/idle-org/bootstraparse/actions/workflows/python-app.yml/badge.svg?branch=main)](https://github.com/idle-org/bootstraparse/actions/workflows/python-app.yml) + - [![Python Tests and Lint](https://github.com/idle-org/bootstraparse/actions/workflows/python-tests.yml/badge.svg?branch=main)](https://github.com/idle-org/bootstraparse/actions/workflows/python-tests.yml) - Releases - - [![Python Release](https://github.com/idle-org/bootstraparse/actions/workflows/python-app.yml/badge.svg?branch=release)](https://github.com/idle-org/bootstraparse/actions/workflows/python-app.yml) - + - [![Python Tests and Lint](https://github.com/idle-org/bootstraparse/actions/workflows/python-tests.yml/badge.svg?branch=develop)](https://github.com/idle-org/bootstraparse/actions/workflows/python-tests.yml) +- Deploy + - [![Python Deployment](https://github.com/idle-org/bootstraparse/actions/workflows/python-deploy.yml/badge.svg)](https://github.com/idle-org/bootstraparse/actions/workflows/python-deploy.yml) --- ## Release Notes ### V1.0.1 diff --git a/src/bootstraparse/__main__.py b/src/bootstraparse/__main__.py index 1d7f38d..d4028d3 100755 --- a/src/bootstraparse/__main__.py +++ b/src/bootstraparse/__main__.py @@ -2,7 +2,7 @@ # Main program, use this to start parsing -from bootstraparse.modules import sitecreator +from bootstraparse.modules import sitecreator, error_mngr import argparse import sys @@ -21,5 +21,6 @@ def parse(_args): if __name__ == "__main__": # pragma: no cover args = parse(sys.argv[1:]) + error_mngr.init_logging(filename=None, loglevel="DEBUG", filemode='w', handler=None) if sitecreator.create_website(args.origin, args.destination) == 0: print("Bootstraparse run successful!") diff --git a/src/bootstraparse/configs/parser_config.yml b/src/bootstraparse/configs/parser_config.yml index 0dcc7ae..699352f 100644 --- a/src/bootstraparse/configs/parser_config.yml +++ b/src/bootstraparse/configs/parser_config.yml @@ -21,4 +21,4 @@ export: intermediate_files: true type: "html" force_rewrite: true - copy_unparsable_files: "copy" # "copy" or "ignore" + copy_unparsable_files: copy diff --git a/src/bootstraparse/modules/config.py b/src/bootstraparse/modules/config.py index 49ffaaa..b5eef6d 100644 --- a/src/bootstraparse/modules/config.py +++ b/src/bootstraparse/modules/config.py @@ -74,11 +74,16 @@ class ConfigLoader: if name not in self.loaded_conf: self.loaded_conf[name] = yaml.safe_load(f) else: - self.loaded_conf[name].update(yaml.safe_load(f)) - error_mngr.log_message(f"Warning: {name} is already in {self.loaded_conf}", level='INFO') + new_config = yaml.safe_load(f) + for key in new_config: + if key in self.loaded_conf[name]: + self.loaded_conf[name][key].update(new_config[key]) + else: + self.loaded_conf[name][key] = new_config[key] + error_mngr.log_message(f"Warning: {name} is already in {self.loaded_conf}", level='CRITICAL') except BaseException as e: error_mngr.log_message(f'Error parsing in file {basename} at {filepath}.', level='CRITICAL') - error_mngr.log_exception(str(e), level='CRITICAL') + error_mngr.log_exception(e, level='CRITICAL') def load_from_folder(self, folder): """ diff --git a/tests/modules/test_config.py b/tests/modules/test_config.py index 343f29e..eba2e08 100644 --- a/tests/modules/test_config.py +++ b/tests/modules/test_config.py @@ -1,5 +1,7 @@ import os +from yaml.scanner import ScannerError + import bootstraparse.modules.config as config import pytest import tempfile @@ -10,7 +12,8 @@ configs = { "app/configs/parser_config.yaml": '{"parser": {"name": "test_parser", "type": "test_parser"}}', "app/configs/aliases.yaml": '{"parser": {"name": "test_parser", "type": "test_parser"}}', "user/config/aliases.yaml": '{"aliases": {"test_alias": "test_parser"}}', - "user/config/glossary.yaml": '{"glossary": {"test_glossary": "test_parser"}}', + "app/configs/glossary.yaml": '{"glossary": {"test_glossary": "test_parser"}}', + "user/config/glossary.yaml": '{"glossary": {"test_glossary": "second_parser"}}', "user/config/custom_template.yaml": '{"custom_template": {"test_template": "test_parser"}}', } bad_configs = { @@ -36,7 +39,9 @@ def make_false_configs(): def test_config_load(): usr_c = config.ConfigLoader(user_conf) assert usr_c["aliases"] == {"aliases": {"test_alias": "test_parser"}} - assert usr_c["glossary"] == {"glossary": {"test_glossary": "test_parser"}} + assert usr_c["glossary"] == {"glossary": {"test_glossary": "second_parser"}} + assert usr_c["glossary"]["glossary"] == {"test_glossary": "second_parser"} + assert usr_c["glossary"]["glossary"]["test_glossary"] == "second_parser" assert usr_c["aliases"]["aliases"] == {"test_alias": "test_parser"} assert usr_c["aliases"]["aliases"]["test_alias"] == "test_parser" with pytest.raises(KeyError): @@ -54,7 +59,6 @@ def test_add_to_config(): assert from_list["aliases"] == {"aliases": {"test_alias": "test_parser"}, "parser": {"name": "test_parser", "type": "test_parser"}} assert from_list["parser_config"] == {"parser": {"name": "test_parser", "type": "test_parser"}} - with pytest.raises(KeyError): # Not FileNotFoundError assert from_empty["aliases"] from_empty.add_folder(user_conf) @@ -64,7 +68,7 @@ def test_add_to_config(): assert from_empty["aliases"] == {"aliases": {"test_alias": "test_parser"}, "parser": {"name": "test_parser", "type": "test_parser"}} - with pytest.raises(TypeError): + with pytest.raises(ScannerError): from_empty.load_from_file(os.path.join(bad_conf, "bad_config.yaml")) From 6e4babfa142cc2eda292aec9886fa5e75ccfd24c Mon Sep 17 00:00:00 2001 From: Maximilien Rigaut Date: Wed, 15 Jun 2022 21:27:41 +0200 Subject: [PATCH 2/2] hotfix2:deploy --- .github/workflows/python-deploy.yml | 4 +- .github/workflows/python-tests.yml | 4 +- .gitignore | 2 +- src/bootstraparse/modules/context_mngr.py | 104 ++++++++-------------- src/bootstraparse/modules/export.py | 30 +++---- src/bootstraparse/modules/parser.py | 8 +- src/bootstraparse/modules/syntax.py | 20 ++--- 7 files changed, 64 insertions(+), 108 deletions(-) diff --git a/.github/workflows/python-deploy.yml b/.github/workflows/python-deploy.yml index 6160295..a27e88e 100644 --- a/.github/workflows/python-deploy.yml +++ b/.github/workflows/python-deploy.yml @@ -5,9 +5,9 @@ name: Python Deployment on: push: - branches: [ main ] + branches: [ main, test*] pull_request: - branches: [ main ] + branches: [ main, test* ] jobs: diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index d962337..2a60a57 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -5,9 +5,9 @@ name: Python Tests and Lint on: push: - branches: [ main, release, releases/*] + branches: [ main, release, releases/*, test*] pull_request: - branches: [ main, release, releases/* ] + branches: [ main, release, releases/*, test* ] jobs: diff --git a/.gitignore b/.gitignore index 1f7e2ed..e60c03b 100644 --- a/.gitignore +++ b/.gitignore @@ -157,4 +157,4 @@ example_userfiles/output/ example_userfiles/output/* dev_outputs/* example_output/* -dics/* +docs/ diff --git a/src/bootstraparse/modules/context_mngr.py b/src/bootstraparse/modules/context_mngr.py index 4a0e29b..33b126b 100644 --- a/src/bootstraparse/modules/context_mngr.py +++ b/src/bootstraparse/modules/context_mngr.py @@ -436,14 +436,12 @@ class ContextManager: def __init__(self, parsed_list, name=None, ident=0): """ Takes a list of parsed tokens. - Parameters - ---------- - parsed_list : list[syntax.SemanticType] - List of parsed tokens output by our parser. - name : str - Name of the file being parsed. - ident : int - Number of spaces to indent the output. + :type parsed_list : list[syntax.SemanticType] + :param parsed_list: List of parsed tokens output by our parser. + :type name: str + :param name: Name of the file being parsed. + :type ident: int + :param ident: Number of spaces to indent the output. """ self.output = [] self.parsed_list = parsed_list @@ -464,15 +462,12 @@ class ContextManager: """ Method to encapsulate a number of tokens together as a final container object and replace all elements in the pile with None, except the first one which becomes the container. - Parameters - ---------- - start : int - Index of the pile where the encapsulation must begin. - end : int - Index of the pile where the encapsulation must end. - Returns - ------- - list[(BaseContainer|None)] + :type start : int + :param start: Index of the pile where the encapsulation must begin. + :type end: int + :param end: Index of the pile where the encapsulation must end. + :rtype: list[(BaseContainer|None)] + :return: List of containers and None objects. """ # Enabling this will allow us to debug the pile # print(f"Step : [{end}] - encapsulation:") @@ -514,18 +509,16 @@ class ContextManager: container.add(self.pile[end]) self.pile[end] = None self.pile[start] = container - + return self.pile # ? I think this is the right way to return the pile # print(" <<< Pile:", self.pile) def _add_matched(self, label, index): """ Method to add a matched element to the dictionary of matched elements. - Parameters - ---------- - label : str - Label of the matched element. - index : int - Index of the matched element in the pile. + :type label : str + :param label: Label of the matched element. + :type index: int + :param index: Index of the matched element in the pile. """ if label not in self.matched_elements: self.matched_elements[label] = [] @@ -534,13 +527,10 @@ class ContextManager: def _get_matched(self, label): """ Method to get the index of the last matched element of a given label. - Parameters - ---------- - label : str - Label of the matched element. - Returns - ------- - int : Index of the last matched element of the given label. + :type label: str + :param label: Label of the matched element. + :rtype: int + :return: Index of the last matched element """ return self.matched_elements[label].pop() @@ -548,14 +538,9 @@ class ContextManager: """ Interprets the list of tokens provided and deduces context, encapsulating them to their closest neighbour and containing all tokens inbetween. - Returns - ------- - list[BaseContainer] - Returns the pile entirely processed as a list of containers. - Raise - ----- - MismatchedContainerError - Raised when a container is not matched to its corresponding token. + :rtype: list[BaseContainer] + :return: Returns the pile entirely processed as a list of containers. + :raises: MismatchedContainerError if a container is not matched to its corresponding token. """ # self.pile = self.parsed_list.copy() if self.contextualised: @@ -655,15 +640,12 @@ class ContextManager: def lookahead(self, token, index): """ Iterates the pile beginning from index and looks for all tokens matching labels with token. - Parameters - ---------- - token : syntax.BaseContainer - Token to look for. - index : int - Index of the token in the pile. - Returns - ------- - (int, int) : (number of tokens matched, number of linebreaks) + :type token : syntax.BaseContainer + :param token: Token to look for. + :type index : int + :param index: Index of the token in the pile. + :return: (number of tokens matched, number of linebreaks) + :rtype: (int, int) """ range_to_encapsulate = 0 line_skipped = 0 @@ -691,29 +673,21 @@ class ContextManager: def recontext(self, token): """ Function to recontextualise the content of a token. - Parameters - ---------- - token : syntax.SemanticType - Token to recontextualise. + :param token: Token to recontextualise. + :type token: syntax.SemanticType + :return: Token to recontextualise. + :rtype: syntax.SemanticType """ token.content = ContextManager(token.content, name=self.name)() def get_last_container_in_pile(self, index): """ Returns the last element in the pile if it is a Container, raises an error otherwise. - Parameters - ---------- - index : int - Index of the element to check. - - Returns - ------- - BaseContainer - - Raises - ------ - error_mngr.LonelyOptionalError - Raised when an element is found in the pile that is not encapsulated in a container. + :type index : int + :param index: Index of the element to check. + :return: Last element in the pile if it is a Container, raises an error otherwise. + :rtype: BaseContainer + :raise: error_mngr.LonelyOptionalError when an element is found in the pile that is not encapsulated in a container. """ i = index-1 # skip last token as it is self while i >= 0: diff --git a/src/bootstraparse/modules/export.py b/src/bootstraparse/modules/export.py index c0854a4..38f7edc 100644 --- a/src/bootstraparse/modules/export.py +++ b/src/bootstraparse/modules/export.py @@ -94,18 +94,10 @@ class ExportManager: """ Function for initializing other transform functions. - Parameters: - ---------- - export_request: ExportRequest tuples - - Returns: - ------- - start: str - Start string of the markup element - end: str - End string of the markup element - optionals: syntax.Optionals - Optionals object containing html_insert and class_insert + :param export_request: ExportRequest tuples + :type export_request: ExportRequest + :return: start, end, optionals + :rtype: str, str, str """ start, end = None, None @@ -206,14 +198,12 @@ class ContextConverter: def __init__(self, pile, exporter, destination): """ Takes a pile and the exporter object - Parameters - ---------- - pile : list[context_mngr.BaseContainer] - pile output from the context manager. - exporter : ExportManager - Our ExportManager - destination : str - Destination of the output file. + :type pile : list[context_mngr.BaseContainer] + :param pile: output from the context manager. + :type exporter : ExportManager + :param exporter: Our ExportManager + :type destination : str + :param destination: Destination of the output file. """ self.io_output = StringIO() self.pile = pile diff --git a/src/bootstraparse/modules/parser.py b/src/bootstraparse/modules/parser.py index b3f68b5..180065b 100644 --- a/src/bootstraparse/modules/parser.py +++ b/src/bootstraparse/modules/parser.py @@ -12,10 +12,10 @@ import bootstraparse.modules.syntax as syntax def parse_line(io): """ Takes an io string and returns the parsed output. - Parameters: - io (StringIO): The io string to parse. - Returns: - list[syntax.SemanticType]: The parsed output. + :param io: The io string to parse. + :type io: StringIO + :return: The parsed output. + :rtype: list[syntax.SemanticType] """ output = [] for line in io.readlines(): diff --git a/src/bootstraparse/modules/syntax.py b/src/bootstraparse/modules/syntax.py index c37233c..1d6d031 100644 --- a/src/bootstraparse/modules/syntax.py +++ b/src/bootstraparse/modules/syntax.py @@ -98,21 +98,13 @@ class SemanticType: def to_container(self, filter_func=None): """ Function called by the context manager to know how to deal with the encapsulation of the token. - - Parameters - ---------- - filter_func: (function | None) - A function that takes a token and returns a boolean. If the token is not valid, it should return False. - - Returns - ------- - SemanticType | context_mngr.BaseContainer - Self if containable, the expected replacement token if it can, raises an error otherwise. - Raises - ------ - CannotBeContainedError - Raises error if the token we were trying to encapsulate cannot be (probably mismatched). + :param filter_func: A function that will be used to filter the content of the token. + :type filter_func: (callable| None) + :return: Self if containable, the expected replacement token if it can, raises an error otherwise. + :rtype: SemanticType | context_mngr.BaseContainer + :raises CannotBeContainedError: Raises error if the token we were trying to encapsulate cannot be (probably mismatched). """ + raise MismatchedContainerError(self)