From b9ce985747a4022e5b49ccc29f57c5c483ef0c94 Mon Sep 17 00:00:00 2001 From: vicky Date: Mon, 6 Jun 2022 23:15:38 +0200 Subject: [PATCH] added sitecrawler --- src/bootstraparse/modules/context_mngr.py | 8 ++ src/bootstraparse/modules/environment.py | 8 +- src/bootstraparse/modules/error_mngr.py | 2 +- src/bootstraparse/modules/export.py | 6 +- src/bootstraparse/modules/preparser.py | 40 +++++----- src/bootstraparse/modules/sitecrawler.py | 92 +++++++++++++++++++++++ 6 files changed, 129 insertions(+), 27 deletions(-) diff --git a/src/bootstraparse/modules/context_mngr.py b/src/bootstraparse/modules/context_mngr.py index c9d1aac..5deb5d4 100644 --- a/src/bootstraparse/modules/context_mngr.py +++ b/src/bootstraparse/modules/context_mngr.py @@ -38,9 +38,11 @@ class BaseContainer: output = "" if not arbitrary_list: arbitrary_list = self.content + # rich.inspect(arbitrary_list) for element in arbitrary_list: if isinstance(element, BaseContainer): output += element.export(exm) + " " + # rich.inspect(output[:-1]) return output[:-1] def get_optionals(self): @@ -229,6 +231,9 @@ class HyperLinkContainer(BaseContainer): self.others["url"] = self.content[0].content.url return super().export(exm) + def get_content(self, exm, arbitrary_list=None): + return self.content[0].content.text + # class IlImageContainer(BaseContainer): # pass @@ -251,6 +256,9 @@ class HeaderContainer(BaseContainer): self.others["header_level"] = len(self.content[0].content[0]) return super().export(exm) + def get_content(self, exm, arbitrary_list=None): + return self.content[0].content[1] + class DisplayContainer(BaseContainer): type = "structural_elements" diff --git a/src/bootstraparse/modules/environment.py b/src/bootstraparse/modules/environment.py index 36ff7ac..dae7f84 100644 --- a/src/bootstraparse/modules/environment.py +++ b/src/bootstraparse/modules/environment.py @@ -6,8 +6,8 @@ # from bootstraparse.modules.environment import Environment # env = Environment() # env.integrity_check() -> Checks if all mandatory parameters are set. -# env["Config"] -> returns config object -# env["context_mngr"] -> returns context manager object +# env["config"] -> returns config object +# env["template"] -> returns template object # env["export_mngr"] -> returns export manager object # env["global_path"] -> returns global path ??? # env["user_path"] -> returns user path ??? @@ -29,8 +29,10 @@ class Environment: # mandatoryParameters self._mParams = { 'config': None, - 'context_mngr': None, + 'template': None, 'export_mngr': None, + 'origin': None, + 'destination': None, 'global_path': None, 'user_path': None, 'site_path': None, diff --git a/src/bootstraparse/modules/error_mngr.py b/src/bootstraparse/modules/error_mngr.py index 6f8df13..e3fc36d 100644 --- a/src/bootstraparse/modules/error_mngr.py +++ b/src/bootstraparse/modules/error_mngr.py @@ -75,7 +75,7 @@ def log_exception(exception, level="ERROR"): logging.__getattribute__(level)(' ' + line) if level in ["critical", "error"]: print("An unrecoverable error occurred, please check the log file for more information.") - raise exception # Could also re-raise the exception + raise exception # TODO: drop the last stack if exception.__class__.__name__ == "ParsingError": logging.error(exception.__str__()) diff --git a/src/bootstraparse/modules/export.py b/src/bootstraparse/modules/export.py index fb679b7..7370ec3 100644 --- a/src/bootstraparse/modules/export.py +++ b/src/bootstraparse/modules/export.py @@ -244,13 +244,13 @@ if __name__ == '__main__': # pragma: no cover from bootstraparse.modules import parser io_string = StringIO( - """- list - - this is a *list* - - list""" + """[this is a link]('http://te.st')""" ) test = parser.parse_line(io_string) exm = ExportManager(None, None) cxm = context_mngr.ContextManager(test) cxc = ContextConverter(cxm(), exm) + rich.inspect(cxc) cxc.process_pile() + rich.inspect(cxc.pile[0]) cxc.print_all() diff --git a/src/bootstraparse/modules/preparser.py b/src/bootstraparse/modules/preparser.py index b48a9cb..fe6e466 100644 --- a/src/bootstraparse/modules/preparser.py +++ b/src/bootstraparse/modules/preparser.py @@ -2,7 +2,7 @@ # Usage: # from bootstraparse.modules.preparser import preparser # pp = preparser(file, enviroment) -# pp.do_import() # imports all the modules and adds them to the file, do the same for all the files that are to be imported +# pp.do_import() # imports all the modules and adds them to the file, do the same for all the files that are to be imported # noqa # pp.do_replacements() # replaces all images and shortcuts in the file # pp.readlines() # returns the lines of ORIGINAL file # pp.get_all_lines() # returns the lines of the file after replacements and imports @@ -29,12 +29,12 @@ class PreParser: """ Takes a path and environment, executes all pre-parsing methods on the specified file. """ - def __init__(self, file_path, __env, list_of_paths=None, dict_of_imports=None): + def __init__(self, file_path, _env, list_of_paths=None, dict_of_imports=None): """ Initializes the PreParser object. Takes the following parameters: :param file_path: the path of the file to be parsed - :param __env: the environment object + :param _env: the environment object :param list_of_paths: the list of files that have been imported in this branch of the import tree :param dict_of_imports: Dictionary of all imports made to avoid duplicate file opening / pre-parsing """ @@ -44,7 +44,7 @@ class PreParser: dict_of_imports = {} # Set the environment & path - self.__env = __env + self._env = _env self.path = file_path self.name = os.path.basename(file_path) self.base_path = os.path.dirname(file_path) @@ -148,7 +148,7 @@ class PreParser: self.local_dict_of_imports[e] = pp else: try: - pp = PreParser(e, self.__env, self.list_of_paths.copy(), self.global_dict_of_imports) + pp = PreParser(e, self._env, self.list_of_paths.copy(), self.global_dict_of_imports) self.global_dict_of_imports[e] = pp pp.make_import_list() self.local_dict_of_imports[e] = pp @@ -191,8 +191,8 @@ class PreParser: if self.imports_done: self.file_with_all_imports.seek(0) error_mngr.log_message( - level='WARNING', - message=f'Imports were already done on {self.path}, returning as is ;' + level='INFO', + message=f'Imports were already done on {self.path}, returning as is;' f' rewound to the beginning of the file.' ) return self.file_with_all_imports @@ -240,7 +240,7 @@ class PreParser: """ Fetches an element from the config """ - sub_dict = self.__env.config.loaded_conf + sub_dict = self._env.config.loaded_conf validated_elements = [] for key in list_keys: try: @@ -248,7 +248,7 @@ class PreParser: validated_elements.append(f'[{key}]') except KeyError: error_mngr.log_exception( - KeyError(f"Could not find key {key} in dict config{''.join(validated_elements)} in configs {'; '.join(self.__env.config.config_folders)}"), + KeyError(f"Could not find key {key} in dict config{''.join(validated_elements)} in configs {'; '.join(self._env.config.config_folders)}"), level='CRITICAL' ) return sub_dict @@ -276,7 +276,7 @@ class PreParser: :param optionals: optional parameters along with alias """ output = self.get_element_from_config('aliases', 'shortcuts', shortcut) - _, _, var_list, var_dict = export.split_optionals(optionals) + _, _, var_list, var_dict = syntax.split_optionals(optionals) return self.make_replacements(output, *var_list, **var_dict) @@ -289,9 +289,9 @@ class PreParser: """ shortcut_s = self.get_element_from_config('aliases', 'images', shortcut) - _, _, var_list, var_dict = export.split_optionals(optionals) + _, _, var_list, var_dict = syntax.split_optionals(optionals) request = export.ExportRequest('inline_elements', 'image', optionals) # noqa : F841 - output = self.__env.export_mngr(request) + output = self._env.export_mngr(request) return output.start + self.make_replacements(shortcut_s, *var_list, **var_dict) + output.end @@ -300,11 +300,11 @@ class PreParser: Returns a string representation of the PreParser object. :return: a string representation of the PreParser object """ - return "PreParser(path={}, name={}, base_path={}, " \ - "relative_path_resolver={}, ist_of_paths={}, " \ + return "PreParser[{}](path={}, name={}, base_path={}, " \ + "relative_path_resolver={}, list_of_paths={}, " \ "global_dict_of_imports={}, local_dict_of_imports={}" \ - ")".format(self.path, self.name, self.base_path, - self.relative_path_resolver, self.list_of_paths, + ")".format(id(self), self.path, self.name, self.base_path, + id(self.relative_path_resolver), [os.path.basename(elt) for elt in self.list_of_paths], self.global_dict_of_imports, self.local_dict_of_imports) def __str__(self): @@ -368,9 +368,9 @@ if __name__ == "__main__": # pragma: no cover from bootstraparse.modules import pathresolver site_path = pathresolver.b_path("../../example_userfiles/test.bpr") config_path = pathresolver.b_path("../../example_userfiles/config/") - __env = environment.Environment() - __env.config = config.ConfigLoader(config_path) - __env.export_mngr = export.ExportManager('', '') - t_pp = PreParser(site_path, __env) + _env = environment.Environment() + _env.config = config.ConfigLoader(config_path) + _env.export_mngr = export.ExportManager('', '') + t_pp = PreParser(site_path, _env) out = t_pp.do_replacements() rich.print(t_pp.get_all_lines()) diff --git a/src/bootstraparse/modules/sitecrawler.py b/src/bootstraparse/modules/sitecrawler.py index e583313..98ba1e1 100644 --- a/src/bootstraparse/modules/sitecrawler.py +++ b/src/bootstraparse/modules/sitecrawler.py @@ -1 +1,93 @@ # Module for file and directories repartition +import os +import rich +from bootstraparse.modules import pathresolver, preparser, error_mngr, environment, export + + +class SiteCrawler: + def __init__(self, path, destination, _env): + if not os.path.exists(path): + error_mngr.log_exception( + FileNotFoundError(f'The specified path "{path}" could not be found.'), + level="CRITICAL" + ) + + # saved variables + self._env = _env + self.initial_path = path + self.destination_path = destination + + # initialize variables + self.force_rewrite = True # TODO: read from config + self.directories = [] + self.files = [] + self.preparsers = [] + self.global_dict_of_imports = {} + + # dictionaries + self.authorised_extensions = [".bpr"] + self.forbidden_folders = ["configs", "config", "templates", "template"] + + # startup operations + self.get_all_paths() + self.create_all_paths() + + def get_all_paths(self): + for root, directory, files in os.walk(self.initial_path): + for name in files: + if os.path.splitext(name)[1] in self.authorised_extensions and name[0] != "_": + self.files.append((os.path.relpath(root, self.initial_path), name)) + + for name in directory: + if name not in self.forbidden_folders: + self.directories.append((os.path.relpath(root, self.initial_path), name)) + # TODO: handle recursively + + def create_all_paths(self): + if not os.path.exists(self.destination_path): + os.mkdir(self.destination_path) + for root, di in self.directories: + temp_path = os.path.join(self.destination_path, root, di) + if not os.path.exists(temp_path): + os.mkdir(temp_path) + + def set_all_preparsers(self): + for root, file in self.files: + preparser_path = os.path.join(self.initial_path, root, file) + pp = preparser.PreParser(preparser_path, self._env, dict_of_imports=self.global_dict_of_imports) + pp.do_imports() + p = self.create_file(os.path.join(self.destination_path, root, file)) + self.preparsers.append((pp, p)) + + return self.preparsers + + def create_file(self, path): + if os.path.exists(path): + if not self.force_rewrite: + error_mngr.log_exception( + FileExistsError(f'File already exists at path "{path}".') + ) + else: + os.remove(path) + open(path, "w+").close() + return path + + def __iter__(self): + for pre in self.preparsers: + yield pre + + +if __name__ == "__main__": + from bootstraparse.modules import config + from bootstraparse.modules import pathresolver + site_path = pathresolver.b_path("../../example_userfiles/test.bpr") + config_path = pathresolver.b_path("../../example_userfiles/configs/") + _env = environment.Environment() + _env.config = config.ConfigLoader(config_path) + _env.export_mngr = export.ExportManager('', '') + xpath = pathresolver.b_path("../../example_userfiles") + dpath = pathresolver.b_path("../../example_output") + sc = SiteCrawler(xpath, dpath, _env) + sc.set_all_preparsers() + # for p in sc.preparsers: + # rich.print(p, "\n")