added sitecrawler

This commit is contained in:
vicky 2022-06-06 23:15:38 +02:00
parent 44601e6b95
commit b9ce985747
6 changed files with 129 additions and 27 deletions

View File

@ -38,9 +38,11 @@ class BaseContainer:
output = ""
if not arbitrary_list:
arbitrary_list = self.content
# rich.inspect(arbitrary_list)
for element in arbitrary_list:
if isinstance(element, BaseContainer):
output += element.export(exm) + " "
# rich.inspect(output[:-1])
return output[:-1]
def get_optionals(self):
@ -229,6 +231,9 @@ class HyperLinkContainer(BaseContainer):
self.others["url"] = self.content[0].content.url
return super().export(exm)
def get_content(self, exm, arbitrary_list=None):
return self.content[0].content.text
# class IlImageContainer(BaseContainer):
# pass
@ -251,6 +256,9 @@ class HeaderContainer(BaseContainer):
self.others["header_level"] = len(self.content[0].content[0])
return super().export(exm)
def get_content(self, exm, arbitrary_list=None):
return self.content[0].content[1]
class DisplayContainer(BaseContainer):
type = "structural_elements"

View File

@ -6,8 +6,8 @@
# from bootstraparse.modules.environment import Environment
# env = Environment()
# env.integrity_check() -> Checks if all mandatory parameters are set.
# env["Config"] -> returns config object
# env["context_mngr"] -> returns context manager object
# env["config"] -> returns config object
# env["template"] -> returns template object
# env["export_mngr"] -> returns export manager object
# env["global_path"] -> returns global path ???
# env["user_path"] -> returns user path ???
@ -29,8 +29,10 @@ class Environment:
# mandatoryParameters
self._mParams = {
'config': None,
'context_mngr': None,
'template': None,
'export_mngr': None,
'origin': None,
'destination': None,
'global_path': None,
'user_path': None,
'site_path': None,

View File

@ -75,7 +75,7 @@ def log_exception(exception, level="ERROR"):
logging.__getattribute__(level)(' ' + line)
if level in ["critical", "error"]:
print("An unrecoverable error occurred, please check the log file for more information.")
raise exception # Could also re-raise the exception
raise exception # TODO: drop the last stack
if exception.__class__.__name__ == "ParsingError":
logging.error(exception.__str__())

View File

@ -244,13 +244,13 @@ if __name__ == '__main__': # pragma: no cover
from bootstraparse.modules import parser
io_string = StringIO(
"""- list
- this is a *list*
- list"""
"""[this is a link]('http://te.st')"""
)
test = parser.parse_line(io_string)
exm = ExportManager(None, None)
cxm = context_mngr.ContextManager(test)
cxc = ContextConverter(cxm(), exm)
rich.inspect(cxc)
cxc.process_pile()
rich.inspect(cxc.pile[0])
cxc.print_all()

View File

@ -2,7 +2,7 @@
# Usage:
# from bootstraparse.modules.preparser import preparser
# pp = preparser(file, enviroment)
# pp.do_import() # imports all the modules and adds them to the file, do the same for all the files that are to be imported
# pp.do_import() # imports all the modules and adds them to the file, do the same for all the files that are to be imported # noqa
# pp.do_replacements() # replaces all images and shortcuts in the file
# pp.readlines() # returns the lines of ORIGINAL file
# pp.get_all_lines() # returns the lines of the file after replacements and imports
@ -29,12 +29,12 @@ class PreParser:
"""
Takes a path and environment, executes all pre-parsing methods on the specified file.
"""
def __init__(self, file_path, __env, list_of_paths=None, dict_of_imports=None):
def __init__(self, file_path, _env, list_of_paths=None, dict_of_imports=None):
"""
Initializes the PreParser object.
Takes the following parameters:
:param file_path: the path of the file to be parsed
:param __env: the environment object
:param _env: the environment object
:param list_of_paths: the list of files that have been imported in this branch of the import tree
:param dict_of_imports: Dictionary of all imports made to avoid duplicate file opening / pre-parsing
"""
@ -44,7 +44,7 @@ class PreParser:
dict_of_imports = {}
# Set the environment & path
self.__env = __env
self._env = _env
self.path = file_path
self.name = os.path.basename(file_path)
self.base_path = os.path.dirname(file_path)
@ -148,7 +148,7 @@ class PreParser:
self.local_dict_of_imports[e] = pp
else:
try:
pp = PreParser(e, self.__env, self.list_of_paths.copy(), self.global_dict_of_imports)
pp = PreParser(e, self._env, self.list_of_paths.copy(), self.global_dict_of_imports)
self.global_dict_of_imports[e] = pp
pp.make_import_list()
self.local_dict_of_imports[e] = pp
@ -191,8 +191,8 @@ class PreParser:
if self.imports_done:
self.file_with_all_imports.seek(0)
error_mngr.log_message(
level='WARNING',
message=f'Imports were already done on {self.path}, returning as is ;'
level='INFO',
message=f'Imports were already done on {self.path}, returning as is;'
f' rewound to the beginning of the file.'
)
return self.file_with_all_imports
@ -240,7 +240,7 @@ class PreParser:
"""
Fetches an element from the config
"""
sub_dict = self.__env.config.loaded_conf
sub_dict = self._env.config.loaded_conf
validated_elements = []
for key in list_keys:
try:
@ -248,7 +248,7 @@ class PreParser:
validated_elements.append(f'[{key}]')
except KeyError:
error_mngr.log_exception(
KeyError(f"Could not find key {key} in dict config{''.join(validated_elements)} in configs {'; '.join(self.__env.config.config_folders)}"),
KeyError(f"Could not find key {key} in dict config{''.join(validated_elements)} in configs {'; '.join(self._env.config.config_folders)}"),
level='CRITICAL'
)
return sub_dict
@ -276,7 +276,7 @@ class PreParser:
:param optionals: optional parameters along with alias
"""
output = self.get_element_from_config('aliases', 'shortcuts', shortcut)
_, _, var_list, var_dict = export.split_optionals(optionals)
_, _, var_list, var_dict = syntax.split_optionals(optionals)
return self.make_replacements(output, *var_list, **var_dict)
@ -289,9 +289,9 @@ class PreParser:
"""
shortcut_s = self.get_element_from_config('aliases', 'images', shortcut)
_, _, var_list, var_dict = export.split_optionals(optionals)
_, _, var_list, var_dict = syntax.split_optionals(optionals)
request = export.ExportRequest('inline_elements', 'image', optionals) # noqa : F841
output = self.__env.export_mngr(request)
output = self._env.export_mngr(request)
return output.start + self.make_replacements(shortcut_s, *var_list, **var_dict) + output.end
@ -300,11 +300,11 @@ class PreParser:
Returns a string representation of the PreParser object.
:return: a string representation of the PreParser object
"""
return "PreParser(path={}, name={}, base_path={}, " \
"relative_path_resolver={}, ist_of_paths={}, " \
return "PreParser[{}](path={}, name={}, base_path={}, " \
"relative_path_resolver={}, list_of_paths={}, " \
"global_dict_of_imports={}, local_dict_of_imports={}" \
")".format(self.path, self.name, self.base_path,
self.relative_path_resolver, self.list_of_paths,
")".format(id(self), self.path, self.name, self.base_path,
id(self.relative_path_resolver), [os.path.basename(elt) for elt in self.list_of_paths],
self.global_dict_of_imports, self.local_dict_of_imports)
def __str__(self):
@ -368,9 +368,9 @@ if __name__ == "__main__": # pragma: no cover
from bootstraparse.modules import pathresolver
site_path = pathresolver.b_path("../../example_userfiles/test.bpr")
config_path = pathresolver.b_path("../../example_userfiles/config/")
__env = environment.Environment()
__env.config = config.ConfigLoader(config_path)
__env.export_mngr = export.ExportManager('', '')
t_pp = PreParser(site_path, __env)
_env = environment.Environment()
_env.config = config.ConfigLoader(config_path)
_env.export_mngr = export.ExportManager('', '')
t_pp = PreParser(site_path, _env)
out = t_pp.do_replacements()
rich.print(t_pp.get_all_lines())

View File

@ -1 +1,93 @@
# Module for file and directories repartition
import os
import rich
from bootstraparse.modules import pathresolver, preparser, error_mngr, environment, export
class SiteCrawler:
def __init__(self, path, destination, _env):
if not os.path.exists(path):
error_mngr.log_exception(
FileNotFoundError(f'The specified path "{path}" could not be found.'),
level="CRITICAL"
)
# saved variables
self._env = _env
self.initial_path = path
self.destination_path = destination
# initialize variables
self.force_rewrite = True # TODO: read from config
self.directories = []
self.files = []
self.preparsers = []
self.global_dict_of_imports = {}
# dictionaries
self.authorised_extensions = [".bpr"]
self.forbidden_folders = ["configs", "config", "templates", "template"]
# startup operations
self.get_all_paths()
self.create_all_paths()
def get_all_paths(self):
for root, directory, files in os.walk(self.initial_path):
for name in files:
if os.path.splitext(name)[1] in self.authorised_extensions and name[0] != "_":
self.files.append((os.path.relpath(root, self.initial_path), name))
for name in directory:
if name not in self.forbidden_folders:
self.directories.append((os.path.relpath(root, self.initial_path), name))
# TODO: handle recursively
def create_all_paths(self):
if not os.path.exists(self.destination_path):
os.mkdir(self.destination_path)
for root, di in self.directories:
temp_path = os.path.join(self.destination_path, root, di)
if not os.path.exists(temp_path):
os.mkdir(temp_path)
def set_all_preparsers(self):
for root, file in self.files:
preparser_path = os.path.join(self.initial_path, root, file)
pp = preparser.PreParser(preparser_path, self._env, dict_of_imports=self.global_dict_of_imports)
pp.do_imports()
p = self.create_file(os.path.join(self.destination_path, root, file))
self.preparsers.append((pp, p))
return self.preparsers
def create_file(self, path):
if os.path.exists(path):
if not self.force_rewrite:
error_mngr.log_exception(
FileExistsError(f'File already exists at path "{path}".')
)
else:
os.remove(path)
open(path, "w+").close()
return path
def __iter__(self):
for pre in self.preparsers:
yield pre
if __name__ == "__main__":
from bootstraparse.modules import config
from bootstraparse.modules import pathresolver
site_path = pathresolver.b_path("../../example_userfiles/test.bpr")
config_path = pathresolver.b_path("../../example_userfiles/configs/")
_env = environment.Environment()
_env.config = config.ConfigLoader(config_path)
_env.export_mngr = export.ExportManager('', '')
xpath = pathresolver.b_path("../../example_userfiles")
dpath = pathresolver.b_path("../../example_output")
sc = SiteCrawler(xpath, dpath, _env)
sc.set_all_preparsers()
# for p in sc.preparsers:
# rich.print(p, "\n")