added sitecrawler
This commit is contained in:
parent
44601e6b95
commit
b9ce985747
|
|
@ -38,9 +38,11 @@ class BaseContainer:
|
|||
output = ""
|
||||
if not arbitrary_list:
|
||||
arbitrary_list = self.content
|
||||
# rich.inspect(arbitrary_list)
|
||||
for element in arbitrary_list:
|
||||
if isinstance(element, BaseContainer):
|
||||
output += element.export(exm) + " "
|
||||
# rich.inspect(output[:-1])
|
||||
return output[:-1]
|
||||
|
||||
def get_optionals(self):
|
||||
|
|
@ -229,6 +231,9 @@ class HyperLinkContainer(BaseContainer):
|
|||
self.others["url"] = self.content[0].content.url
|
||||
return super().export(exm)
|
||||
|
||||
def get_content(self, exm, arbitrary_list=None):
|
||||
return self.content[0].content.text
|
||||
|
||||
|
||||
# class IlImageContainer(BaseContainer):
|
||||
# pass
|
||||
|
|
@ -251,6 +256,9 @@ class HeaderContainer(BaseContainer):
|
|||
self.others["header_level"] = len(self.content[0].content[0])
|
||||
return super().export(exm)
|
||||
|
||||
def get_content(self, exm, arbitrary_list=None):
|
||||
return self.content[0].content[1]
|
||||
|
||||
|
||||
class DisplayContainer(BaseContainer):
|
||||
type = "structural_elements"
|
||||
|
|
|
|||
|
|
@ -6,8 +6,8 @@
|
|||
# from bootstraparse.modules.environment import Environment
|
||||
# env = Environment()
|
||||
# env.integrity_check() -> Checks if all mandatory parameters are set.
|
||||
# env["Config"] -> returns config object
|
||||
# env["context_mngr"] -> returns context manager object
|
||||
# env["config"] -> returns config object
|
||||
# env["template"] -> returns template object
|
||||
# env["export_mngr"] -> returns export manager object
|
||||
# env["global_path"] -> returns global path ???
|
||||
# env["user_path"] -> returns user path ???
|
||||
|
|
@ -29,8 +29,10 @@ class Environment:
|
|||
# mandatoryParameters
|
||||
self._mParams = {
|
||||
'config': None,
|
||||
'context_mngr': None,
|
||||
'template': None,
|
||||
'export_mngr': None,
|
||||
'origin': None,
|
||||
'destination': None,
|
||||
'global_path': None,
|
||||
'user_path': None,
|
||||
'site_path': None,
|
||||
|
|
|
|||
|
|
@ -75,7 +75,7 @@ def log_exception(exception, level="ERROR"):
|
|||
logging.__getattribute__(level)(' ' + line)
|
||||
if level in ["critical", "error"]:
|
||||
print("An unrecoverable error occurred, please check the log file for more information.")
|
||||
raise exception # Could also re-raise the exception
|
||||
raise exception # TODO: drop the last stack
|
||||
|
||||
if exception.__class__.__name__ == "ParsingError":
|
||||
logging.error(exception.__str__())
|
||||
|
|
|
|||
|
|
@ -244,13 +244,13 @@ if __name__ == '__main__': # pragma: no cover
|
|||
from bootstraparse.modules import parser
|
||||
|
||||
io_string = StringIO(
|
||||
"""- list
|
||||
- this is a *list*
|
||||
- list"""
|
||||
"""[this is a link]('http://te.st')"""
|
||||
)
|
||||
test = parser.parse_line(io_string)
|
||||
exm = ExportManager(None, None)
|
||||
cxm = context_mngr.ContextManager(test)
|
||||
cxc = ContextConverter(cxm(), exm)
|
||||
rich.inspect(cxc)
|
||||
cxc.process_pile()
|
||||
rich.inspect(cxc.pile[0])
|
||||
cxc.print_all()
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
# Usage:
|
||||
# from bootstraparse.modules.preparser import preparser
|
||||
# pp = preparser(file, enviroment)
|
||||
# pp.do_import() # imports all the modules and adds them to the file, do the same for all the files that are to be imported
|
||||
# pp.do_import() # imports all the modules and adds them to the file, do the same for all the files that are to be imported # noqa
|
||||
# pp.do_replacements() # replaces all images and shortcuts in the file
|
||||
# pp.readlines() # returns the lines of ORIGINAL file
|
||||
# pp.get_all_lines() # returns the lines of the file after replacements and imports
|
||||
|
|
@ -29,12 +29,12 @@ class PreParser:
|
|||
"""
|
||||
Takes a path and environment, executes all pre-parsing methods on the specified file.
|
||||
"""
|
||||
def __init__(self, file_path, __env, list_of_paths=None, dict_of_imports=None):
|
||||
def __init__(self, file_path, _env, list_of_paths=None, dict_of_imports=None):
|
||||
"""
|
||||
Initializes the PreParser object.
|
||||
Takes the following parameters:
|
||||
:param file_path: the path of the file to be parsed
|
||||
:param __env: the environment object
|
||||
:param _env: the environment object
|
||||
:param list_of_paths: the list of files that have been imported in this branch of the import tree
|
||||
:param dict_of_imports: Dictionary of all imports made to avoid duplicate file opening / pre-parsing
|
||||
"""
|
||||
|
|
@ -44,7 +44,7 @@ class PreParser:
|
|||
dict_of_imports = {}
|
||||
|
||||
# Set the environment & path
|
||||
self.__env = __env
|
||||
self._env = _env
|
||||
self.path = file_path
|
||||
self.name = os.path.basename(file_path)
|
||||
self.base_path = os.path.dirname(file_path)
|
||||
|
|
@ -148,7 +148,7 @@ class PreParser:
|
|||
self.local_dict_of_imports[e] = pp
|
||||
else:
|
||||
try:
|
||||
pp = PreParser(e, self.__env, self.list_of_paths.copy(), self.global_dict_of_imports)
|
||||
pp = PreParser(e, self._env, self.list_of_paths.copy(), self.global_dict_of_imports)
|
||||
self.global_dict_of_imports[e] = pp
|
||||
pp.make_import_list()
|
||||
self.local_dict_of_imports[e] = pp
|
||||
|
|
@ -191,8 +191,8 @@ class PreParser:
|
|||
if self.imports_done:
|
||||
self.file_with_all_imports.seek(0)
|
||||
error_mngr.log_message(
|
||||
level='WARNING',
|
||||
message=f'Imports were already done on {self.path}, returning as is ;'
|
||||
level='INFO',
|
||||
message=f'Imports were already done on {self.path}, returning as is;'
|
||||
f' rewound to the beginning of the file.'
|
||||
)
|
||||
return self.file_with_all_imports
|
||||
|
|
@ -240,7 +240,7 @@ class PreParser:
|
|||
"""
|
||||
Fetches an element from the config
|
||||
"""
|
||||
sub_dict = self.__env.config.loaded_conf
|
||||
sub_dict = self._env.config.loaded_conf
|
||||
validated_elements = []
|
||||
for key in list_keys:
|
||||
try:
|
||||
|
|
@ -248,7 +248,7 @@ class PreParser:
|
|||
validated_elements.append(f'[{key}]')
|
||||
except KeyError:
|
||||
error_mngr.log_exception(
|
||||
KeyError(f"Could not find key {key} in dict config{''.join(validated_elements)} in configs {'; '.join(self.__env.config.config_folders)}"),
|
||||
KeyError(f"Could not find key {key} in dict config{''.join(validated_elements)} in configs {'; '.join(self._env.config.config_folders)}"),
|
||||
level='CRITICAL'
|
||||
)
|
||||
return sub_dict
|
||||
|
|
@ -276,7 +276,7 @@ class PreParser:
|
|||
:param optionals: optional parameters along with alias
|
||||
"""
|
||||
output = self.get_element_from_config('aliases', 'shortcuts', shortcut)
|
||||
_, _, var_list, var_dict = export.split_optionals(optionals)
|
||||
_, _, var_list, var_dict = syntax.split_optionals(optionals)
|
||||
|
||||
return self.make_replacements(output, *var_list, **var_dict)
|
||||
|
||||
|
|
@ -289,9 +289,9 @@ class PreParser:
|
|||
"""
|
||||
shortcut_s = self.get_element_from_config('aliases', 'images', shortcut)
|
||||
|
||||
_, _, var_list, var_dict = export.split_optionals(optionals)
|
||||
_, _, var_list, var_dict = syntax.split_optionals(optionals)
|
||||
request = export.ExportRequest('inline_elements', 'image', optionals) # noqa : F841
|
||||
output = self.__env.export_mngr(request)
|
||||
output = self._env.export_mngr(request)
|
||||
|
||||
return output.start + self.make_replacements(shortcut_s, *var_list, **var_dict) + output.end
|
||||
|
||||
|
|
@ -300,11 +300,11 @@ class PreParser:
|
|||
Returns a string representation of the PreParser object.
|
||||
:return: a string representation of the PreParser object
|
||||
"""
|
||||
return "PreParser(path={}, name={}, base_path={}, " \
|
||||
"relative_path_resolver={}, ist_of_paths={}, " \
|
||||
return "PreParser[{}](path={}, name={}, base_path={}, " \
|
||||
"relative_path_resolver={}, list_of_paths={}, " \
|
||||
"global_dict_of_imports={}, local_dict_of_imports={}" \
|
||||
")".format(self.path, self.name, self.base_path,
|
||||
self.relative_path_resolver, self.list_of_paths,
|
||||
")".format(id(self), self.path, self.name, self.base_path,
|
||||
id(self.relative_path_resolver), [os.path.basename(elt) for elt in self.list_of_paths],
|
||||
self.global_dict_of_imports, self.local_dict_of_imports)
|
||||
|
||||
def __str__(self):
|
||||
|
|
@ -368,9 +368,9 @@ if __name__ == "__main__": # pragma: no cover
|
|||
from bootstraparse.modules import pathresolver
|
||||
site_path = pathresolver.b_path("../../example_userfiles/test.bpr")
|
||||
config_path = pathresolver.b_path("../../example_userfiles/config/")
|
||||
__env = environment.Environment()
|
||||
__env.config = config.ConfigLoader(config_path)
|
||||
__env.export_mngr = export.ExportManager('', '')
|
||||
t_pp = PreParser(site_path, __env)
|
||||
_env = environment.Environment()
|
||||
_env.config = config.ConfigLoader(config_path)
|
||||
_env.export_mngr = export.ExportManager('', '')
|
||||
t_pp = PreParser(site_path, _env)
|
||||
out = t_pp.do_replacements()
|
||||
rich.print(t_pp.get_all_lines())
|
||||
|
|
|
|||
|
|
@ -1 +1,93 @@
|
|||
# Module for file and directories repartition
|
||||
import os
|
||||
import rich
|
||||
from bootstraparse.modules import pathresolver, preparser, error_mngr, environment, export
|
||||
|
||||
|
||||
class SiteCrawler:
|
||||
def __init__(self, path, destination, _env):
|
||||
if not os.path.exists(path):
|
||||
error_mngr.log_exception(
|
||||
FileNotFoundError(f'The specified path "{path}" could not be found.'),
|
||||
level="CRITICAL"
|
||||
)
|
||||
|
||||
# saved variables
|
||||
self._env = _env
|
||||
self.initial_path = path
|
||||
self.destination_path = destination
|
||||
|
||||
# initialize variables
|
||||
self.force_rewrite = True # TODO: read from config
|
||||
self.directories = []
|
||||
self.files = []
|
||||
self.preparsers = []
|
||||
self.global_dict_of_imports = {}
|
||||
|
||||
# dictionaries
|
||||
self.authorised_extensions = [".bpr"]
|
||||
self.forbidden_folders = ["configs", "config", "templates", "template"]
|
||||
|
||||
# startup operations
|
||||
self.get_all_paths()
|
||||
self.create_all_paths()
|
||||
|
||||
def get_all_paths(self):
|
||||
for root, directory, files in os.walk(self.initial_path):
|
||||
for name in files:
|
||||
if os.path.splitext(name)[1] in self.authorised_extensions and name[0] != "_":
|
||||
self.files.append((os.path.relpath(root, self.initial_path), name))
|
||||
|
||||
for name in directory:
|
||||
if name not in self.forbidden_folders:
|
||||
self.directories.append((os.path.relpath(root, self.initial_path), name))
|
||||
# TODO: handle recursively
|
||||
|
||||
def create_all_paths(self):
|
||||
if not os.path.exists(self.destination_path):
|
||||
os.mkdir(self.destination_path)
|
||||
for root, di in self.directories:
|
||||
temp_path = os.path.join(self.destination_path, root, di)
|
||||
if not os.path.exists(temp_path):
|
||||
os.mkdir(temp_path)
|
||||
|
||||
def set_all_preparsers(self):
|
||||
for root, file in self.files:
|
||||
preparser_path = os.path.join(self.initial_path, root, file)
|
||||
pp = preparser.PreParser(preparser_path, self._env, dict_of_imports=self.global_dict_of_imports)
|
||||
pp.do_imports()
|
||||
p = self.create_file(os.path.join(self.destination_path, root, file))
|
||||
self.preparsers.append((pp, p))
|
||||
|
||||
return self.preparsers
|
||||
|
||||
def create_file(self, path):
|
||||
if os.path.exists(path):
|
||||
if not self.force_rewrite:
|
||||
error_mngr.log_exception(
|
||||
FileExistsError(f'File already exists at path "{path}".')
|
||||
)
|
||||
else:
|
||||
os.remove(path)
|
||||
open(path, "w+").close()
|
||||
return path
|
||||
|
||||
def __iter__(self):
|
||||
for pre in self.preparsers:
|
||||
yield pre
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
from bootstraparse.modules import config
|
||||
from bootstraparse.modules import pathresolver
|
||||
site_path = pathresolver.b_path("../../example_userfiles/test.bpr")
|
||||
config_path = pathresolver.b_path("../../example_userfiles/configs/")
|
||||
_env = environment.Environment()
|
||||
_env.config = config.ConfigLoader(config_path)
|
||||
_env.export_mngr = export.ExportManager('', '')
|
||||
xpath = pathresolver.b_path("../../example_userfiles")
|
||||
dpath = pathresolver.b_path("../../example_output")
|
||||
sc = SiteCrawler(xpath, dpath, _env)
|
||||
sc.set_all_preparsers()
|
||||
# for p in sc.preparsers:
|
||||
# rich.print(p, "\n")
|
||||
|
|
|
|||
Loading…
Reference in New Issue