sitecrawler.py : Doc and rtypes
This commit is contained in:
parent
00786d8480
commit
dfc8a7b2d2
|
|
@ -4,7 +4,22 @@ from bootstraparse.modules import pathresolver, preparser, error_mngr, environme
|
||||||
|
|
||||||
|
|
||||||
class SiteCrawler:
|
class SiteCrawler:
|
||||||
|
"""
|
||||||
|
A sitecrawler is a generator that yields a tuple of the form (PreParser, file)
|
||||||
|
Its goal is to create a list of preparsers and files to be parsed based on
|
||||||
|
the files and directories in the initial path.
|
||||||
|
This generator is to be used in a for loop to parse all the files and produce
|
||||||
|
the final website.
|
||||||
|
"""
|
||||||
def __init__(self, path, destination, _env):
|
def __init__(self, path, destination, _env):
|
||||||
|
"""
|
||||||
|
:param path: The path to the directory to be crawled
|
||||||
|
:param destination: The path to the directory where the website will be created
|
||||||
|
:param _env: The environment object
|
||||||
|
:type path: str
|
||||||
|
:type destination: str
|
||||||
|
:type _env: environment.Environment
|
||||||
|
"""
|
||||||
if not os.path.exists(path):
|
if not os.path.exists(path):
|
||||||
error_mngr.log_exception(
|
error_mngr.log_exception(
|
||||||
FileNotFoundError(f'The specified path "{path}" could not be found.'),
|
FileNotFoundError(f'The specified path "{path}" could not be found.'),
|
||||||
|
|
@ -32,9 +47,23 @@ class SiteCrawler:
|
||||||
self.create_all_paths()
|
self.create_all_paths()
|
||||||
|
|
||||||
def get_all_paths(self):
|
def get_all_paths(self):
|
||||||
|
"""
|
||||||
|
This method is used to get all the paths to be crawled.
|
||||||
|
Its goal is to get all the paths to be crawled and to store them in
|
||||||
|
the self.directories and self.files variables.
|
||||||
|
"""
|
||||||
self.files, self.directories = self.list_recursively(self.initial_path, self.initial_path)
|
self.files, self.directories = self.list_recursively(self.initial_path, self.initial_path)
|
||||||
|
|
||||||
def list_recursively(self, path, root):
|
def list_recursively(self, path, root):
|
||||||
|
"""
|
||||||
|
Recursive method to get all the paths to be crawled.
|
||||||
|
:param path: The current path to be crawled
|
||||||
|
:param root: The root path (initial path)
|
||||||
|
:type path: str
|
||||||
|
:type root: str
|
||||||
|
:return: A tuple of the form (files, directories)
|
||||||
|
:rtype: (list[(str, str)], list[(str, str)])
|
||||||
|
"""
|
||||||
files = []
|
files = []
|
||||||
directories = []
|
directories = []
|
||||||
for element in os.listdir(path):
|
for element in os.listdir(path):
|
||||||
|
|
@ -52,6 +81,9 @@ class SiteCrawler:
|
||||||
return files, directories
|
return files, directories
|
||||||
|
|
||||||
def create_all_paths(self):
|
def create_all_paths(self):
|
||||||
|
"""
|
||||||
|
This method is used to create all the directories in the destination path.
|
||||||
|
"""
|
||||||
if not os.path.exists(self.destination_path):
|
if not os.path.exists(self.destination_path):
|
||||||
os.mkdir(self.destination_path)
|
os.mkdir(self.destination_path)
|
||||||
for root, di in self.directories:
|
for root, di in self.directories:
|
||||||
|
|
@ -60,6 +92,12 @@ class SiteCrawler:
|
||||||
os.mkdir(temp_path)
|
os.mkdir(temp_path)
|
||||||
|
|
||||||
def set_all_preparsers(self):
|
def set_all_preparsers(self):
|
||||||
|
"""
|
||||||
|
This method is used to set all the preparsers and initialize them.
|
||||||
|
The preparsers are stored in the self.preparsers variable.
|
||||||
|
:return: self.preparsers
|
||||||
|
:rtype: list[preparser.PreParser]
|
||||||
|
"""
|
||||||
for root, file in self.files:
|
for root, file in self.files:
|
||||||
preparser_path = os.path.join(self.initial_path, root, file)
|
preparser_path = os.path.join(self.initial_path, root, file)
|
||||||
pp = preparser.PreParser(preparser_path, self._env, dict_of_imports=self.global_dict_of_imports)
|
pp = preparser.PreParser(preparser_path, self._env, dict_of_imports=self.global_dict_of_imports)
|
||||||
|
|
@ -70,6 +108,12 @@ class SiteCrawler:
|
||||||
return self.preparsers
|
return self.preparsers
|
||||||
|
|
||||||
def create_file(self, path):
|
def create_file(self, path):
|
||||||
|
"""
|
||||||
|
This method is used to create a file.
|
||||||
|
:raises FileExistsError: If the file already exists and the force_rewrite option is False
|
||||||
|
:param path: The path to the file to be created
|
||||||
|
:type path: str
|
||||||
|
"""
|
||||||
if os.path.exists(path):
|
if os.path.exists(path):
|
||||||
if not self.force_rewrite:
|
if not self.force_rewrite:
|
||||||
error_mngr.log_exception(
|
error_mngr.log_exception(
|
||||||
|
|
@ -81,6 +125,11 @@ class SiteCrawler:
|
||||||
return path
|
return path
|
||||||
|
|
||||||
def __iter__(self):
|
def __iter__(self):
|
||||||
|
"""
|
||||||
|
This method is used to iterate over the preparsers and files.
|
||||||
|
:yield: A tuple of the form (PreParser, file)
|
||||||
|
:ytype: (preparser.PreParser, str)
|
||||||
|
"""
|
||||||
for pre in self.preparsers:
|
for pre in self.preparsers:
|
||||||
yield pre
|
yield pre
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue