commit
ffb8479e53
38
README.md
38
README.md
|
|
@ -10,37 +10,37 @@ You can refer to the `example_userfiles` for an idea of what the parser expects
|
|||
---
|
||||
## Roadmap
|
||||
### V1.0.1
|
||||
- Make the default example site way more useful
|
||||
- Add the type and return types of all functions
|
||||
- Add documentation to remaining lone functions
|
||||
- Write a documentation for the application (use specs.yaml)
|
||||
- Make the default example site way more useful ☐
|
||||
- Add the type and return types of all functions ☑
|
||||
- Add documentation to remaining lone functions ☑
|
||||
- Write a documentation for the application (use specs.yaml) ☐
|
||||
|
||||
### V1.0.2
|
||||
- Move the WARNING-level output to INFO for config overwrite
|
||||
- Move the WARNING-level output to INFO for config overwrite ☐
|
||||
|
||||
### V1.0.3
|
||||
- Copy un-parsable files to destination folder without modifications
|
||||
- Add copy behaviour to config file
|
||||
- Copy un-parsable files to destination folder without modifications ☑
|
||||
- Add copy behaviour to config file ☑
|
||||
|
||||
### V1.1
|
||||
- Functioning `table` Token
|
||||
- Functioning `code` Token
|
||||
- Establish a list of all configurable parameters to implement in the future, and update the roadmap with them.
|
||||
- Decide level of logic to be implemented, and whether it should be configurable
|
||||
- Update the uses of remaining files
|
||||
- Functioning `table` Token ☐
|
||||
- Functioning `code` Token ☐
|
||||
- Establish a list of all configurable parameters to implement in the future, and update the roadmap with them. ☐
|
||||
- Decide level of logic to be implemented, and whether it should be configurable ☐
|
||||
- Update the uses of remaining files ☐
|
||||
|
||||
|
||||
### V1.2
|
||||
- Check error messages and add a real debug level to parameters
|
||||
- Functioning `blockquote` Token
|
||||
- Remove old regex usage
|
||||
- Check error messages and add a real debug level to parameters ☐
|
||||
- Functioning `blockquote` Token ☐
|
||||
- Remove old regex usage ☐
|
||||
|
||||
### V1.3
|
||||
- Add advanced lookahead logic for `*`
|
||||
- Add advanced lookahead logic for `*` ☐
|
||||
|
||||
### V1.4
|
||||
- Add html indentation for human readability of the output.
|
||||
- Add html indentation for human readability of the output. ☐
|
||||
|
||||
### V2
|
||||
- Add a functioning post-context enhancer able to generate menus and elements from arbitrary logic.
|
||||
- Achieve perfect markdown compatibility with appropriate config parameters
|
||||
- Add a functioning post-context enhancer able to generate menus and elements from arbitrary logic. ☐
|
||||
- Achieve perfect markdown compatibility with appropriate config parameters ☐
|
||||
|
|
|
|||
|
|
@ -21,3 +21,4 @@ export:
|
|||
intermediate_files: true
|
||||
type: "html"
|
||||
force_rewrite: true
|
||||
copy_unparsable_files: "copy" # "copy" or "ignore"
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
# Module for file and directories repartition
|
||||
import os
|
||||
import shutil
|
||||
from bootstraparse.modules import pathresolver, preparser, error_mngr, environment, export
|
||||
|
||||
|
||||
|
|
@ -35,6 +36,7 @@ class SiteCrawler:
|
|||
self.force_rewrite = self._env.config["parser_config"]["export"]["force_rewrite"]
|
||||
self.directories = []
|
||||
self.files = []
|
||||
self.files_to_copy = []
|
||||
self.preparsers = []
|
||||
self.global_dict_of_imports = {}
|
||||
|
||||
|
|
@ -50,9 +52,9 @@ class SiteCrawler:
|
|||
"""
|
||||
This method is used to get all the paths to be crawled.
|
||||
Its goal is to get all the paths to be crawled and to store them in
|
||||
the self.directories and self.files variables.
|
||||
the self.directories and self.files, self.files_to_copy variables.
|
||||
"""
|
||||
self.files, self.directories = self.list_recursively(self.initial_path, self.initial_path)
|
||||
self.files, self.files_to_copy, self.directories = self.list_recursively(self.initial_path, self.initial_path)
|
||||
|
||||
def list_recursively(self, path, root):
|
||||
"""
|
||||
|
|
@ -62,23 +64,28 @@ class SiteCrawler:
|
|||
:type path: str
|
||||
:type root: str
|
||||
:return: A tuple of the form (files, directories)
|
||||
:rtype: (list[(str, str)], list[(str, str)])
|
||||
:rtype: (list[(str, str)], list[(str, str)], list[(str, str)])
|
||||
"""
|
||||
files = []
|
||||
directories = []
|
||||
files_to_copy = []
|
||||
for element in os.listdir(path):
|
||||
element_fpath = os.path.join(path, element)
|
||||
element_rpath = os.path.relpath(path, root)
|
||||
if os.path.isdir(element_fpath):
|
||||
if element not in self.forbidden_folders:
|
||||
directories.append((element_rpath, element))
|
||||
f, d = self.list_recursively(element_fpath, root)
|
||||
f, fc, d = self.list_recursively(element_fpath, root)
|
||||
files += f
|
||||
directories += d
|
||||
files_to_copy += fc
|
||||
else:
|
||||
if os.path.splitext(element)[1] in self.authorised_extensions and element[0] != "_":
|
||||
files.append((element_rpath, element))
|
||||
return files, directories
|
||||
if element[0] != "_":
|
||||
if os.path.splitext(element)[1] in self.authorised_extensions:
|
||||
files.append((element_rpath, element))
|
||||
else:
|
||||
files_to_copy.append((element_rpath, element))
|
||||
return files, files_to_copy, directories
|
||||
|
||||
def create_all_paths(self):
|
||||
"""
|
||||
|
|
@ -107,6 +114,14 @@ class SiteCrawler:
|
|||
|
||||
return self.preparsers
|
||||
|
||||
def copy_unparsable_files(self):
|
||||
"""
|
||||
This method is used to copy all the files that could not be parsed.
|
||||
"""
|
||||
if self._env.config["parser_config"]["export"]["copy_unparsable_files"].lower() == "copy":
|
||||
for root, file in self.files_to_copy:
|
||||
shutil.copy(os.path.join(self.initial_path, root, file), os.path.join(self.destination_path, root, file))
|
||||
|
||||
def create_file(self, path):
|
||||
"""
|
||||
This method is used to create a file.
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ def create_website(origin, destination):
|
|||
env = create_environment(origin, destination)
|
||||
crwlr = create_crawler(origin, destination, env)
|
||||
crwlr.set_all_preparsers()
|
||||
crwlr.copy_unparsable_files()
|
||||
for element, destination in crwlr:
|
||||
save(preparse_parse(element), destination, env)
|
||||
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ files = [
|
|||
("base/config/test6.bpr", "", None),
|
||||
("base/config/test7.yml", "", None),
|
||||
("base/templates/test8.yml", "", None),
|
||||
("base/unparsable.php", "", ""),
|
||||
]
|
||||
|
||||
|
||||
|
|
@ -96,6 +97,7 @@ def test_result_crawler(list_files, env):
|
|||
"""
|
||||
crw = sitecrawler.SiteCrawler(_BASE, _DEST, env)
|
||||
crw.set_all_preparsers()
|
||||
crw.copy_unparsable_files()
|
||||
for pp, dest in crw:
|
||||
assert os.path.exists(dest)
|
||||
assert os.path.isfile(dest)
|
||||
|
|
|
|||
Loading…
Reference in New Issue