From e9837dbf9977afe561fa3ac3b1be16c884cc5d03 Mon Sep 17 00:00:00 2001 From: Maximilien Rigaut Date: Sun, 12 Jun 2022 14:07:03 +0200 Subject: [PATCH] Copy feature --- README.md | 38 ++++++++++----------- src/bootstraparse/configs/parser_config.yml | 1 + src/bootstraparse/modules/sitecrawler.py | 28 +++++++++++---- src/bootstraparse/modules/sitecreator.py | 1 + 4 files changed, 42 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index da087b2..19626b6 100644 --- a/README.md +++ b/README.md @@ -10,37 +10,37 @@ You can refer to the `example_userfiles` for an idea of what the parser expects --- ## Roadmap ### V1.0.1 -- Make the default example site way more useful -- Add the type and return types of all functions -- Add documentation to remaining lone functions -- Write a documentation for the application (use specs.yaml) +- Make the default example site way more useful ☐ +- Add the type and return types of all functions ☑ +- Add documentation to remaining lone functions ☑ +- Write a documentation for the application (use specs.yaml) ☐ ### V1.0.2 -- Move the WARNING-level output to INFO for config overwrite +- Move the WARNING-level output to INFO for config overwrite ☐ ### V1.0.3 -- Copy un-parsable files to destination folder without modifications -- Add copy behaviour to config file +- Copy un-parsable files to destination folder without modifications ☑ +- Add copy behaviour to config file ☑ ### V1.1 -- Functioning `table` Token -- Functioning `code` Token -- Establish a list of all configurable parameters to implement in the future, and update the roadmap with them. -- Decide level of logic to be implemented, and whether it should be configurable -- Update the uses of remaining files +- Functioning `table` Token ☐ +- Functioning `code` Token ☐ +- Establish a list of all configurable parameters to implement in the future, and update the roadmap with them. ☐ +- Decide level of logic to be implemented, and whether it should be configurable ☐ +- Update the uses of remaining files ☐ ### V1.2 -- Check error messages and add a real debug level to parameters -- Functioning `blockquote` Token -- Remove old regex usage +- Check error messages and add a real debug level to parameters ☐ +- Functioning `blockquote` Token ☐ +- Remove old regex usage ☐ ### V1.3 -- Add advanced lookahead logic for `*` +- Add advanced lookahead logic for `*` ☐ ### V1.4 -- Add html indentation for human readability of the output. +- Add html indentation for human readability of the output. ☐ ### V2 -- Add a functioning post-context enhancer able to generate menus and elements from arbitrary logic. -- Achieve perfect markdown compatibility with appropriate config parameters +- Add a functioning post-context enhancer able to generate menus and elements from arbitrary logic. ☐ +- Achieve perfect markdown compatibility with appropriate config parameters ☐ diff --git a/src/bootstraparse/configs/parser_config.yml b/src/bootstraparse/configs/parser_config.yml index d6899af..2deb3ce 100644 --- a/src/bootstraparse/configs/parser_config.yml +++ b/src/bootstraparse/configs/parser_config.yml @@ -21,3 +21,4 @@ export: intermediate_files: true type: "html" force_rewrite: true + unparsable_files: "copy" # "copy" or "ignore" diff --git a/src/bootstraparse/modules/sitecrawler.py b/src/bootstraparse/modules/sitecrawler.py index 6246f16..ac702dc 100644 --- a/src/bootstraparse/modules/sitecrawler.py +++ b/src/bootstraparse/modules/sitecrawler.py @@ -1,5 +1,6 @@ # Module for file and directories repartition import os +import shutil from bootstraparse.modules import pathresolver, preparser, error_mngr, environment, export @@ -35,6 +36,7 @@ class SiteCrawler: self.force_rewrite = self._env.config["parser_config"]["export"]["force_rewrite"] self.directories = [] self.files = [] + self.files_to_copy = [] self.preparsers = [] self.global_dict_of_imports = {} @@ -50,9 +52,9 @@ class SiteCrawler: """ This method is used to get all the paths to be crawled. Its goal is to get all the paths to be crawled and to store them in - the self.directories and self.files variables. + the self.directories and self.files, self.files_to_copy variables. """ - self.files, self.directories = self.list_recursively(self.initial_path, self.initial_path) + self.files, self.files_to_copy, self.directories = self.list_recursively(self.initial_path, self.initial_path) def list_recursively(self, path, root): """ @@ -62,23 +64,28 @@ class SiteCrawler: :type path: str :type root: str :return: A tuple of the form (files, directories) - :rtype: (list[(str, str)], list[(str, str)]) + :rtype: (list[(str, str)], list[(str, str)], list[(str, str)]) """ files = [] directories = [] + files_to_copy = [] for element in os.listdir(path): element_fpath = os.path.join(path, element) element_rpath = os.path.relpath(path, root) if os.path.isdir(element_fpath): if element not in self.forbidden_folders: directories.append((element_rpath, element)) - f, d = self.list_recursively(element_fpath, root) + f, fc, d = self.list_recursively(element_fpath, root) files += f directories += d + files_to_copy += fc else: - if os.path.splitext(element)[1] in self.authorised_extensions and element[0] != "_": - files.append((element_rpath, element)) - return files, directories + if element[0] != "_": + if os.path.splitext(element)[1] in self.authorised_extensions: + files.append((element_rpath, element)) + else: + files_to_copy.append((element_rpath, element)) + return files, files_to_copy, directories def create_all_paths(self): """ @@ -107,6 +114,13 @@ class SiteCrawler: return self.preparsers + def copy_unparsable_files(self): + """ + This method is used to copy all the files that could not be parsed. + """ + for root, file in self.files_to_copy: + shutil.copy(os.path.join(self.initial_path, root, file), os.path.join(self.destination_path, root, file)) + def create_file(self, path): """ This method is used to create a file. diff --git a/src/bootstraparse/modules/sitecreator.py b/src/bootstraparse/modules/sitecreator.py index ecaec58..6bad6e0 100644 --- a/src/bootstraparse/modules/sitecreator.py +++ b/src/bootstraparse/modules/sitecreator.py @@ -17,6 +17,7 @@ def create_website(origin, destination): env = create_environment(origin, destination) crwlr = create_crawler(origin, destination, env) crwlr.set_all_preparsers() + crwlr.copy_unparsable_files() for element, destination in crwlr: save(preparse_parse(element), destination, env)