From 3abb5c53f74a128be41529b9fecfd6f2443917f9 Mon Sep 17 00:00:00 2001 From: Maximilien Rigaut Date: Sun, 12 Jun 2022 21:00:10 +0200 Subject: [PATCH] Added copy behaviour to config file --- src/bootstraparse/configs/parser_config.yml | 2 +- src/bootstraparse/modules/sitecrawler.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/bootstraparse/configs/parser_config.yml b/src/bootstraparse/configs/parser_config.yml index 2deb3ce..0dcc7ae 100644 --- a/src/bootstraparse/configs/parser_config.yml +++ b/src/bootstraparse/configs/parser_config.yml @@ -21,4 +21,4 @@ export: intermediate_files: true type: "html" force_rewrite: true - unparsable_files: "copy" # "copy" or "ignore" + copy_unparsable_files: "copy" # "copy" or "ignore" diff --git a/src/bootstraparse/modules/sitecrawler.py b/src/bootstraparse/modules/sitecrawler.py index ac702dc..6620475 100644 --- a/src/bootstraparse/modules/sitecrawler.py +++ b/src/bootstraparse/modules/sitecrawler.py @@ -118,8 +118,9 @@ class SiteCrawler: """ This method is used to copy all the files that could not be parsed. """ - for root, file in self.files_to_copy: - shutil.copy(os.path.join(self.initial_path, root, file), os.path.join(self.destination_path, root, file)) + if self._env.config["parser_config"]["export"]["copy_unparsable_files"].lower() == "copy": + for root, file in self.files_to_copy: + shutil.copy(os.path.join(self.initial_path, root, file), os.path.join(self.destination_path, root, file)) def create_file(self, path): """