From bc41cb8541b6b54cf4784bc78c34aeac061b6ab4 Mon Sep 17 00:00:00 2001 From: dandaedre Date: Sun, 20 Feb 2022 21:24:45 +0100 Subject: [PATCH] Finished a better implementation of the config --- src/bootstraparse/modules/config.py | 77 ++++++++++++++++-------- src/bootstraparse/modules/sitecreator.py | 2 +- tests/modules/test_config.py | 22 +++++-- 3 files changed, 70 insertions(+), 31 deletions(-) diff --git a/src/bootstraparse/modules/config.py b/src/bootstraparse/modules/config.py index beb458e..cf93379 100644 --- a/src/bootstraparse/modules/config.py +++ b/src/bootstraparse/modules/config.py @@ -12,17 +12,40 @@ class ConfigLoader: :param config_folder: The folder to read """ - def __init__(self, config_folder, extensions="yaml"): + def __init__(self, config_folder=None, extensions="yaml"): """ Defines the config folder and loads all configs :param config_folder: path to config file :param extensions: file extensions to load :return: None """ - self.config_folder = config_folder + if config_folder is None: + self.config_folders = [] + elif isinstance(config_folder, str): + self.config_folders = [config_folder] + else: + self.config_folders = config_folder self.loaded_conf = {} self.extensions = extensions - self.load_configs() + self.reload_all() + + def reload_all(self): + """ + Reloads all configs + :return: None + """ + self.loaded_conf = {} + for folder in self.config_folders: + self.load_from_folder(folder) + + def add_folder(self, folder): + """ + Add a Folder to the list of configs + :param folder: path to folder + :return: None + """ + self.config_folders.append(folder) + self.load_from_folder(folder) def load_from_file(self, filepath): """ @@ -33,17 +56,21 @@ class ConfigLoader: basename = os.path.basename(filepath) name, ext = os.path.splitext(basename) with open(filepath, "r") as f: - self.loaded_conf[name] = yaml.safe_load(f) + if name not in self.loaded_conf: + self.loaded_conf[name] = yaml.safe_load(f) + else: + self.loaded_conf[name].update(yaml.safe_load(f)) + print(f"Warning: {name} is already in {self.loaded_conf}") - def load_configs(self): + def load_from_folder(self, folder): """ Loads all configs in the config folder :return: None """ - for filename in os.listdir(self.config_folder): + for filename in os.listdir(folder): file_ext = os.path.splitext(filename)[1][1:] if file_ext in self.extensions: - self.load_from_file(os.path.join(self.config_folder, filename)) + self.load_from_file(os.path.join(folder, filename)) def __getitem__(self, item): """ @@ -65,24 +92,24 @@ class ConfigLoader: return self.loaded_conf.__repr__() -class UserConfig(ConfigLoader): - """ - Reads user config file. - :param config_folder: path to config file - :return: None - """ - def __init__(self, config_folder): - super().__init__(config_folder) - - -class GlobalConfig(ConfigLoader): - """ - Reads global config file. - :param config_folder: path to config file - :return: None - """ - def __init__(self, config_folder): - super().__init__(config_folder) +# class UserConfig(ConfigLoader): +# """ +# Reads user config file. +# :param config_folder: path to config file +# :return: None +# """ +# def __init__(self, config_folder): +# super().__init__(config_folder) +# +# +# class GlobalConfig(ConfigLoader): +# """ +# Reads global config file. +# :param config_folder: path to config file +# :return: None +# """ +# def __init__(self, config_folder): +# super().__init__(config_folder) # conf = ConfigLoader("./example_userfiles/config/aliases.yaml") diff --git a/src/bootstraparse/modules/sitecreator.py b/src/bootstraparse/modules/sitecreator.py index 6e84e9c..f940397 100644 --- a/src/bootstraparse/modules/sitecreator.py +++ b/src/bootstraparse/modules/sitecreator.py @@ -24,7 +24,7 @@ def create_config(): """ Returns globalConfig as an object from parser_config.yml settings. """ - return config.GlobalConfig(b_path("configs/")) + return config.ConfigLoader(b_path("configs/")) def create_crawler(): diff --git a/tests/modules/test_config.py b/tests/modules/test_config.py index 544a5bb..f27fe2b 100644 --- a/tests/modules/test_config.py +++ b/tests/modules/test_config.py @@ -8,6 +8,7 @@ import tempfile temp_folder = tempfile.TemporaryDirectory() configs = { "app/configs/parser_config.yaml": '{"parser": {"name": "test_parser", "type": "test_parser"}}', + "app/configs/aliases.yaml": '{"parser": {"name": "test_parser", "type": "test_parser"}}', "user/config/aliases.yaml": '{"aliases": {"test_alias": "test_parser"}}', "user/config/glossary.yaml": '{"glossary": {"test_glossary": "test_parser"}}', "user/config/custom_template.yaml": '{"custom_template": {"test_template": "test_parser"}}', @@ -37,9 +38,20 @@ def test_config_load(): assert usr_c["aliases2"] is not None assert usr_c.__repr__() +def test_add_to_config(): + usr_c = config.ConfigLoader(user_conf) + usr_c.add_folder(app_conf) + from_empty = config.ConfigLoader() + from_list = config.ConfigLoader([user_conf, app_conf]) + assert from_list["aliases"] == {"aliases": {"test_alias": "test_parser"}, + "parser": {"name": "test_parser", "type": "test_parser"}} + assert from_list["parser_config"] == {"parser": {"name": "test_parser", "type": "test_parser"}} -def test_children(): - usr_c = config.UserConfig(user_conf) - site_c = config.GlobalConfig(app_conf) - assert usr_c["glossary"] == {"glossary": {"test_glossary": "test_parser"}} - assert site_c["parser_config"] == {"parser": {"name": "test_parser", "type": "test_parser"}} + with pytest.raises(FileNotFoundError): + assert from_empty["aliases"] + from_empty.add_folder(user_conf) + assert from_empty["aliases"] == {"aliases": {"test_alias": "test_parser"}} + + from_empty.load_from_file(os.path.join(app_conf, "aliases.yaml")) + assert from_empty["aliases"] == {"aliases": {"test_alias": "test_parser"}, + "parser": {"name": "test_parser", "type": "test_parser"}}