From c5d46215d94b9f9d98c3a6b2082eb0eb4e7bafd5 Mon Sep 17 00:00:00 2001 From: Maximilien Rigaut Date: Wed, 9 Feb 2022 17:52:20 +0100 Subject: [PATCH] Added the use of tempfile and started the test_sort --- src/criteriaSorter/modules/criteriaSorter.py | 184 +++++++++++++++---- tests/modules/test_criteriaSorter.py | 75 +++++++- 2 files changed, 218 insertions(+), 41 deletions(-) diff --git a/src/criteriaSorter/modules/criteriaSorter.py b/src/criteriaSorter/modules/criteriaSorter.py index 9f29a27..6d02bb1 100644 --- a/src/criteriaSorter/modules/criteriaSorter.py +++ b/src/criteriaSorter/modules/criteriaSorter.py @@ -11,40 +11,63 @@ from rich.logging import RichHandler from criteriaSorter.modules.fileops import DirectoryHandler, FileHandler, ArtistHandler -def action_sort(argsp): - # Load the config file - with open(argsp.config, 'r') as stream: - try: - config = yaml.load(stream, Loader=yaml.FullLoader) - except yaml.YAMLError as exc: - logging.critical(exc) - sys.exit(1) +def load_config(config_file): + """ + Load the config file + :param config_file: path to the config file + :return: config (the loaded config) + """ + with open(config_file, 'r') as stream: + config = yaml.safe_load(stream) + return config - # Load the operations + +def load_operations(operation_to_load, config): + """ + Load the operations from the config file + Operations are stored in a dictionary with the key being the operation name + The operations are the various operations to perform on the files, according to a list of criteria + :param operation_to_load: the operation to load (string) (default: default_operations) + :param config: the config file (yaml) + :return: operations (list of operations) + """ try: - if argsp.operations == "default_operations": + if operation_to_load == "default_operations": logging.info("Loading default operations") operations_name = config["general"]['default_operations'] else: - operations_name = config[argsp.operation] + operations_name = config[operation_to_load] operations_config = config["operations"][operations_name] except Exception: logging.critical("Could not load operations") sys.exit(1) + return operations_config - # Pick the right handler - if config["general"]["handler"] == "FileHandler": + +def load_handler(handler_name): + """ + Pick the handler to use in the config or raises an error if it does not exist + :param handler_name: The name of the handler to load + :return: Handler (the Handler object) + """ + if handler_name == "FileHandler": Handler = FileHandler - elif config["general"]["handler"] == "ArtistHandler": + elif handler_name == "ArtistHandler": Handler = ArtistHandler else: - logging.critical("Could not load handler "+config["general"]["handler"]) + logging.critical("Could not load handler "+handler_name) sys.exit(1) + return Handler - # Create the DirectorySorter - directory_handler = DirectoryHandler(argsp.folder) - # Get the list of Handler from the directory +def create_handler_list(directory_handler, Handler, config): + """ + Create a list of handlers for every file in the directory, recursively or not + :param directory_handler: The directory handler + :param Handler: The handler to use + :param config: The config file + :return: list of handlers for every file in the directory(ies) + """ handler_list = [] for file in directory_handler.get_files(): try: @@ -56,8 +79,22 @@ def action_sort(argsp): logging.error(e) logging.debug(e, exc_info=True) # sys.exit(1) # Unsure if this is the right thing to do + if config["general"]["recursive"]: + # for directory in directory_handler.get_directories(): + # logging.debug("[Handler list] Processing directory {}".format(directory)) + # directory_handler = DirectoryHandler(directory) + # handler_list.extend(create_handler_list(directory_handler, Handler, operations_config, config)) + pass + return handler_list - # Get the action to perform on each file from the handler and the operations + +def create_operation_list(operations_config, config): + """ + Create a list of operations to perform on the files + :param operations_config: The config file for the operations + :param config: The general config file + :return: list of operations to perform on the files + """ operation_list = [] for operation in operations_config["operation_order"].split("\n")[:-1]: try: @@ -67,14 +104,18 @@ def action_sort(argsp): logging.error("[Operation list] Could not load operation {} : {}".format(operation, e)) logging.debug(e, exc_info=True) # sys.exit(1) # Unsure if this is the right thing to do + return operation_list - # Find the default destination - if "default_destination" in operations_config: - default_destination = operations_config["default_destination"]["destination"] - else: - default_destination = None - # Sort the files +def execute_sorting(handler_list, operation_list, default_destination, argsp): + """ + Execute the sorting process on all files + :param handler_list: The list of handlers to use + :param operation_list: The list of operations to perform (according to criteria) + :param default_destination: The default destination for the files (if no operation found) + :param argsp: The arguments passed to the program + :return: None + """ for handler in handler_list: try: # rich.inspect(operation_list) @@ -86,31 +127,87 @@ def action_sort(argsp): logging.error(e) logging.debug(e, exc_info=True) - # Move the files - list_of_operations = [] + +def execute_moves(handler_list, argsp, output_directory): + list_of_moves = [] for handler in handler_list: try: logging.debug("[File moving] Processing {}".format(handler.file_name)) - operation = handler.move(destination=argsp.output, dry_run=argsp.dry_run) + operation = handler.move(destination=output_directory, dry_run=argsp.dry_run) if operation: - list_of_operations.append(operation) + list_of_moves.append(operation) except Exception as e: logging.error("[File moving] Could not move {}".format(handler.file_name)) logging.error(e) logging.debug(e, exc_info=True) + return list_of_moves - # Write a cancel file - if not argsp.dry_run and len(list_of_operations) > 0 or argsp.verbose > 3: + +def write_cancel_file(list_of_operations, output, verbose, dry_run): + """ + Write the cancel file + :param list_of_operations: The list of operations to cancel + :param output: The output file to write to + :param verbose: The verbosity of the program + :param dry_run: Whether the program is in dry run mode + :return: None + """ + if not dry_run and len(list_of_operations) > 0 or verbose > 3: logging.info("Writing cancel file...") - with open(os.path.join(argsp.output, argsp.cancel_file), "w", encoding="utf-8") as f: + with open(output, "w", encoding="utf-8") as f: for origin, destination in list_of_operations: f.write("{} : {}\n".format(origin, destination)) - logging.info("Written " + argsp.cancel_file) + logging.info("Written " + output) - logging.info("Done") + +def action_sort(argsp): + """ + Sort the files + :param argsp: the arguments passed to the program + :return: None + """ + # Load the config file + config = load_config(argsp.config) + + # Load the operations + operations_config = load_operations(argsp.operations, config) + + # Pick the right handler + Handler = load_handler(config["general"]["handler"]) + + # Create the DirectorySorter + directory_handler = DirectoryHandler(argsp.folder) + + # Get the list of Handler from the directory + handler_list = create_handler_list(directory_handler, Handler, config) + + # Get the action to perform on each file from the handler and the operations + operation_list = create_operation_list(operations_config, handler_list) + + # Find the default destination + if "default_destination" in operations_config: + default_destination = operations_config["default_destination"]["destination"] + else: + default_destination = None + + # Sort the files + execute_sorting(handler_list, operation_list, default_destination, argsp) + + # Move the files + list_of_moves = execute_moves(handler_list, argsp, default_destination) + + # Write a cancel file + write_cancel_file(list_of_moves, os.path.join(argsp.output, argsp.cancel_file), argsp.verbose, argsp.dry_run) + + logging.info("All operations done") def action_list(argsp): + """ + List the sorting operations + :param argsp: the arguments passed to the program + :return: None + """ try: logging.info("Listing operations") with open(argsp.config, 'r') as stream: @@ -133,6 +230,11 @@ def action_list(argsp): def action_cancel(argsp): + """ + Cancel the operations done by the program, given a cancel file + :param argsp: The arguments passed to the program + :return: None + """ with open(os.path.join(argsp.cancel_file), "r") as f: for line in f: try: @@ -153,6 +255,11 @@ _LIST_ACTIONS = { def config_log(argsp): + """ + Configure the logging + :param argsp: The arguments passed to the program + :return: None + """ # Set up logging # _LOG_FORMAT = ' > [%(levelname)s:%(filename)s] - %(message)s' _LOG_FORMAT = '%(message)s' @@ -169,6 +276,11 @@ def config_log(argsp): def parse_args(argvp): + """ + Parse the arguments passed to the program + :param argvp: The arguments passed to the program + :return: argsp: The arguments, parsed as an object + """ if len(argvp) == 1: sys.argv.append('--help') @@ -202,8 +314,8 @@ def parse_args(argvp): def main(argvp): """ - Main function - :return: + Main function, called by __main__ + :return: None """ # Parse arguments args = parse_args(argvp) diff --git a/tests/modules/test_criteriaSorter.py b/tests/modules/test_criteriaSorter.py index 4621125..b4aa606 100644 --- a/tests/modules/test_criteriaSorter.py +++ b/tests/modules/test_criteriaSorter.py @@ -5,6 +5,7 @@ import logging import importlib import pytest from criteriaSorter.modules import criteriaSorter +import tempfile _ARGS_LIST = [ @@ -43,6 +44,12 @@ _GOOD_RESULT = """[MOVED] path/to/destination1 to path/to/file1 """ +def cldl(*l_file): + for f in l_file: + f.close() + os.remove(f.name) + + @pytest.mark.parametrize("args, outp", _EXIT_ARGS_LIST) def test_parse_args_exit(args, outp, capsys): args = args.split(" ") @@ -102,16 +109,18 @@ class falseArg: (io.StringIO(_GOOD_FILE_LIST), _GOOD_RESULT), (io.StringIO(_JUNK_FILE_LIST), _JUNK_RESULT) ]) def test_action_cancel(monkeypatch, list_of_cancels, result_expected, capsys): + cancel_file = tempfile.NamedTemporaryFile(mode="w+", suffix=".txt", delete=False) monkeypatch.setattr(os, "rename", move) - monkeypatch.setattr(os, "remove", move) + # monkeypatch.setattr(os, "remove", move) monkeypatch.setattr(os, "rmdir", move) monkeypatch.setattr(os.path, "join", lambda a: a) - with open("cancelfile.txt", "w") as f: + with open(cancel_file.name, "w") as f: f.write(list_of_cancels.read()) - criteriaSorter.action_cancel(falseArg("cancelfile.txt")) + criteriaSorter.action_cancel(falseArg(cancel_file.name)) assert result_expected in capsys.readouterr().out + cldl(cancel_file) @pytest.mark.parametrize("verbose", [0, 1, 2, 3, 4]) def test_action_list(verbose, capsys): @@ -129,8 +138,8 @@ def test_action_list(verbose, capsys): [["--config=unexisting_conf", "list"], "Config unexisting_conf file not found"], ]) def test_action_list_error(args, error, caplog, monkeypatch): - - with open("noconf.yaml", "w") as f: + no_conf = tempfile.NamedTemporaryFile(mode="w+t", delete=False) + with open(no_conf.name, "w") as f: f.write("Bad config =:/\n+3") args = criteriaSorter.parse_args(args) @@ -138,3 +147,59 @@ def test_action_list_error(args, error, caplog, monkeypatch): with caplog.at_level(logging.ERROR): criteriaSorter.action_list(args) assert error in caplog.text + cldl(no_conf) + +def test_load_config(): + good_conf = tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) + with open(good_conf.name, "w") as f: + f.write("""test: + - path: path/to/file1 + """) + + assert criteriaSorter.load_config(good_conf.name) == {"test": [{"path": "path/to/file1"}]} + cldl(good_conf) + + +def test_load_config_error(): + bad_conf = tempfile.NamedTemporaryFile(suffix=".yaml", delete=False) + good_conf = tempfile.NamedTemporaryFile(suffix=".yaml", delete=False) + + with open(bad_conf.name, "w") as f: + f.write("""test: + - path: path/to/file1 + condition: + - path: path/to/file1 + """) + + with open(good_conf.name, "w") as f: + f.write("""operations: + sort_junk_folder: + operation_order: | + operation1 + operation2 + operation3 + operation4 + operation1: + conditions : | + has_artist + is_image + destination : Artists/{obj.artist}/{obj.name} + operation2: + conditions : | + is_video + destination : vids/{obj.name} + operation3: + conditions : | + is_audio + destination : audios/{obj.name} + operation4: + conditions : | + is_document + destination : others/{obj.name} +""") + + with pytest.raises(SystemExit): + conf = criteriaSorter.load_config(bad_conf.name) + criteriaSorter.load_operations("default_operation", conf) + + cldl(good_conf, bad_conf) \ No newline at end of file