Added the use of tempfile and started the test_sort
This commit is contained in:
parent
36d0c0c30d
commit
c5d46215d9
|
|
@ -11,40 +11,63 @@ from rich.logging import RichHandler
|
|||
from criteriaSorter.modules.fileops import DirectoryHandler, FileHandler, ArtistHandler
|
||||
|
||||
|
||||
def action_sort(argsp):
|
||||
# Load the config file
|
||||
with open(argsp.config, 'r') as stream:
|
||||
try:
|
||||
config = yaml.load(stream, Loader=yaml.FullLoader)
|
||||
except yaml.YAMLError as exc:
|
||||
logging.critical(exc)
|
||||
sys.exit(1)
|
||||
def load_config(config_file):
|
||||
"""
|
||||
Load the config file
|
||||
:param config_file: path to the config file
|
||||
:return: config (the loaded config)
|
||||
"""
|
||||
with open(config_file, 'r') as stream:
|
||||
config = yaml.safe_load(stream)
|
||||
return config
|
||||
|
||||
# Load the operations
|
||||
|
||||
def load_operations(operation_to_load, config):
|
||||
"""
|
||||
Load the operations from the config file
|
||||
Operations are stored in a dictionary with the key being the operation name
|
||||
The operations are the various operations to perform on the files, according to a list of criteria
|
||||
:param operation_to_load: the operation to load (string) (default: default_operations)
|
||||
:param config: the config file (yaml)
|
||||
:return: operations (list of operations)
|
||||
"""
|
||||
try:
|
||||
if argsp.operations == "default_operations":
|
||||
if operation_to_load == "default_operations":
|
||||
logging.info("Loading default operations")
|
||||
operations_name = config["general"]['default_operations']
|
||||
else:
|
||||
operations_name = config[argsp.operation]
|
||||
operations_name = config[operation_to_load]
|
||||
operations_config = config["operations"][operations_name]
|
||||
except Exception:
|
||||
logging.critical("Could not load operations")
|
||||
sys.exit(1)
|
||||
return operations_config
|
||||
|
||||
# Pick the right handler
|
||||
if config["general"]["handler"] == "FileHandler":
|
||||
|
||||
def load_handler(handler_name):
|
||||
"""
|
||||
Pick the handler to use in the config or raises an error if it does not exist
|
||||
:param handler_name: The name of the handler to load
|
||||
:return: Handler (the Handler object)
|
||||
"""
|
||||
if handler_name == "FileHandler":
|
||||
Handler = FileHandler
|
||||
elif config["general"]["handler"] == "ArtistHandler":
|
||||
elif handler_name == "ArtistHandler":
|
||||
Handler = ArtistHandler
|
||||
else:
|
||||
logging.critical("Could not load handler "+config["general"]["handler"])
|
||||
logging.critical("Could not load handler "+handler_name)
|
||||
sys.exit(1)
|
||||
return Handler
|
||||
|
||||
# Create the DirectorySorter
|
||||
directory_handler = DirectoryHandler(argsp.folder)
|
||||
|
||||
# Get the list of Handler from the directory
|
||||
def create_handler_list(directory_handler, Handler, config):
|
||||
"""
|
||||
Create a list of handlers for every file in the directory, recursively or not
|
||||
:param directory_handler: The directory handler
|
||||
:param Handler: The handler to use
|
||||
:param config: The config file
|
||||
:return: list of handlers for every file in the directory(ies)
|
||||
"""
|
||||
handler_list = []
|
||||
for file in directory_handler.get_files():
|
||||
try:
|
||||
|
|
@ -56,8 +79,22 @@ def action_sort(argsp):
|
|||
logging.error(e)
|
||||
logging.debug(e, exc_info=True)
|
||||
# sys.exit(1) # Unsure if this is the right thing to do
|
||||
if config["general"]["recursive"]:
|
||||
# for directory in directory_handler.get_directories():
|
||||
# logging.debug("[Handler list] Processing directory {}".format(directory))
|
||||
# directory_handler = DirectoryHandler(directory)
|
||||
# handler_list.extend(create_handler_list(directory_handler, Handler, operations_config, config))
|
||||
pass
|
||||
return handler_list
|
||||
|
||||
# Get the action to perform on each file from the handler and the operations
|
||||
|
||||
def create_operation_list(operations_config, config):
|
||||
"""
|
||||
Create a list of operations to perform on the files
|
||||
:param operations_config: The config file for the operations
|
||||
:param config: The general config file
|
||||
:return: list of operations to perform on the files
|
||||
"""
|
||||
operation_list = []
|
||||
for operation in operations_config["operation_order"].split("\n")[:-1]:
|
||||
try:
|
||||
|
|
@ -67,14 +104,18 @@ def action_sort(argsp):
|
|||
logging.error("[Operation list] Could not load operation {} : {}".format(operation, e))
|
||||
logging.debug(e, exc_info=True)
|
||||
# sys.exit(1) # Unsure if this is the right thing to do
|
||||
return operation_list
|
||||
|
||||
# Find the default destination
|
||||
if "default_destination" in operations_config:
|
||||
default_destination = operations_config["default_destination"]["destination"]
|
||||
else:
|
||||
default_destination = None
|
||||
|
||||
# Sort the files
|
||||
def execute_sorting(handler_list, operation_list, default_destination, argsp):
|
||||
"""
|
||||
Execute the sorting process on all files
|
||||
:param handler_list: The list of handlers to use
|
||||
:param operation_list: The list of operations to perform (according to criteria)
|
||||
:param default_destination: The default destination for the files (if no operation found)
|
||||
:param argsp: The arguments passed to the program
|
||||
:return: None
|
||||
"""
|
||||
for handler in handler_list:
|
||||
try:
|
||||
# rich.inspect(operation_list)
|
||||
|
|
@ -86,31 +127,87 @@ def action_sort(argsp):
|
|||
logging.error(e)
|
||||
logging.debug(e, exc_info=True)
|
||||
|
||||
# Move the files
|
||||
list_of_operations = []
|
||||
|
||||
def execute_moves(handler_list, argsp, output_directory):
|
||||
list_of_moves = []
|
||||
for handler in handler_list:
|
||||
try:
|
||||
logging.debug("[File moving] Processing {}".format(handler.file_name))
|
||||
operation = handler.move(destination=argsp.output, dry_run=argsp.dry_run)
|
||||
operation = handler.move(destination=output_directory, dry_run=argsp.dry_run)
|
||||
if operation:
|
||||
list_of_operations.append(operation)
|
||||
list_of_moves.append(operation)
|
||||
except Exception as e:
|
||||
logging.error("[File moving] Could not move {}".format(handler.file_name))
|
||||
logging.error(e)
|
||||
logging.debug(e, exc_info=True)
|
||||
return list_of_moves
|
||||
|
||||
# Write a cancel file
|
||||
if not argsp.dry_run and len(list_of_operations) > 0 or argsp.verbose > 3:
|
||||
|
||||
def write_cancel_file(list_of_operations, output, verbose, dry_run):
|
||||
"""
|
||||
Write the cancel file
|
||||
:param list_of_operations: The list of operations to cancel
|
||||
:param output: The output file to write to
|
||||
:param verbose: The verbosity of the program
|
||||
:param dry_run: Whether the program is in dry run mode
|
||||
:return: None
|
||||
"""
|
||||
if not dry_run and len(list_of_operations) > 0 or verbose > 3:
|
||||
logging.info("Writing cancel file...")
|
||||
with open(os.path.join(argsp.output, argsp.cancel_file), "w", encoding="utf-8") as f:
|
||||
with open(output, "w", encoding="utf-8") as f:
|
||||
for origin, destination in list_of_operations:
|
||||
f.write("{} : {}\n".format(origin, destination))
|
||||
logging.info("Written " + argsp.cancel_file)
|
||||
logging.info("Written " + output)
|
||||
|
||||
logging.info("Done")
|
||||
|
||||
def action_sort(argsp):
|
||||
"""
|
||||
Sort the files
|
||||
:param argsp: the arguments passed to the program
|
||||
:return: None
|
||||
"""
|
||||
# Load the config file
|
||||
config = load_config(argsp.config)
|
||||
|
||||
# Load the operations
|
||||
operations_config = load_operations(argsp.operations, config)
|
||||
|
||||
# Pick the right handler
|
||||
Handler = load_handler(config["general"]["handler"])
|
||||
|
||||
# Create the DirectorySorter
|
||||
directory_handler = DirectoryHandler(argsp.folder)
|
||||
|
||||
# Get the list of Handler from the directory
|
||||
handler_list = create_handler_list(directory_handler, Handler, config)
|
||||
|
||||
# Get the action to perform on each file from the handler and the operations
|
||||
operation_list = create_operation_list(operations_config, handler_list)
|
||||
|
||||
# Find the default destination
|
||||
if "default_destination" in operations_config:
|
||||
default_destination = operations_config["default_destination"]["destination"]
|
||||
else:
|
||||
default_destination = None
|
||||
|
||||
# Sort the files
|
||||
execute_sorting(handler_list, operation_list, default_destination, argsp)
|
||||
|
||||
# Move the files
|
||||
list_of_moves = execute_moves(handler_list, argsp, default_destination)
|
||||
|
||||
# Write a cancel file
|
||||
write_cancel_file(list_of_moves, os.path.join(argsp.output, argsp.cancel_file), argsp.verbose, argsp.dry_run)
|
||||
|
||||
logging.info("All operations done")
|
||||
|
||||
|
||||
def action_list(argsp):
|
||||
"""
|
||||
List the sorting operations
|
||||
:param argsp: the arguments passed to the program
|
||||
:return: None
|
||||
"""
|
||||
try:
|
||||
logging.info("Listing operations")
|
||||
with open(argsp.config, 'r') as stream:
|
||||
|
|
@ -133,6 +230,11 @@ def action_list(argsp):
|
|||
|
||||
|
||||
def action_cancel(argsp):
|
||||
"""
|
||||
Cancel the operations done by the program, given a cancel file
|
||||
:param argsp: The arguments passed to the program
|
||||
:return: None
|
||||
"""
|
||||
with open(os.path.join(argsp.cancel_file), "r") as f:
|
||||
for line in f:
|
||||
try:
|
||||
|
|
@ -153,6 +255,11 @@ _LIST_ACTIONS = {
|
|||
|
||||
|
||||
def config_log(argsp):
|
||||
"""
|
||||
Configure the logging
|
||||
:param argsp: The arguments passed to the program
|
||||
:return: None
|
||||
"""
|
||||
# Set up logging
|
||||
# _LOG_FORMAT = ' > [%(levelname)s:%(filename)s] - %(message)s'
|
||||
_LOG_FORMAT = '%(message)s'
|
||||
|
|
@ -169,6 +276,11 @@ def config_log(argsp):
|
|||
|
||||
|
||||
def parse_args(argvp):
|
||||
"""
|
||||
Parse the arguments passed to the program
|
||||
:param argvp: The arguments passed to the program
|
||||
:return: argsp: The arguments, parsed as an object
|
||||
"""
|
||||
if len(argvp) == 1:
|
||||
sys.argv.append('--help')
|
||||
|
||||
|
|
@ -202,8 +314,8 @@ def parse_args(argvp):
|
|||
|
||||
def main(argvp):
|
||||
"""
|
||||
Main function
|
||||
:return:
|
||||
Main function, called by __main__
|
||||
:return: None
|
||||
"""
|
||||
# Parse arguments
|
||||
args = parse_args(argvp)
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import logging
|
|||
import importlib
|
||||
import pytest
|
||||
from criteriaSorter.modules import criteriaSorter
|
||||
import tempfile
|
||||
|
||||
|
||||
_ARGS_LIST = [
|
||||
|
|
@ -43,6 +44,12 @@ _GOOD_RESULT = """[MOVED] path/to/destination1 to path/to/file1
|
|||
"""
|
||||
|
||||
|
||||
def cldl(*l_file):
|
||||
for f in l_file:
|
||||
f.close()
|
||||
os.remove(f.name)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("args, outp", _EXIT_ARGS_LIST)
|
||||
def test_parse_args_exit(args, outp, capsys):
|
||||
args = args.split(" ")
|
||||
|
|
@ -102,16 +109,18 @@ class falseArg:
|
|||
(io.StringIO(_GOOD_FILE_LIST), _GOOD_RESULT), (io.StringIO(_JUNK_FILE_LIST), _JUNK_RESULT)
|
||||
])
|
||||
def test_action_cancel(monkeypatch, list_of_cancels, result_expected, capsys):
|
||||
cancel_file = tempfile.NamedTemporaryFile(mode="w+", suffix=".txt", delete=False)
|
||||
monkeypatch.setattr(os, "rename", move)
|
||||
monkeypatch.setattr(os, "remove", move)
|
||||
# monkeypatch.setattr(os, "remove", move)
|
||||
monkeypatch.setattr(os, "rmdir", move)
|
||||
monkeypatch.setattr(os.path, "join", lambda a: a)
|
||||
|
||||
with open("cancelfile.txt", "w") as f:
|
||||
with open(cancel_file.name, "w") as f:
|
||||
f.write(list_of_cancels.read())
|
||||
criteriaSorter.action_cancel(falseArg("cancelfile.txt"))
|
||||
criteriaSorter.action_cancel(falseArg(cancel_file.name))
|
||||
assert result_expected in capsys.readouterr().out
|
||||
|
||||
cldl(cancel_file)
|
||||
|
||||
@pytest.mark.parametrize("verbose", [0, 1, 2, 3, 4])
|
||||
def test_action_list(verbose, capsys):
|
||||
|
|
@ -129,8 +138,8 @@ def test_action_list(verbose, capsys):
|
|||
[["--config=unexisting_conf", "list"], "Config unexisting_conf file not found"],
|
||||
])
|
||||
def test_action_list_error(args, error, caplog, monkeypatch):
|
||||
|
||||
with open("noconf.yaml", "w") as f:
|
||||
no_conf = tempfile.NamedTemporaryFile(mode="w+t", delete=False)
|
||||
with open(no_conf.name, "w") as f:
|
||||
f.write("Bad config =:/\n+3")
|
||||
|
||||
args = criteriaSorter.parse_args(args)
|
||||
|
|
@ -138,3 +147,59 @@ def test_action_list_error(args, error, caplog, monkeypatch):
|
|||
with caplog.at_level(logging.ERROR):
|
||||
criteriaSorter.action_list(args)
|
||||
assert error in caplog.text
|
||||
cldl(no_conf)
|
||||
|
||||
def test_load_config():
|
||||
good_conf = tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False)
|
||||
with open(good_conf.name, "w") as f:
|
||||
f.write("""test:
|
||||
- path: path/to/file1
|
||||
""")
|
||||
|
||||
assert criteriaSorter.load_config(good_conf.name) == {"test": [{"path": "path/to/file1"}]}
|
||||
cldl(good_conf)
|
||||
|
||||
|
||||
def test_load_config_error():
|
||||
bad_conf = tempfile.NamedTemporaryFile(suffix=".yaml", delete=False)
|
||||
good_conf = tempfile.NamedTemporaryFile(suffix=".yaml", delete=False)
|
||||
|
||||
with open(bad_conf.name, "w") as f:
|
||||
f.write("""test:
|
||||
- path: path/to/file1
|
||||
condition:
|
||||
- path: path/to/file1
|
||||
""")
|
||||
|
||||
with open(good_conf.name, "w") as f:
|
||||
f.write("""operations:
|
||||
sort_junk_folder:
|
||||
operation_order: |
|
||||
operation1
|
||||
operation2
|
||||
operation3
|
||||
operation4
|
||||
operation1:
|
||||
conditions : |
|
||||
has_artist
|
||||
is_image
|
||||
destination : Artists/{obj.artist}/{obj.name}
|
||||
operation2:
|
||||
conditions : |
|
||||
is_video
|
||||
destination : vids/{obj.name}
|
||||
operation3:
|
||||
conditions : |
|
||||
is_audio
|
||||
destination : audios/{obj.name}
|
||||
operation4:
|
||||
conditions : |
|
||||
is_document
|
||||
destination : others/{obj.name}
|
||||
""")
|
||||
|
||||
with pytest.raises(SystemExit):
|
||||
conf = criteriaSorter.load_config(bad_conf.name)
|
||||
criteriaSorter.load_operations("default_operation", conf)
|
||||
|
||||
cldl(good_conf, bad_conf)
|
||||
Loading…
Reference in New Issue