diff --git a/discordHelperWin.py b/discordHelperWin.py index 1b9a7d7..051b82e 100644 --- a/discordHelperWin.py +++ b/discordHelperWin.py @@ -25,7 +25,7 @@ if __name__ == "__main__": # print(" > No user database was found, the bundled one will be used.") run_db = os.path.join(directory_path, "discordHelper_last_run.json") - argv = f"--all --autodetect --launch --continue --timeout 10 {db_path}--gen-data {run_db}.yaml".split(" ") + argv = f"--all --autodetect --launch --continue --timeout 1 {db_path}--gen-data {run_db}.yaml".split(" ") # print(" > Starting the discordHelper with the following arguments:") # print(" > " + " ".join(argv)) diff --git a/modules/argument_parser.py b/modules/argument_parser.py index 9f477f8..9faed5d 100644 --- a/modules/argument_parser.py +++ b/modules/argument_parser.py @@ -25,6 +25,8 @@ def parse(_args): parser.add_argument("--only-known-paths", "--fast", action='store_true', help="Only run the check on the known paths") parser.add_argument("--gen-data", "--generate-data", "--gen", nargs=1, default="", help="Generate data for the analysis") parser.add_argument("--database", "--db", nargs=1, default="", help="Alternative database for the analysis") + parser.add_argument("--pollrate", "--poll", default=0.1, type=float, help="Refresh rate of the status, 0.1 is the default") + parser.add_argument("--printrate", "-refresh", default=5, type=int, help="Print the tests every X seconds") # Tests parser.add_argument('--spidey', action='store_true', help='Runs the spidey test') diff --git a/modules/main_runner.py b/modules/main_runner.py index 7ccefcc..13cdbf7 100644 --- a/modules/main_runner.py +++ b/modules/main_runner.py @@ -5,6 +5,8 @@ import importlib import threading import time from collections import deque +from datetime import datetime + from modules.internal_io import global_status # , test_status, return_code_dict """ @@ -22,10 +24,11 @@ _TEST_MODULES = { } PROGRAM_VERSION = "1.0.0" -POST_RUN_ALLOWED = False +POST_RUN_ALLOWED = False # TODO : Remove this line, only use is_infected RUN_OVER = False queue_lock = threading.Lock() +processes_lock = threading.Lock() CLEAR_CODE = colored(r""" _______ _ @@ -80,6 +83,7 @@ class TestRunner: self.loaded_test_modules = {} self.loaded_test_classes = {} self.list_of_tests = [] + self.dict_of_processes = {} # To get a more granular view of the processes self.update_module_list() self.finished_tests = deque() @@ -117,7 +121,15 @@ class TestRunner: self.list_of_tests = [] for module_name, test_class in self.loaded_test_classes.items(): print(colored(f" > Starting test {module_name}...", "blue")) - tc = test_class(self.args, self.agnostic_path, None, self.finished_tests, queue_lock) + tc = test_class( + self.args, + self.agnostic_path, + None, + self.finished_tests, + queue_lock, + self.dict_of_processes, + processes_lock + ) tc = threading.Thread(target=tc.run_test, args=()) tc.name = str(module_name) tc.daemon = True @@ -158,6 +170,7 @@ class TestRunner: RUN_OVER = True return except KeyboardInterrupt: + RUN_OVER = True return except: sys.exit() @@ -197,26 +210,24 @@ class TestRunner: :return: The exit code :rtype: int """ - exit_code = -1 status = self.get_status() if status.tests_finished == status.tests_total: if status.tests_failed != 0: - exit_code = 1 + exit_code = 1 # There is one failure, so the test failed elif status.tests_success == status.tests_total: - exit_code = 0 + exit_code = 0 # All tests passed else: - exit_code = -1 + # print("Something went wrong, please report this bug.") + # print(status) + exit_code = 2 # Something went wrong else: - exit_code = 2 + global RUN_OVER + if RUN_OVER: + print(colored("\n > Tests aborted, please check the logs.", "red")) + exit_code = 3 + else: + exit_code = -1 # The tests are not finished return exit_code - # global POST_RUN_ALLOWED - # if POST_RUN_ALLOWED: - # for t in self.list_of_tests: - # if t.is_alive(): - # return 2 # 2 means that the user stopped the tests before the end - # return self.is_infected # The user didn't stop the tests - # - # return -1 # The program is not finished yet def get_status(self): """ @@ -240,8 +251,13 @@ class TestRunner: nb_tests_skipped += 1 elif test.status == "problem": nb_tests_failure += 1 - progress += test.progress + # progress += test.progress queue_lock.release() + + for test_name, test in self.dict_of_processes.items(): + progress += test.progress + progress = int(progress / nb_tests) + nb_tests_ran = nb_tests_success + nb_tests_failure + nb_tests_skipped nb_test_running = nb_tests - nb_tests_ran return global_status( @@ -251,7 +267,8 @@ class TestRunner: tests_failed=nb_tests_failure, tests_running=nb_test_running, tests_skipped=nb_tests_skipped, - tests_error=nb_tests_failure + tests_error=nb_tests_failure, + progress=progress ) def get_test_data(self): @@ -268,7 +285,6 @@ class TestRunner: data[key] = value else: data[key].update(value) - # data.update(test.data) queue_lock.release() return data @@ -294,11 +310,13 @@ def export_data(test_runner, data): return main_data -def print_status(status): +def print_status(status, timer_start): """ Prints the status :param status: The status :type status: internal_io.global_status + :param timer_start: The start time of the tests + :type timer_start: float """ if status.tests_failed != 0: color = "red" @@ -316,6 +334,19 @@ def print_status(status): f"{status.tests_failed} failed, " f"{status.tests_error} problem", color )) + if status.tests_running != 0: + ellapsed_time = time.time() - timer_start + if status.progress == 0: + estimated_time = "∞" + else: + mon, sec = divmod(ellapsed_time * 100 / status.progress, 60) + hr, mon = divmod(mon, 60) + estimated_time = "%d:%02d:%02d" % (hr, mon, sec) + print(colored( + f" > Overall progress: {status.progress}%" + f" in {time.time() - timer_start:.2f} seconds" + f" estimaded time ({estimated_time})", color + )) def run_check(args, agnpath): @@ -328,12 +359,17 @@ def run_check(args, agnpath): print(LOGO) start = time.time() tr = TestRunner(args, agnpath, start_time=start) # All operation are executed in parallel and the user can stop them - # print("Waiting for the end of the tests") + + POLLRATE, PRINT_RATE, COUNTER = args.pollrate, args.printrate, 0 + COUNTER_MAX = int(PRINT_RATE / POLLRATE) + while tr.get_exit_code() == -1: # print("Waiting for the end of the tests") - print_status(tr.get_status()) # TODO : Print status in a more readable way - time.sleep(0.5) - time.sleep(0.5) # TODO : Remove this line + if (COUNTER % COUNTER_MAX) == 0: + print_status(tr.get_status(), start) # TODO : Print status in a more readable way + time.sleep(POLLRATE) + COUNTER += 1 + global RUN_OVER RUN_OVER = True @@ -348,10 +384,10 @@ def run_check(args, agnpath): # tr.is_infected = False print(colored(f"\nThe test sequence is now finished, the program ran for {round(time.time() - start,2)} seconds", color)) - print_status(status) # TODO: Print the final status, test still running are actually skipped + print_status(status, start) # TODO: Print the final status, test still running are actually skipped if args.gen_data: - print(colored(" > Generating data...", "blue")) + print(colored("\n > Generating data...", "blue")) data = tr.get_test_data() file_path = os.path.join(os.path.join("databases", args.gen_data[0])) folder = os.path.dirname(file_path) @@ -365,6 +401,6 @@ def run_check(args, agnpath): json.dump(main_data, f, indent=4) msg_error = "sucessfully" if status.tests_failed == 0 else f"with {status.tests_error} errors" - print(colored(f"The program executed {msg_error}, exiting in {args.timeout} seconds...", color)) + print(colored(f"\nThe program executed {msg_error}, exiting in {args.timeout} seconds...", color)) time.sleep(args.timeout) sys.exit(tr.get_exit_code()) diff --git a/modules/spidey_test.py b/modules/spidey_test.py index b37c77c..b3fa3f8 100644 --- a/modules/spidey_test.py +++ b/modules/spidey_test.py @@ -1,14 +1,16 @@ import asyncio import os +import time + from modules import test_template class SpideyTest(test_template.TestTemplate): - def __init__(self, args, agnpath, test_data, queue, queue_lock): + def __init__(self, args, agnpath, test_data, queue, queue_lock, dict_process, dict_process_lock): """ Simple test runner, for testing the detection of Spidey viruses. """ - super().__init__(args, agnpath, test_data, queue, queue_lock) + super().__init__(args, agnpath, test_data, queue, queue_lock, dict_process, dict_process_lock) def run_test(self): """ @@ -18,7 +20,7 @@ class SpideyTest(test_template.TestTemplate): :rtype: bool """ self.set_status("running") - # self.slowdown() # This function is only used for testing + paths = self.get_path() self.is_infected = False lines = 0 @@ -28,7 +30,10 @@ class SpideyTest(test_template.TestTemplate): if lines > 1: self.is_infected = True + self.progress = 50 + # time.sleep(5) lines = 0 + with open(os.path.join(paths[1], "index.js")) as f: for line in f: lines += 1 # TODO: you don't actually need to count the lines, you can just check if it's more than 1 @@ -43,6 +48,7 @@ class SpideyTest(test_template.TestTemplate): "If you've recently downloaded a .exe directly sent from someone, " "do also consider a clean windows reinstall.") + self.progress = 100 return self.finish("success") def get_path(self): diff --git a/modules/test_template.py b/modules/test_template.py index d3db84f..f45696f 100644 --- a/modules/test_template.py +++ b/modules/test_template.py @@ -13,7 +13,7 @@ class TestDataError(Exception): class TestTemplate: - def __init__(self, args, agnpath, test_data, queue, queue_lock): + def __init__(self, args, agnpath, test_data, queue, queue_lock, dict_process, dict_process_lock): """ Simple test template. :param args: List of arguments @@ -26,6 +26,10 @@ class TestTemplate: :type queue: queue.deque :param queue_lock: The lock to use when putting the test status in the queue :type queue_lock: threading.Lock + :param dict_process: The dictionary of processes + :type dict_process: dict + :param dict_process_lock: The lock to use when putting the process in the dictionary + :type dict_process_lock: threading.Lock """ self.args = args self.ptb = args.ptb @@ -42,6 +46,11 @@ class TestTemplate: self.failure_dict = {} self.new_data = {} self.progress = 0 + self.dict_process = dict_process + self.dict_process_lock = dict_process_lock + self.dict_process_lock.acquire() + self.dict_process[self.name()] = self + self.dict_process_lock.release() def run_test(self): """ @@ -71,12 +80,12 @@ class TestTemplate: :rtype: internal_io.test_status """ return internal_io.test_status( - self.name(), - self.status_code, - self.status, - self.failure_dict, - self.new_data, - self.progress + name=self.name(), + status=self.status_code, + message=self.status, + failure_dict=self.failure_dict, + data=self.new_data, + progress=self.progress, ) @staticmethod @@ -139,7 +148,7 @@ class TestTemplate: class TestWalkTemplate(TestTemplate): - def __init__(self, args, agnpath, test_data, queue, queue_lock): + def __init__(self, args, agnpath, test_data, queue, queue_lock, dict_process, dict_process_lock): """ Simple test template. :param args: List of arguments @@ -149,7 +158,7 @@ class TestWalkTemplate(TestTemplate): :param test_data: The data to be tested :type test_data: str """ - super().__init__(args, agnpath, test_data, queue, queue_lock) + super().__init__(args, agnpath, test_data, queue, queue_lock, dict_process, dict_process_lock) if test_data is None: self.set_status("problem", "No test data given.")