From f305dfcc04dde83c3f987dc9d4dfe296a0affc21 Mon Sep 17 00:00:00 2001 From: Maximilien Rigaut Date: Thu, 23 Jun 2022 17:38:05 +0200 Subject: [PATCH 1/2] Walking implemnetation --- database/windows_base.yml | 19 +++++++++++++ modules/agnostic_paths.py | 11 ++++++++ modules/argument_parser.py | 1 + modules/internal_io.py | 4 +-- modules/test_interface.py | 6 +++- modules/test_template.py | 56 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 94 insertions(+), 3 deletions(-) create mode 100644 database/windows_base.yml diff --git a/database/windows_base.yml b/database/windows_base.yml new file mode 100644 index 0000000..1d79ee1 --- /dev/null +++ b/database/windows_base.yml @@ -0,0 +1,19 @@ +global: + name: Windows Base + version: 1.0 + + +files: + /etc/passwd: + sha256: "6c3d65f8b8b8f5a8f62b60c0e15d5d084d3b20f470f5d8b3d26bfb3c350d37a3" + type: file + size: "1024" + permissions: "0644" + owner: "root" + group: "root" + date: "2018-01-01T00:00:00Z" + executable: "true" + symlink: "false" + md5: "6c3d65f8b8b8f5a8f62b60c0e15d5d084d3b20f470f5d8b3d26bfb3c350d37a3" + sha1: "6c3d65f8b8b8f5a8f62b60c0e15d5d084d3b20f470f5d8b3d26bfb3c350d37a3" + diff --git a/modules/agnostic_paths.py b/modules/agnostic_paths.py index ba0e728..f1f92d0 100644 --- a/modules/agnostic_paths.py +++ b/modules/agnostic_paths.py @@ -144,6 +144,17 @@ class AgnosticPaths: :rtype: str """ return self.main_path + + def walk_all_files(self, *args): + """ + Walks all files in the given path + :param args: The path + :return: A list of all files in the given path + :rtype: list[str] + """ + return [os.path.join(dirpath, filename) + for dirpath, dirnames, filenames in os.walk(self(*args)) + for filename in filenames] def __repr__(self): """ diff --git a/modules/argument_parser.py b/modules/argument_parser.py index 39fe8f0..730d167 100644 --- a/modules/argument_parser.py +++ b/modules/argument_parser.py @@ -30,5 +30,6 @@ def parse(_args): parser.add_argument("--gen-data-path", "--generate-data-path", "--gen-path", nargs=1, default="", help="Path to generate the data in") parser.add_argument("--database", "--db", nargs=1, default="", help="Alternative database for the analysis") parser.add_argument("--continue", action="store_true", help="Continue the analysis in case of failure") + parser.add_argument("--test-walk", action="store_true", help="Test walk function") return parser.parse_args(_args) diff --git a/modules/internal_io.py b/modules/internal_io.py index 8804dbb..af1784d 100644 --- a/modules/internal_io.py +++ b/modules/internal_io.py @@ -7,8 +7,8 @@ from collections import namedtuple global_status = namedtuple( 'global_status', - 'total_tests, tests_finished, test_success, tests_failed, tests_running, tests_skipped', - defaults=(0, 0, 0, 0, 0, 0) + 'total_tests, tests_finished, test_success, tests_failed, tests_running, tests_skipped, test_error', + defaults=(0, 0, 0, 0, 0, 0, 0) ) test_status = namedtuple('test_status', 'name, status, message', defaults=(None, None, None)) return_code_dict = { diff --git a/modules/test_interface.py b/modules/test_interface.py index 4b1a9f9..3c27eaf 100644 --- a/modules/test_interface.py +++ b/modules/test_interface.py @@ -13,6 +13,7 @@ argparse argument : [module_name, class_name] """ _TEST_MODULES = { "spidey": ["spidey_test", "SpideyTest"], + "test_walk": ["test_template", "TestWalkTemplate"], } POST_RUN_ALLOWED = False @@ -61,8 +62,9 @@ class TestRunner: modules[values[0]] = importlib.import_module(f"modules.{values[0]}") classes[values[0]] = getattr(modules[values[0]], values[1]) - except AttributeError: + except AttributeError as e: print(f"The module {module_name} was not loaded because it was not specified in the config.") + print(e) self.loaded_test_modules.update(modules) self.loaded_test_classes.update(classes) @@ -173,6 +175,8 @@ class TestRunner: nb_tests_failure += 1 elif test.status == "skipped": nb_tests_skipped += 1 + elif test.status == "problem": + nb_tests_failure += 1 nb_tests_ran = nb_tests_success + nb_tests_failure + nb_tests_skipped nb_test_running = nb_tests - nb_tests_ran diff --git a/modules/test_template.py b/modules/test_template.py index 9d10d92..748e43b 100644 --- a/modules/test_template.py +++ b/modules/test_template.py @@ -81,3 +81,59 @@ class TestTemplate: else: self.status = self.get_status_from_code(statuscode) return self.status_code + + +class TestWalkTemplate(TestTemplate): + def __init__(self, args, agnpath, test_data, queue): + """ + Simple test template. + :param args: List of arguments + :type args: argparse.Namespace + :param agnpath: The path to the discord folder + :type agnpath: agnostic_path.AgnosticPath + :param test_data: The data to be tested + :type test_data: str + """ + super().__init__(args, agnpath, test_data, queue) + if test_data is None: + self.set_status("problem", "No test data given.") + + def run_test(self): + if self.test_data is None: + self.set_status("problem", "No test data given.") + return self.get_status() + self.set_status("running", "The test will begin shortly, please wait...") + for path in self.walk(): + self.compare(path, "test", lambda x: True, [], {}) + + def walk(self): + """ + Walks the given path and returns a list of all files and directories. + :return: List of all files and directories + :return: + """ + return self.agnostic_path.walk_all_files(self.agnostic_path()) + + def compare(self, path, entry_name, function, args, kwargs): + """ + Use the function on the given path, with given arguments and compare it with the entry in the test data. + :param path: Path to the file or directory + :type path: agnostic_path.AgnosticPath + :param entry_name: Name of the entry in the test data + :type entry_name: str + :param function: Function to use on the path + :type function: function + :param args: Arguments to use on the function + :type args: list + :param kwargs: Keyword arguments to use on the function + :type kwargs: dict + """ + if path not in self.test_data["files"]: + raise ValueError("No entry in test data for path: " + str(path)) + if entry_name not in self.test_data["files"][path]: + raise ValueError("No entry in test data for path: " + str(path) + " and entry name: " + str(entry_name)) + data_origin = self.test_data["files"][path][entry_name] + data_result = function(path, *args, **kwargs) + if data_origin != data_result: + return False + return True From a1b44022addf484bd7134565fb5248e69e2a4ec6 Mon Sep 17 00:00:00 2001 From: Maximilien Rigaut Date: Thu, 23 Jun 2022 18:16:50 +0200 Subject: [PATCH 2/2] Threading and lock improvement --- modules/spidey_test.py | 17 ++--------- modules/test_interface.py | 6 ++-- modules/test_template.py | 60 +++++++++++++++++++++++++++++++-------- 3 files changed, 55 insertions(+), 28 deletions(-) diff --git a/modules/spidey_test.py b/modules/spidey_test.py index 53212ea..f06b058 100644 --- a/modules/spidey_test.py +++ b/modules/spidey_test.py @@ -4,19 +4,11 @@ from modules import test_template class SpideyTest(test_template.TestTemplate): - def __init__(self, args, agnpath, test_data, queue): + def __init__(self, args, agnpath, test_data, queue, queue_lock): """ Simple test runner, for testing the detection of Spidey viruses. - :param args: List of arguments - :type args: argparse.Namespace - :param agnpath: The path to the discord folder - :type agnpath: agnostic_path.AgnosticPath - :param test_data: The data to be tested - :type test_data: str - :param queue: The queue to put the status in - :type queue: queue.Queue """ - super().__init__(args, agnpath, test_data, queue) + super().__init__(args, agnpath, test_data, queue, queue_lock) def slowdown(self): """ @@ -60,10 +52,7 @@ class SpideyTest(test_template.TestTemplate): "do also consider a clean windows reinstall.") print("Spidey test finished") - self.set_status("success") - status = self.get_status() - self.queue.append(status) - return self.get_status_code() + self.finish("success") def get_path(self): """ diff --git a/modules/test_interface.py b/modules/test_interface.py index 3c27eaf..dca8d48 100644 --- a/modules/test_interface.py +++ b/modules/test_interface.py @@ -16,6 +16,7 @@ _TEST_MODULES = { "test_walk": ["test_template", "TestWalkTemplate"], } POST_RUN_ALLOWED = False +queue_lock = threading.Lock() class TestRunner: @@ -79,7 +80,7 @@ class TestRunner: for module_name, test_class in self.loaded_test_classes.items(): for i in range(10): print(f"Starting test {module_name}") - tc = test_class(self.args, self.agnostic_path, None, self.finished_tests) + tc = test_class(self.args, self.agnostic_path, None, self.finished_tests, queue_lock) tc = threading.Thread(target=tc.run_test, args=()) tc.name = str(module_name) tc.daemon = True @@ -163,6 +164,7 @@ class TestRunner: :return: The status :rtype: str """ + queue_lock.acquire() nb_tests = len(self.list_of_tests) nb_tests_success = 0 nb_tests_failure = 0 @@ -177,7 +179,7 @@ class TestRunner: nb_tests_skipped += 1 elif test.status == "problem": nb_tests_failure += 1 - + queue_lock.release() nb_tests_ran = nb_tests_success + nb_tests_failure + nb_tests_skipped nb_test_running = nb_tests - nb_tests_ran return global_status( diff --git a/modules/test_template.py b/modules/test_template.py index 748e43b..9d80065 100644 --- a/modules/test_template.py +++ b/modules/test_template.py @@ -4,9 +4,14 @@ A global template for all tests. from modules import internal_io +class TestDataError(Exception): + """ + Raised when the data is not valid. + """ + pass class TestTemplate: - def __init__(self, args, agnpath, test_data, queue): + def __init__(self, args, agnpath, test_data, queue, queue_lock): """ Simple test template. :param args: List of arguments @@ -15,6 +20,10 @@ class TestTemplate: :type agnpath: agnostic_path.AgnosticPath :param test_data: The data to be tested :type test_data: str + :param queue: The queue to put the test status in + :type queue: queue.deque + :param queue_lock: The lock to use when putting the test status in the queue + :type queue_lock: threading.Lock """ self.ptb = args.ptb self.agnostic_path = agnpath @@ -26,6 +35,7 @@ class TestTemplate: self.status = "Test not yet initialized." self.queue = queue self.set_status("idle") + self.queue_lock = queue_lock def run_test(self): """ @@ -82,9 +92,23 @@ class TestTemplate: self.status = self.get_status_from_code(statuscode) return self.status_code + def finish(self, statuscode: str = None, message: str = None): + """ + Finishes the test. + """ + if message is not None: + self.status = message + if statuscode is not None: + self.status_code = statuscode + + self.queue_lock.acquire() + self.queue.append(self.get_status()) + self.queue_lock.release() + return self.get_status_code() + class TestWalkTemplate(TestTemplate): - def __init__(self, args, agnpath, test_data, queue): + def __init__(self, args, agnpath, test_data, queue, queue_lock): """ Simple test template. :param args: List of arguments @@ -94,17 +118,22 @@ class TestWalkTemplate(TestTemplate): :param test_data: The data to be tested :type test_data: str """ - super().__init__(args, agnpath, test_data, queue) + super().__init__(args, agnpath, test_data, queue, queue_lock) + self.data = {} if test_data is None: self.set_status("problem", "No test data given.") def run_test(self): + + self.finish() + return if self.test_data is None: self.set_status("problem", "No test data given.") return self.get_status() self.set_status("running", "The test will begin shortly, please wait...") for path in self.walk(): self.compare(path, "test", lambda x: True, [], {}) + self.set_status("sucess", "The test was a success but kinda not a success.") def walk(self): """ @@ -128,12 +157,19 @@ class TestWalkTemplate(TestTemplate): :param kwargs: Keyword arguments to use on the function :type kwargs: dict """ - if path not in self.test_data["files"]: - raise ValueError("No entry in test data for path: " + str(path)) - if entry_name not in self.test_data["files"][path]: - raise ValueError("No entry in test data for path: " + str(path) + " and entry name: " + str(entry_name)) - data_origin = self.test_data["files"][path][entry_name] - data_result = function(path, *args, **kwargs) - if data_origin != data_result: - return False - return True + + try: + self.data[path] = function(path, *args, **kwargs) + if path not in self.test_data["files"]: + raise TestDataError("No entry in test data for path: " + str(path)) + if entry_name not in self.test_data["files"][path]: + raise TestDataError("No entry in test data for path: " + str(path) + " and entry name: " + str(entry_name)) + data_origin = self.test_data["files"][path][entry_name] + self.data[path] = function(path, *args, **kwargs) + if data_origin != self.data[path]: + return False + return True + except TestDataError as e: + if not self.args.continue_on_error: + self.set_status("error", "Error: " + str(e)) + raise e