Mostly printing issues

This commit is contained in:
Maximilien Rigaut 2022-06-25 02:18:16 +02:00
parent 25159ac83c
commit b7fbc2ae32
No known key found for this signature in database
GPG Key ID: 4F14B23709ADA5B1
5 changed files with 92 additions and 39 deletions

View File

@ -25,7 +25,7 @@ if __name__ == "__main__":
# print(" > No user database was found, the bundled one will be used.")
run_db = os.path.join(directory_path, "discordHelper_last_run.json")
argv = f"--all --autodetect --launch --continue --timeout 10 {db_path}--gen-data {run_db}.yaml".split(" ")
argv = f"--all --autodetect --launch --continue --timeout 1 {db_path}--gen-data {run_db}.yaml".split(" ")
# print(" > Starting the discordHelper with the following arguments:")
# print(" > " + " ".join(argv))

View File

@ -25,6 +25,8 @@ def parse(_args):
parser.add_argument("--only-known-paths", "--fast", action='store_true', help="Only run the check on the known paths")
parser.add_argument("--gen-data", "--generate-data", "--gen", nargs=1, default="", help="Generate data for the analysis")
parser.add_argument("--database", "--db", nargs=1, default="", help="Alternative database for the analysis")
parser.add_argument("--pollrate", "--poll", default=0.1, type=float, help="Refresh rate of the status, 0.1 is the default")
parser.add_argument("--printrate", "-refresh", default=5, type=int, help="Print the tests every X seconds")
# Tests
parser.add_argument('--spidey', action='store_true', help='Runs the spidey test')

View File

@ -5,6 +5,8 @@ import importlib
import threading
import time
from collections import deque
from datetime import datetime
from modules.internal_io import global_status # , test_status, return_code_dict
"""
@ -22,10 +24,11 @@ _TEST_MODULES = {
}
PROGRAM_VERSION = "1.0.0"
POST_RUN_ALLOWED = False
POST_RUN_ALLOWED = False # TODO : Remove this line, only use is_infected
RUN_OVER = False
queue_lock = threading.Lock()
processes_lock = threading.Lock()
CLEAR_CODE = colored(r"""
_______ _
@ -80,6 +83,7 @@ class TestRunner:
self.loaded_test_modules = {}
self.loaded_test_classes = {}
self.list_of_tests = []
self.dict_of_processes = {} # To get a more granular view of the processes
self.update_module_list()
self.finished_tests = deque()
@ -117,7 +121,15 @@ class TestRunner:
self.list_of_tests = []
for module_name, test_class in self.loaded_test_classes.items():
print(colored(f" > Starting test {module_name}...", "blue"))
tc = test_class(self.args, self.agnostic_path, None, self.finished_tests, queue_lock)
tc = test_class(
self.args,
self.agnostic_path,
None,
self.finished_tests,
queue_lock,
self.dict_of_processes,
processes_lock
)
tc = threading.Thread(target=tc.run_test, args=())
tc.name = str(module_name)
tc.daemon = True
@ -158,6 +170,7 @@ class TestRunner:
RUN_OVER = True
return
except KeyboardInterrupt:
RUN_OVER = True
return
except:
sys.exit()
@ -197,26 +210,24 @@ class TestRunner:
:return: The exit code
:rtype: int
"""
exit_code = -1
status = self.get_status()
if status.tests_finished == status.tests_total:
if status.tests_failed != 0:
exit_code = 1
exit_code = 1 # There is one failure, so the test failed
elif status.tests_success == status.tests_total:
exit_code = 0
exit_code = 0 # All tests passed
else:
exit_code = -1
# print("Something went wrong, please report this bug.")
# print(status)
exit_code = 2 # Something went wrong
else:
exit_code = 2
global RUN_OVER
if RUN_OVER:
print(colored("\n > Tests aborted, please check the logs.", "red"))
exit_code = 3
else:
exit_code = -1 # The tests are not finished
return exit_code
# global POST_RUN_ALLOWED
# if POST_RUN_ALLOWED:
# for t in self.list_of_tests:
# if t.is_alive():
# return 2 # 2 means that the user stopped the tests before the end
# return self.is_infected # The user didn't stop the tests
#
# return -1 # The program is not finished yet
def get_status(self):
"""
@ -240,8 +251,13 @@ class TestRunner:
nb_tests_skipped += 1
elif test.status == "problem":
nb_tests_failure += 1
progress += test.progress
# progress += test.progress
queue_lock.release()
for test_name, test in self.dict_of_processes.items():
progress += test.progress
progress = int(progress / nb_tests)
nb_tests_ran = nb_tests_success + nb_tests_failure + nb_tests_skipped
nb_test_running = nb_tests - nb_tests_ran
return global_status(
@ -251,7 +267,8 @@ class TestRunner:
tests_failed=nb_tests_failure,
tests_running=nb_test_running,
tests_skipped=nb_tests_skipped,
tests_error=nb_tests_failure
tests_error=nb_tests_failure,
progress=progress
)
def get_test_data(self):
@ -268,7 +285,6 @@ class TestRunner:
data[key] = value
else:
data[key].update(value)
# data.update(test.data)
queue_lock.release()
return data
@ -294,11 +310,13 @@ def export_data(test_runner, data):
return main_data
def print_status(status):
def print_status(status, timer_start):
"""
Prints the status
:param status: The status
:type status: internal_io.global_status
:param timer_start: The start time of the tests
:type timer_start: float
"""
if status.tests_failed != 0:
color = "red"
@ -316,6 +334,19 @@ def print_status(status):
f"{status.tests_failed} failed, "
f"{status.tests_error} problem", color
))
if status.tests_running != 0:
ellapsed_time = time.time() - timer_start
if status.progress == 0:
estimated_time = "∞"
else:
mon, sec = divmod(ellapsed_time * 100 / status.progress, 60)
hr, mon = divmod(mon, 60)
estimated_time = "%d:%02d:%02d" % (hr, mon, sec)
print(colored(
f" > Overall progress: {status.progress}%"
f" in {time.time() - timer_start:.2f} seconds"
f" estimaded time ({estimated_time})", color
))
def run_check(args, agnpath):
@ -328,12 +359,17 @@ def run_check(args, agnpath):
print(LOGO)
start = time.time()
tr = TestRunner(args, agnpath, start_time=start) # All operation are executed in parallel and the user can stop them
# print("Waiting for the end of the tests")
POLLRATE, PRINT_RATE, COUNTER = args.pollrate, args.printrate, 0
COUNTER_MAX = int(PRINT_RATE / POLLRATE)
while tr.get_exit_code() == -1:
# print("Waiting for the end of the tests")
print_status(tr.get_status()) # TODO : Print status in a more readable way
time.sleep(0.5)
time.sleep(0.5) # TODO : Remove this line
if (COUNTER % COUNTER_MAX) == 0:
print_status(tr.get_status(), start) # TODO : Print status in a more readable way
time.sleep(POLLRATE)
COUNTER += 1
global RUN_OVER
RUN_OVER = True
@ -348,10 +384,10 @@ def run_check(args, agnpath):
# tr.is_infected = False
print(colored(f"\nThe test sequence is now finished, the program ran for {round(time.time() - start,2)} seconds", color))
print_status(status) # TODO: Print the final status, test still running are actually skipped
print_status(status, start) # TODO: Print the final status, test still running are actually skipped
if args.gen_data:
print(colored(" > Generating data...", "blue"))
print(colored("\n > Generating data...", "blue"))
data = tr.get_test_data()
file_path = os.path.join(os.path.join("databases", args.gen_data[0]))
folder = os.path.dirname(file_path)
@ -365,6 +401,6 @@ def run_check(args, agnpath):
json.dump(main_data, f, indent=4)
msg_error = "sucessfully" if status.tests_failed == 0 else f"with {status.tests_error} errors"
print(colored(f"The program executed {msg_error}, exiting in {args.timeout} seconds...", color))
print(colored(f"\nThe program executed {msg_error}, exiting in {args.timeout} seconds...", color))
time.sleep(args.timeout)
sys.exit(tr.get_exit_code())

View File

@ -1,14 +1,16 @@
import asyncio
import os
import time
from modules import test_template
class SpideyTest(test_template.TestTemplate):
def __init__(self, args, agnpath, test_data, queue, queue_lock):
def __init__(self, args, agnpath, test_data, queue, queue_lock, dict_process, dict_process_lock):
"""
Simple test runner, for testing the detection of Spidey viruses.
"""
super().__init__(args, agnpath, test_data, queue, queue_lock)
super().__init__(args, agnpath, test_data, queue, queue_lock, dict_process, dict_process_lock)
def run_test(self):
"""
@ -18,7 +20,7 @@ class SpideyTest(test_template.TestTemplate):
:rtype: bool
"""
self.set_status("running")
# self.slowdown() # This function is only used for testing
paths = self.get_path()
self.is_infected = False
lines = 0
@ -28,7 +30,10 @@ class SpideyTest(test_template.TestTemplate):
if lines > 1:
self.is_infected = True
self.progress = 50
# time.sleep(5)
lines = 0
with open(os.path.join(paths[1], "index.js")) as f:
for line in f:
lines += 1 # TODO: you don't actually need to count the lines, you can just check if it's more than 1
@ -43,6 +48,7 @@ class SpideyTest(test_template.TestTemplate):
"If you've recently downloaded a .exe directly sent from someone, "
"do also consider a clean windows reinstall.")
self.progress = 100
return self.finish("success")
def get_path(self):

View File

@ -13,7 +13,7 @@ class TestDataError(Exception):
class TestTemplate:
def __init__(self, args, agnpath, test_data, queue, queue_lock):
def __init__(self, args, agnpath, test_data, queue, queue_lock, dict_process, dict_process_lock):
"""
Simple test template.
:param args: List of arguments
@ -26,6 +26,10 @@ class TestTemplate:
:type queue: queue.deque
:param queue_lock: The lock to use when putting the test status in the queue
:type queue_lock: threading.Lock
:param dict_process: The dictionary of processes
:type dict_process: dict
:param dict_process_lock: The lock to use when putting the process in the dictionary
:type dict_process_lock: threading.Lock
"""
self.args = args
self.ptb = args.ptb
@ -42,6 +46,11 @@ class TestTemplate:
self.failure_dict = {}
self.new_data = {}
self.progress = 0
self.dict_process = dict_process
self.dict_process_lock = dict_process_lock
self.dict_process_lock.acquire()
self.dict_process[self.name()] = self
self.dict_process_lock.release()
def run_test(self):
"""
@ -71,12 +80,12 @@ class TestTemplate:
:rtype: internal_io.test_status
"""
return internal_io.test_status(
self.name(),
self.status_code,
self.status,
self.failure_dict,
self.new_data,
self.progress
name=self.name(),
status=self.status_code,
message=self.status,
failure_dict=self.failure_dict,
data=self.new_data,
progress=self.progress,
)
@staticmethod
@ -139,7 +148,7 @@ class TestTemplate:
class TestWalkTemplate(TestTemplate):
def __init__(self, args, agnpath, test_data, queue, queue_lock):
def __init__(self, args, agnpath, test_data, queue, queue_lock, dict_process, dict_process_lock):
"""
Simple test template.
:param args: List of arguments
@ -149,7 +158,7 @@ class TestWalkTemplate(TestTemplate):
:param test_data: The data to be tested
:type test_data: str
"""
super().__init__(args, agnpath, test_data, queue, queue_lock)
super().__init__(args, agnpath, test_data, queue, queue_lock, dict_process, dict_process_lock)
if test_data is None:
self.set_status("problem", "No test data given.")