diff --git a/modules/agnostic_paths.py b/modules/agnostic_paths.py index eea756a..8f5fdd4 100644 --- a/modules/agnostic_paths.py +++ b/modules/agnostic_paths.py @@ -7,7 +7,7 @@ import sys import random import argparse -from typing import List +from typing import List, Dict class AgnosticPaths: @@ -33,12 +33,39 @@ class AgnosticPaths: self.all_files = self.walk_all_files() random.shuffle(self.all_files) self.size = len(self.all_files) + self.cached_files = {} + self.cached_dirs = {} + self._precache_files() + + def walk_cached(self) -> Dict[str, bytes]: + """ + Returns the cached files and directories + :return: The list of files and directories + """ + return self.cached_files + + def walk_files(self) -> List[str]: + """ + Returns the list of all files in the discord install + :return: The list of all files in the discord install + """ + return self.all_files + + def cached_file(self, path: str) -> bytes: + """ + Returns the file at the given path, if it is cached, otherwise it will return None + :param path: The path to the file + :return: The file at the given path, if it is cached, otherwise it will return None + """ + if path in self.cached_files: + return self.cached_files[path] + return None def get_os(self) -> str: """ Get the os string in a short, spydey-detector way. - For now only windows and linux are supported, note that due to the nature of linux builds, strings will likely become - linux-arch-deb, linux-arch-snap etc... + For now only windows and linux are supported, note that due to the nature of linux builds, + strings will likely become linux-arch-deb, linux-arch-snap etc... :return: ("windows", "linux", "mac") """ platform = sys.platform @@ -65,7 +92,7 @@ class AgnosticPaths: return self.force_path if self.os == 'windows': base_path = os.path.join(os.path.expanduser('~'), "AppData", "Local", "Discord") - ptb_path = base_path+"PTB" + ptb_path = base_path + "PTB" if self.args.autodetect: if os.path.exists(ptb_path): self.ptb = "PTB" @@ -77,7 +104,7 @@ class AgnosticPaths: return base_path if self.ptb == 'PTB': self.default_database = "databases/windows_ptb.jsone" - return base_path+"PTB" + return base_path + "PTB" else: self.default_database = "databases/windows_base.json" return base_path # Try to find the default version @@ -147,7 +174,7 @@ class AgnosticPaths: def __call__(self, *args: str) -> str: """ Tries to open the path /discord_path/args[0]/args[1].../args[n] - :param args: The path of the file or directory as a list of parameters __call__("folder1", "folder2", ..., "foldern") + :param args: The path of the file or directory as a list of parameters __call__("folder_1", ..., "folder_n") :return: The path is it exists :raises: FileNotFoundError if the path doesn't exist """ @@ -181,6 +208,15 @@ class AgnosticPaths: """ return os.path.relpath(path, self.main_path) + def _precache_files(self) -> None: + """ + Precache all files in the discord folder + """ + self.cached_files = {} + for file in self.all_files: + with open(file, "rb") as f: + self.cached_files[file] = f.read() + def __repr__(self) -> str: """ :return: A representation of the AgnosticPath diff --git a/modules/main_runner.py b/modules/main_runner.py index 0cfcd24..f225e92 100644 --- a/modules/main_runner.py +++ b/modules/main_runner.py @@ -474,7 +474,7 @@ def run_check(args: argparse.Namespace, agnpath: agnostic_paths.AgnosticPaths) - tr = TestRunner(args, agnpath, start_time=start) # All operation are executed in parallel and the user can stop them # Poll information from the tests - POLLRATE, PRINT_RATE, COUNTER = args.pollrate, args.printrate, 0 + POLLRATE, PRINT_RATE, COUNTER = args.pollrate, args.printrate, 1 COUNTER_MAX = int(PRINT_RATE / POLLRATE) # Main test polling loop @@ -554,7 +554,7 @@ def fill_it(size: int, string1: str, string2: str) -> str: Makes a string with the format "string1......string2" with as many "." as needed to fill the size """ - return string1 + max(size - len(string1)-len(string2), 0) * "." + string2 + return string1 + max(size - len(string1) - len(string2), 0) * "." + string2 def at_max_elements(table: list, msize: int, fill: str) -> list: @@ -566,7 +566,7 @@ def at_max_elements(table: list, msize: int, fill: str) -> list: # if en(table) > lmsize: table = table[:msize] + [ - fill+"..."+str(len(table)-msize)+" more lines" if len(table) > msize else ""] + fill + "..." + str(len(table) - msize)+" more lines" if len(table) > msize else ""] return table diff --git a/modules/test_adler.py b/modules/test_adler.py index 8612aa0..62aab93 100644 --- a/modules/test_adler.py +++ b/modules/test_adler.py @@ -6,14 +6,18 @@ from modules import test_template, internal_io from zlib import adler32 -def apply_adler(path: str) -> hex: +def apply_adler(path: str, cached_data: bytes = None) -> hex: """ - Apply the adler32 checksum to a file. + Apply the adler32 checksum to a string. :param path: The path to the file. + :param cached_data: The cached data if it exists. :return: The adler32 checksum. """ - with open(path, "rb") as f: - return hex(adler32(f.read())) + if cached_data is None: + with open(path, "rb") as f: + return hex(adler32(f.read())) + + return hex(adler32(cached_data)) class Adler32(test_template.TestWalkTemplateSimpleFunction): @@ -21,4 +25,4 @@ class Adler32(test_template.TestWalkTemplateSimpleFunction): """ Simple test runner, compare adler32 to known good values. """ - super().__init__(thread_parameters, apply_adler, "") + super().__init__(thread_parameters, apply_adler, "", use_cache=True) diff --git a/modules/test_line_count.py b/modules/test_line_count.py index 335d3de..8890b2a 100644 --- a/modules/test_line_count.py +++ b/modules/test_line_count.py @@ -10,4 +10,6 @@ class LineCount(test_template.TestWalkTemplateSimpleFunction): """ Simple test runner, compare line count to known good values. """ - super().__init__(thread_parameters, lambda path: len(open(path, "rb").readlines()), " lines") + super().__init__(thread_parameters, + lambda bstr, cached_data: len(open(bstr, "rb").readlines()), + " lines", use_cache=False) diff --git a/modules/test_sha.py b/modules/test_sha.py index efcc3b3..4eaf1e2 100644 --- a/modules/test_sha.py +++ b/modules/test_sha.py @@ -6,12 +6,21 @@ from hashlib import sha256 from modules import test_template, internal_io -def sha256_hash(path: str) -> hex: +def sha256_hash(path: str, cached_data: bytes = None) -> hex: """ Returns the sha256 hash of the data. """ - with open(path, "rb") as f: - return sha256(f.read()).hexdigest() + if cached_data is None: + with open(path, "rb") as f: + return sha256(f.read()).hexdigest() + return sha256(cached_data).hexdigest() + + +def sha256_hash_on_str(io_s: bytes) -> hex: + """ + Returns the sha256 hash of the data. + """ + return sha256(io_s).hexdigest() class Sha256(test_template.TestWalkTemplateSimpleFunction): @@ -22,4 +31,4 @@ class Sha256(test_template.TestWalkTemplateSimpleFunction): """ Sha256 hash function. """ - super().__init__(thread_parameters, sha256_hash, "") + super().__init__(thread_parameters, sha256_hash, "", use_cache=True) diff --git a/modules/test_size.py b/modules/test_size.py index f9934f5..e40ac96 100644 --- a/modules/test_size.py +++ b/modules/test_size.py @@ -3,9 +3,18 @@ import os from modules import test_template, internal_io +def get_size(path: str, cached_data: bytes = None) -> int: # noqa: E501 + """ + Get the size of a file. + :param path: The path to the file. + :return: The size of the file. + """ + return os.path.getsize(path) + + class SizeCheck(test_template.TestWalkTemplateSimpleFunction): def __init__(self, thread_parameters: internal_io.thread_parameters): """ Simple test runner, compare file size to known good values. """ - super().__init__(thread_parameters, os.path.getsize, " bytes") + super().__init__(thread_parameters, get_size, " bytes", use_cache=False) diff --git a/modules/test_template.py b/modules/test_template.py index b30a884..7ac5538 100644 --- a/modules/test_template.py +++ b/modules/test_template.py @@ -15,7 +15,7 @@ class TestDataError(Exception): class TestTemplate: - def __init__(self, thread_parameters: internal_io.thread_parameters): + def __init__(self, thread_parameters: internal_io.thread_parameters, use_cache: bool = True) -> None: """ Simple test template. :param thread_parameters.args: List of arguments @@ -50,6 +50,7 @@ class TestTemplate: self.failure_dict = {} self.new_data = {} self.progress = 0 + self.use_cache = use_cache self.is_unknown = False self.dict_process = thread_parameters.dict_process self.dict_process_lock = thread_parameters.dict_process_lock @@ -66,7 +67,7 @@ class TestTemplate: try: self.run_test() except Exception as e: - self.add_failure("problem", "DISCORD HELPER:"+str(e)) + self.add_failure("problem", "DISCORD HELPER:" + str(e)) return self.finish("problem", str(e)) def run_test(self) -> None: @@ -203,11 +204,11 @@ class TestTemplate: class TestWalkTemplate(TestTemplate): - def __init__(self, thread_parameters: internal_io.thread_parameters): + def __init__(self, thread_parameters: internal_io.thread_parameters, use_cache: bool = False): """ Simple test template. """ - super().__init__(thread_parameters) + super().__init__(thread_parameters, use_cache) self.to_skip = [] def run_test(self) -> str: @@ -284,7 +285,10 @@ class TestWalkTemplate(TestTemplate): if self.agnostic_path.get_short_path(path) in self.to_skip: return True - + if self.use_cache: + kwargs["cached_data"] = self.agnostic_path.cached_file(path) + else: + kwargs["cached_data"] = None data_origin = self.add_test_result(path, entry_name, function(path, *args, **kwargs)) expected_data = self.get_expected_result(path, entry_name) @@ -303,23 +307,24 @@ class TestWalkTemplate(TestTemplate): class TestWalkTemplateNoLogs(TestWalkTemplate): - def __init__(self, thread_parameters: internal_io.thread_parameters): + def __init__(self, thread_parameters: internal_io.thread_parameters, use_cache: bool = False): """ Simple test template. """ - super().__init__(thread_parameters) + super().__init__(thread_parameters, use_cache) self.to_skip = [r"modules\discord_dispatch-1\discord_dispatch\dispatch.log"] class TestWalkTemplateSimpleFunction(TestWalkTemplateNoLogs): - def __init__(self, thread_parameters: internal_io.thread_parameters, function: Callable, unit_test: str): + def __init__(self, thread_parameters: internal_io.thread_parameters, + function: Callable, unit_test: str, use_cache: bool = False): """ A test template that runs a simple function on every file, and compares the result with the expected data. :param thread_parameters: Thread parameters :param function: Function to use on every file :param unit_test: A descriptor of what the test returns """ - super().__init__(thread_parameters) + super().__init__(thread_parameters, use_cache) self.function = function self.test_name = self.name() self.unit_test = unit_test @@ -334,7 +339,7 @@ class TestWalkTemplateSimpleFunction(TestWalkTemplateNoLogs): error_msg = "" for i, path in enumerate(self.walk()): - self.progress = int(100*i/size) + self.progress = int(100*i / size) short_path = self.agnostic_path.get_short_path(path) if os.path.isfile(path): if not self.compare(path, self.test_name, self.function, (), {}): diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..51e6da7 --- /dev/null +++ b/tox.ini @@ -0,0 +1,11 @@ +[flake8] +ignore = D203, E226 +exclude = + .git, + __pycache__, + docs/source/conf.py, + old, + build, + dist +max-complexity = 16 +max_line_length = 127 \ No newline at end of file