Merge pull request #35 from idle-org/features/fast_access

Features/fast access
This commit is contained in:
Maximilien Rigaut 2022-06-27 18:35:58 +02:00 committed by GitHub
commit 9f00fd8b7e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 106 additions and 30 deletions

View File

@ -7,7 +7,7 @@ import sys
import random
import argparse
from typing import List
from typing import List, Dict
class AgnosticPaths:
@ -33,12 +33,39 @@ class AgnosticPaths:
self.all_files = self.walk_all_files()
random.shuffle(self.all_files)
self.size = len(self.all_files)
self.cached_files = {}
self.cached_dirs = {}
self._precache_files()
def walk_cached(self) -> Dict[str, bytes]:
"""
Returns the cached files and directories
:return: The list of files and directories
"""
return self.cached_files
def walk_files(self) -> List[str]:
"""
Returns the list of all files in the discord install
:return: The list of all files in the discord install
"""
return self.all_files
def cached_file(self, path: str) -> bytes:
"""
Returns the file at the given path, if it is cached, otherwise it will return None
:param path: The path to the file
:return: The file at the given path, if it is cached, otherwise it will return None
"""
if path in self.cached_files:
return self.cached_files[path]
return None
def get_os(self) -> str:
"""
Get the os string in a short, spydey-detector way.
For now only windows and linux are supported, note that due to the nature of linux builds, strings will likely become
linux-arch-deb, linux-arch-snap etc...
For now only windows and linux are supported, note that due to the nature of linux builds,
strings will likely become linux-arch-deb, linux-arch-snap etc...
:return: ("windows", "linux", "mac")
"""
platform = sys.platform
@ -65,7 +92,7 @@ class AgnosticPaths:
return self.force_path
if self.os == 'windows':
base_path = os.path.join(os.path.expanduser('~'), "AppData", "Local", "Discord")
ptb_path = base_path+"PTB"
ptb_path = base_path + "PTB"
if self.args.autodetect:
if os.path.exists(ptb_path):
self.ptb = "PTB"
@ -77,7 +104,7 @@ class AgnosticPaths:
return base_path
if self.ptb == 'PTB':
self.default_database = "databases/windows_ptb.jsone"
return base_path+"PTB"
return base_path + "PTB"
else:
self.default_database = "databases/windows_base.json"
return base_path # Try to find the default version
@ -147,7 +174,7 @@ class AgnosticPaths:
def __call__(self, *args: str) -> str:
"""
Tries to open the path /discord_path/args[0]/args[1].../args[n]
:param args: The path of the file or directory as a list of parameters __call__("folder1", "folder2", ..., "foldern")
:param args: The path of the file or directory as a list of parameters __call__("folder_1", ..., "folder_n")
:return: The path is it exists
:raises: FileNotFoundError if the path doesn't exist
"""
@ -181,6 +208,15 @@ class AgnosticPaths:
"""
return os.path.relpath(path, self.main_path)
def _precache_files(self) -> None:
"""
Precache all files in the discord folder
"""
self.cached_files = {}
for file in self.all_files:
with open(file, "rb") as f:
self.cached_files[file] = f.read()
def __repr__(self) -> str:
"""
:return: A representation of the AgnosticPath

View File

@ -474,7 +474,7 @@ def run_check(args: argparse.Namespace, agnpath: agnostic_paths.AgnosticPaths) -
tr = TestRunner(args, agnpath, start_time=start) # All operation are executed in parallel and the user can stop them
# Poll information from the tests
POLLRATE, PRINT_RATE, COUNTER = args.pollrate, args.printrate, 0
POLLRATE, PRINT_RATE, COUNTER = args.pollrate, args.printrate, 1
COUNTER_MAX = int(PRINT_RATE / POLLRATE)
# Main test polling loop
@ -554,7 +554,7 @@ def fill_it(size: int, string1: str, string2: str) -> str:
Makes a string with the format "string1......string2"
with as many "." as needed to fill the size
"""
return string1 + max(size - len(string1)-len(string2), 0) * "." + string2
return string1 + max(size - len(string1) - len(string2), 0) * "." + string2
def at_max_elements(table: list, msize: int, fill: str) -> list:
@ -566,7 +566,7 @@ def at_max_elements(table: list, msize: int, fill: str) -> list:
# if en(table) > lmsize:
table = table[:msize] + [
fill+"..."+str(len(table)-msize)+" more lines" if len(table) > msize else ""]
fill + "..." + str(len(table) - msize)+" more lines" if len(table) > msize else ""]
return table

View File

@ -6,14 +6,18 @@ from modules import test_template, internal_io
from zlib import adler32
def apply_adler(path: str) -> hex:
def apply_adler(path: str, cached_data: bytes = None) -> hex:
"""
Apply the adler32 checksum to a file.
Apply the adler32 checksum to a string.
:param path: The path to the file.
:param cached_data: The cached data if it exists.
:return: The adler32 checksum.
"""
with open(path, "rb") as f:
return hex(adler32(f.read()))
if cached_data is None:
with open(path, "rb") as f:
return hex(adler32(f.read()))
return hex(adler32(cached_data))
class Adler32(test_template.TestWalkTemplateSimpleFunction):
@ -21,4 +25,4 @@ class Adler32(test_template.TestWalkTemplateSimpleFunction):
"""
Simple test runner, compare adler32 to known good values.
"""
super().__init__(thread_parameters, apply_adler, "")
super().__init__(thread_parameters, apply_adler, "", use_cache=True)

View File

@ -10,4 +10,6 @@ class LineCount(test_template.TestWalkTemplateSimpleFunction):
"""
Simple test runner, compare line count to known good values.
"""
super().__init__(thread_parameters, lambda path: len(open(path, "rb").readlines()), " lines")
super().__init__(thread_parameters,
lambda bstr, cached_data: len(open(bstr, "rb").readlines()),
" lines", use_cache=False)

View File

@ -6,12 +6,21 @@ from hashlib import sha256
from modules import test_template, internal_io
def sha256_hash(path: str) -> hex:
def sha256_hash(path: str, cached_data: bytes = None) -> hex:
"""
Returns the sha256 hash of the data.
"""
with open(path, "rb") as f:
return sha256(f.read()).hexdigest()
if cached_data is None:
with open(path, "rb") as f:
return sha256(f.read()).hexdigest()
return sha256(cached_data).hexdigest()
def sha256_hash_on_str(io_s: bytes) -> hex:
"""
Returns the sha256 hash of the data.
"""
return sha256(io_s).hexdigest()
class Sha256(test_template.TestWalkTemplateSimpleFunction):
@ -22,4 +31,4 @@ class Sha256(test_template.TestWalkTemplateSimpleFunction):
"""
Sha256 hash function.
"""
super().__init__(thread_parameters, sha256_hash, "")
super().__init__(thread_parameters, sha256_hash, "", use_cache=True)

View File

@ -3,9 +3,18 @@ import os
from modules import test_template, internal_io
def get_size(path: str, cached_data: bytes = None) -> int: # noqa: E501
"""
Get the size of a file.
:param path: The path to the file.
:return: The size of the file.
"""
return os.path.getsize(path)
class SizeCheck(test_template.TestWalkTemplateSimpleFunction):
def __init__(self, thread_parameters: internal_io.thread_parameters):
"""
Simple test runner, compare file size to known good values.
"""
super().__init__(thread_parameters, os.path.getsize, " bytes")
super().__init__(thread_parameters, get_size, " bytes", use_cache=False)

View File

@ -15,7 +15,7 @@ class TestDataError(Exception):
class TestTemplate:
def __init__(self, thread_parameters: internal_io.thread_parameters):
def __init__(self, thread_parameters: internal_io.thread_parameters, use_cache: bool = True) -> None:
"""
Simple test template.
:param thread_parameters.args: List of arguments
@ -50,6 +50,7 @@ class TestTemplate:
self.failure_dict = {}
self.new_data = {}
self.progress = 0
self.use_cache = use_cache
self.is_unknown = False
self.dict_process = thread_parameters.dict_process
self.dict_process_lock = thread_parameters.dict_process_lock
@ -66,7 +67,7 @@ class TestTemplate:
try:
self.run_test()
except Exception as e:
self.add_failure("problem", "DISCORD HELPER:"+str(e))
self.add_failure("problem", "DISCORD HELPER:" + str(e))
return self.finish("problem", str(e))
def run_test(self) -> None:
@ -203,11 +204,11 @@ class TestTemplate:
class TestWalkTemplate(TestTemplate):
def __init__(self, thread_parameters: internal_io.thread_parameters):
def __init__(self, thread_parameters: internal_io.thread_parameters, use_cache: bool = False):
"""
Simple test template.
"""
super().__init__(thread_parameters)
super().__init__(thread_parameters, use_cache)
self.to_skip = []
def run_test(self) -> str:
@ -284,7 +285,10 @@ class TestWalkTemplate(TestTemplate):
if self.agnostic_path.get_short_path(path) in self.to_skip:
return True
if self.use_cache:
kwargs["cached_data"] = self.agnostic_path.cached_file(path)
else:
kwargs["cached_data"] = None
data_origin = self.add_test_result(path, entry_name, function(path, *args, **kwargs))
expected_data = self.get_expected_result(path, entry_name)
@ -303,23 +307,24 @@ class TestWalkTemplate(TestTemplate):
class TestWalkTemplateNoLogs(TestWalkTemplate):
def __init__(self, thread_parameters: internal_io.thread_parameters):
def __init__(self, thread_parameters: internal_io.thread_parameters, use_cache: bool = False):
"""
Simple test template.
"""
super().__init__(thread_parameters)
super().__init__(thread_parameters, use_cache)
self.to_skip = [r"modules\discord_dispatch-1\discord_dispatch\dispatch.log"]
class TestWalkTemplateSimpleFunction(TestWalkTemplateNoLogs):
def __init__(self, thread_parameters: internal_io.thread_parameters, function: Callable, unit_test: str):
def __init__(self, thread_parameters: internal_io.thread_parameters,
function: Callable, unit_test: str, use_cache: bool = False):
"""
A test template that runs a simple function on every file, and compares the result with the expected data.
:param thread_parameters: Thread parameters
:param function: Function to use on every file
:param unit_test: A descriptor of what the test returns
"""
super().__init__(thread_parameters)
super().__init__(thread_parameters, use_cache)
self.function = function
self.test_name = self.name()
self.unit_test = unit_test
@ -334,7 +339,7 @@ class TestWalkTemplateSimpleFunction(TestWalkTemplateNoLogs):
error_msg = ""
for i, path in enumerate(self.walk()):
self.progress = int(100*i/size)
self.progress = int(100*i / size)
short_path = self.agnostic_path.get_short_path(path)
if os.path.isfile(path):
if not self.compare(path, self.test_name, self.function, (), {}):

11
tox.ini Normal file
View File

@ -0,0 +1,11 @@
[flake8]
ignore = D203, E226
exclude =
.git,
__pycache__,
docs/source/conf.py,
old,
build,
dist
max-complexity = 16
max_line_length = 127