Better test add / autogen

This commit is contained in:
Maximilien Rigaut 2022-06-24 02:02:36 +02:00
parent 5de827cb0a
commit b99cc4cfd7
No known key found for this signature in database
GPG Key ID: 4F14B23709ADA5B1
7 changed files with 4506 additions and 47 deletions

1
.gitignore vendored
View File

@ -131,3 +131,4 @@ dmypy.json
/.idea/*
databases/*

View File

@ -1,19 +0,0 @@
global:
name: Windows Base
version: 1.0
files:
/etc/passwd:
sha256: "6c3d65f8b8b8f5a8f62b60c0e15d5d084d3b20f470f5d8b3d26bfb3c350d37a3"
type: file
size: "1024"
permissions: "0644"
owner: "root"
group: "root"
date: "2018-01-01T00:00:00Z"
executable: "true"
symlink: "false"
md5: "6c3d65f8b8b8f5a8f62b60c0e15d5d084d3b20f470f5d8b3d26bfb3c350d37a3"
sha1: "6c3d65f8b8b8f5a8f62b60c0e15d5d084d3b20f470f5d8b3d26bfb3c350d37a3"

4399
databases/windows_base.json Normal file

File diff suppressed because it is too large Load Diff

View File

@ -2,7 +2,7 @@
""" Main program """
from modules import test_interface, agnostic_paths, argument_parser
from modules import main_runner, agnostic_paths, argument_parser
import sys
@ -13,7 +13,7 @@ def main(argm):
:type argm: argparse.Namespace
"""
agnostic_path = agnostic_paths.AgnosticPaths(argm.ptb, argm.force_path)
test_runner = test_interface.run_check(argm, agnostic_path)
test_runner = main_runner.run_check(argm, agnostic_path)
if __name__ == "__main__":

View File

@ -5,6 +5,7 @@ A windows executable for the discordHelper.py script.
import sys
import discordHelper
import time
# All modules must be loaded before the main program is run.
from modules import spidey_test, test_template
@ -12,8 +13,7 @@ from modules import spidey_test, test_template
if __name__ == "__main__":
argv = sys.argv[1:]
if len(argv) <= 1:
argv += "--spidey --launch --test-walk --continue --gen-data new.yaml".split(" ")
if len(argv) == 0:
argv = f"--all --launch --continue --gen-data new{int(time.time())}.yaml".split(" ")
args = discordHelper.argument_parser.parse(argv)
discordHelper.main(args)

View File

@ -1,3 +1,5 @@
import json
import os
import sys
import importlib
import threading
@ -14,7 +16,10 @@ _TEST_MODULES = {
"spidey": ["spidey_test", "SpideyTest"],
"test_walk": ["test_template", "TestWalkTemplate"],
}
PROGRAM_VERSION = "1.0.0"
POST_RUN_ALLOWED = False
queue_lock = threading.Lock()
@ -58,7 +63,7 @@ class TestRunner:
module_name = None
try:
for module_name, values in _TEST_MODULES.items():
if self.args.__getattribute__(module_name):
if self.args.all or self.args.__getattribute__(module_name):
modules[values[0]] = importlib.import_module(f"modules.{values[0]}")
classes[values[0]] = getattr(modules[values[0]], values[1])
@ -200,10 +205,35 @@ class TestRunner:
data = {}
queue_lock.acquire()
for test in self.finished_tests:
data[test.name] = test.data
for key, value in test.data.items():
if key not in data:
data[key] = value
else:
data[key].update(value)
# data.update(test.data)
queue_lock.release()
return data
def export_data(test_runner, data):
"""
Exports the test data
:param test_runner: The test runner
:type test_runner: TestRunner
:param data: The test data
:type data: dict
"""
main_data = {
"global":
{
"os": test_runner.os_name,
"ptb": test_runner.ptb,
"discord_version": test_runner.version,
"program_version": PROGRAM_VERSION,
},
"tests": data,
}
return main_data
def run_check(args, agnpath):
"""
@ -223,6 +253,12 @@ def run_check(args, agnpath):
print(tr.get_status()) # TODO: Print the final status, test still running are actually skipped
if args.gen_data:
print("Generating data...")
data = tr.get_test_data()
print("Exporting data...")
main_data = export_data(tr, data)
# print(main_data)
with open(os.path.join("databases", args.gen_data[0]), "w") as f:
json.dump(main_data, f, indent=4)
# print(tr.get_test_data()) # TODO: Transorm the data into a yaml or json file
time.sleep(2)
sys.exit(tr.get_exit_code())

View File

@ -40,7 +40,7 @@ class TestTemplate:
self.set_status("idle")
self.queue_lock = queue_lock
self.failure_dict = {}
self.data = {}
self.new_data = {}
def run_test(self):
"""
@ -69,7 +69,7 @@ class TestTemplate:
:return: Status at the time of call.
:rtype: internal_io.test_status
"""
return internal_io.test_status(self.name(), self.status_code, self.status, self.failure_dict, self.data)
return internal_io.test_status(self.name(), self.status_code, self.status, self.failure_dict, self.new_data)
@staticmethod
def get_status_from_code(code):
@ -141,11 +141,62 @@ class TestWalkTemplate(TestTemplate):
:type test_data: str
"""
super().__init__(args, agnpath, test_data, queue, queue_lock)
self.data = {}
if test_data is None:
self.set_status("problem", "No test data given.")
def add_test_result(self, path, testname, result):
"""
Adds a test result to the data dictionary.
:param path: Path to the file that was tested
:type path: str
:param testname: Name of the test
:type testname: str
:param result: Result of the test
:type result: any
"""
if path not in self.new_data:
self.new_data[path] = {}
self.new_data[path][testname] = result
return result
def get_test_data(self, path, testname):
"""
Gets the test data from the data dictionary.
:param path: Path to the file that was tested
:type path: str
:param testname: Name of the test
:type testname: str
:return: Test data
:rtype: any
"""
if path in self.new_data:
if testname in self.new_data[path]:
return self.new_data[path][testname]
return None
def get_expected_result(self, path, testname):
"""
Gets the expected result from the data dictionary.
:param path: Path to the file that was tested
:type path: str
:param testname: Name of the test
:type testname: str
:return: Expected result
:rtype: any
"""
if self.test_data:
if path in self.test_data:
if testname in self.test_data[path]:
return self.test_data[path][testname]
return None
def run_test(self):
"""
The main runner function, must be implemented by the child class.
Here is an example of how to use the add_test_result function.
:return:
"""
self.set_status("running", "The test will begin shortly, please wait...")
if self.test_data is None and not self.args.continue_on_error:
@ -190,22 +241,13 @@ class TestWalkTemplate(TestTemplate):
:type kwargs: dict
"""
try:
self.data[path] = function(path, *args, **kwargs)
if not self.test_data:
raise TestDataError("No test data given.")
if path not in self.test_data["files"]:
raise TestDataError("No entry in test data for path: " + str(path))
if entry_name not in self.test_data["files"][path]:
raise TestDataError("No entry in test data for path: " + str(path) + " and entry name: " + str(entry_name))
data_origin = self.test_data["files"][path][entry_name]
self.data[path] = function(path, *args, **kwargs)
if data_origin != self.data[path]:
return False
return True
except TestDataError as e:
if not self.args.continue_on_error:
self.set_status("error", "Error: " + str(e))
raise e
finally:
data_origin = self.add_test_result(path, entry_name, function(path, *args, **kwargs))
expected_data = self.get_expected_result(path, entry_name)
if expected_data is None and not self.args.continue_on_error:
self.set_status("problem", f"No expected data found for {path}")
return False
if data_origin != expected_data:
return False
return True