Better test add / autogen
This commit is contained in:
parent
5de827cb0a
commit
b99cc4cfd7
|
|
@ -131,3 +131,4 @@ dmypy.json
|
|||
|
||||
|
||||
/.idea/*
|
||||
databases/*
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
global:
|
||||
name: Windows Base
|
||||
version: 1.0
|
||||
|
||||
|
||||
files:
|
||||
/etc/passwd:
|
||||
sha256: "6c3d65f8b8b8f5a8f62b60c0e15d5d084d3b20f470f5d8b3d26bfb3c350d37a3"
|
||||
type: file
|
||||
size: "1024"
|
||||
permissions: "0644"
|
||||
owner: "root"
|
||||
group: "root"
|
||||
date: "2018-01-01T00:00:00Z"
|
||||
executable: "true"
|
||||
symlink: "false"
|
||||
md5: "6c3d65f8b8b8f5a8f62b60c0e15d5d084d3b20f470f5d8b3d26bfb3c350d37a3"
|
||||
sha1: "6c3d65f8b8b8f5a8f62b60c0e15d5d084d3b20f470f5d8b3d26bfb3c350d37a3"
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
""" Main program """
|
||||
|
||||
from modules import test_interface, agnostic_paths, argument_parser
|
||||
from modules import main_runner, agnostic_paths, argument_parser
|
||||
import sys
|
||||
|
||||
|
||||
|
|
@ -13,7 +13,7 @@ def main(argm):
|
|||
:type argm: argparse.Namespace
|
||||
"""
|
||||
agnostic_path = agnostic_paths.AgnosticPaths(argm.ptb, argm.force_path)
|
||||
test_runner = test_interface.run_check(argm, agnostic_path)
|
||||
test_runner = main_runner.run_check(argm, agnostic_path)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ A windows executable for the discordHelper.py script.
|
|||
import sys
|
||||
|
||||
import discordHelper
|
||||
import time
|
||||
|
||||
# All modules must be loaded before the main program is run.
|
||||
from modules import spidey_test, test_template
|
||||
|
|
@ -12,8 +13,7 @@ from modules import spidey_test, test_template
|
|||
|
||||
if __name__ == "__main__":
|
||||
argv = sys.argv[1:]
|
||||
if len(argv) <= 1:
|
||||
argv += "--spidey --launch --test-walk --continue --gen-data new.yaml".split(" ")
|
||||
|
||||
if len(argv) == 0:
|
||||
argv = f"--all --launch --continue --gen-data new{int(time.time())}.yaml".split(" ")
|
||||
args = discordHelper.argument_parser.parse(argv)
|
||||
discordHelper.main(args)
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
import json
|
||||
import os
|
||||
import sys
|
||||
import importlib
|
||||
import threading
|
||||
|
|
@ -14,7 +16,10 @@ _TEST_MODULES = {
|
|||
"spidey": ["spidey_test", "SpideyTest"],
|
||||
"test_walk": ["test_template", "TestWalkTemplate"],
|
||||
}
|
||||
|
||||
PROGRAM_VERSION = "1.0.0"
|
||||
POST_RUN_ALLOWED = False
|
||||
|
||||
queue_lock = threading.Lock()
|
||||
|
||||
|
||||
|
|
@ -58,7 +63,7 @@ class TestRunner:
|
|||
module_name = None
|
||||
try:
|
||||
for module_name, values in _TEST_MODULES.items():
|
||||
if self.args.__getattribute__(module_name):
|
||||
if self.args.all or self.args.__getattribute__(module_name):
|
||||
modules[values[0]] = importlib.import_module(f"modules.{values[0]}")
|
||||
classes[values[0]] = getattr(modules[values[0]], values[1])
|
||||
|
||||
|
|
@ -200,10 +205,35 @@ class TestRunner:
|
|||
data = {}
|
||||
queue_lock.acquire()
|
||||
for test in self.finished_tests:
|
||||
data[test.name] = test.data
|
||||
for key, value in test.data.items():
|
||||
if key not in data:
|
||||
data[key] = value
|
||||
else:
|
||||
data[key].update(value)
|
||||
# data.update(test.data)
|
||||
queue_lock.release()
|
||||
return data
|
||||
|
||||
def export_data(test_runner, data):
|
||||
"""
|
||||
Exports the test data
|
||||
:param test_runner: The test runner
|
||||
:type test_runner: TestRunner
|
||||
:param data: The test data
|
||||
:type data: dict
|
||||
"""
|
||||
main_data = {
|
||||
"global":
|
||||
{
|
||||
"os": test_runner.os_name,
|
||||
"ptb": test_runner.ptb,
|
||||
"discord_version": test_runner.version,
|
||||
"program_version": PROGRAM_VERSION,
|
||||
},
|
||||
"tests": data,
|
||||
}
|
||||
return main_data
|
||||
|
||||
|
||||
def run_check(args, agnpath):
|
||||
"""
|
||||
|
|
@ -223,6 +253,12 @@ def run_check(args, agnpath):
|
|||
print(tr.get_status()) # TODO: Print the final status, test still running are actually skipped
|
||||
if args.gen_data:
|
||||
print("Generating data...")
|
||||
data = tr.get_test_data()
|
||||
print("Exporting data...")
|
||||
main_data = export_data(tr, data)
|
||||
# print(main_data)
|
||||
with open(os.path.join("databases", args.gen_data[0]), "w") as f:
|
||||
json.dump(main_data, f, indent=4)
|
||||
# print(tr.get_test_data()) # TODO: Transorm the data into a yaml or json file
|
||||
time.sleep(2)
|
||||
sys.exit(tr.get_exit_code())
|
||||
|
|
@ -40,7 +40,7 @@ class TestTemplate:
|
|||
self.set_status("idle")
|
||||
self.queue_lock = queue_lock
|
||||
self.failure_dict = {}
|
||||
self.data = {}
|
||||
self.new_data = {}
|
||||
|
||||
def run_test(self):
|
||||
"""
|
||||
|
|
@ -69,7 +69,7 @@ class TestTemplate:
|
|||
:return: Status at the time of call.
|
||||
:rtype: internal_io.test_status
|
||||
"""
|
||||
return internal_io.test_status(self.name(), self.status_code, self.status, self.failure_dict, self.data)
|
||||
return internal_io.test_status(self.name(), self.status_code, self.status, self.failure_dict, self.new_data)
|
||||
|
||||
@staticmethod
|
||||
def get_status_from_code(code):
|
||||
|
|
@ -141,11 +141,62 @@ class TestWalkTemplate(TestTemplate):
|
|||
:type test_data: str
|
||||
"""
|
||||
super().__init__(args, agnpath, test_data, queue, queue_lock)
|
||||
self.data = {}
|
||||
if test_data is None:
|
||||
self.set_status("problem", "No test data given.")
|
||||
|
||||
def add_test_result(self, path, testname, result):
|
||||
"""
|
||||
Adds a test result to the data dictionary.
|
||||
:param path: Path to the file that was tested
|
||||
:type path: str
|
||||
:param testname: Name of the test
|
||||
:type testname: str
|
||||
:param result: Result of the test
|
||||
:type result: any
|
||||
"""
|
||||
|
||||
if path not in self.new_data:
|
||||
self.new_data[path] = {}
|
||||
self.new_data[path][testname] = result
|
||||
return result
|
||||
|
||||
def get_test_data(self, path, testname):
|
||||
"""
|
||||
Gets the test data from the data dictionary.
|
||||
:param path: Path to the file that was tested
|
||||
:type path: str
|
||||
:param testname: Name of the test
|
||||
:type testname: str
|
||||
:return: Test data
|
||||
:rtype: any
|
||||
"""
|
||||
if path in self.new_data:
|
||||
if testname in self.new_data[path]:
|
||||
return self.new_data[path][testname]
|
||||
return None
|
||||
|
||||
def get_expected_result(self, path, testname):
|
||||
"""
|
||||
Gets the expected result from the data dictionary.
|
||||
:param path: Path to the file that was tested
|
||||
:type path: str
|
||||
:param testname: Name of the test
|
||||
:type testname: str
|
||||
:return: Expected result
|
||||
:rtype: any
|
||||
"""
|
||||
if self.test_data:
|
||||
if path in self.test_data:
|
||||
if testname in self.test_data[path]:
|
||||
return self.test_data[path][testname]
|
||||
return None
|
||||
|
||||
def run_test(self):
|
||||
"""
|
||||
The main runner function, must be implemented by the child class.
|
||||
Here is an example of how to use the add_test_result function.
|
||||
:return:
|
||||
"""
|
||||
self.set_status("running", "The test will begin shortly, please wait...")
|
||||
|
||||
if self.test_data is None and not self.args.continue_on_error:
|
||||
|
|
@ -190,22 +241,13 @@ class TestWalkTemplate(TestTemplate):
|
|||
:type kwargs: dict
|
||||
"""
|
||||
|
||||
try:
|
||||
self.data[path] = function(path, *args, **kwargs)
|
||||
if not self.test_data:
|
||||
raise TestDataError("No test data given.")
|
||||
if path not in self.test_data["files"]:
|
||||
raise TestDataError("No entry in test data for path: " + str(path))
|
||||
if entry_name not in self.test_data["files"][path]:
|
||||
raise TestDataError("No entry in test data for path: " + str(path) + " and entry name: " + str(entry_name))
|
||||
data_origin = self.test_data["files"][path][entry_name]
|
||||
self.data[path] = function(path, *args, **kwargs)
|
||||
if data_origin != self.data[path]:
|
||||
return False
|
||||
return True
|
||||
except TestDataError as e:
|
||||
if not self.args.continue_on_error:
|
||||
self.set_status("error", "Error: " + str(e))
|
||||
raise e
|
||||
finally:
|
||||
data_origin = self.add_test_result(path, entry_name, function(path, *args, **kwargs))
|
||||
expected_data = self.get_expected_result(path, entry_name)
|
||||
if expected_data is None and not self.args.continue_on_error:
|
||||
self.set_status("problem", f"No expected data found for {path}")
|
||||
return False
|
||||
|
||||
if data_origin != expected_data:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
|
|
|||
Reference in New Issue