added testing and generating testing reports

7 files changed, 116 insertions, 20 deletions

diff --git a/ExampleAssessments/CMP-4009B.yml b/ExampleAssessments/CMP-4009B.yml
index 571c97f..fbadc90 100644
--- a/ExampleAssessments/CMP-4009B.yml
+++ b/ExampleAssessments/CMP-4009B.yml
@@ -1,3 +1,4 @@
+assessment_name: CMP-4009B
 files:
     - pjtool.py:
         classes:
@@ -11,18 +12,15 @@ files:
                     - numMonths(2)
         tests:
             - |
-              d1 = Date(2001, 8, 12)
-              d2 = Date(2001, 8, 12)
-              return d1 == d2
+                d1 = pjtool.Date(2001, 8, 12)
+                d2 = pjtool.Date(2001, 8, 12)
+                assert d1 == d2
+            - |
+                d1 = pjtool.Date(2001, 8, 12)
+                d2 = pjtool.Date(1999, 4, 5)
+                assert d1 != d2
     - tester.py:
         functions:
             - dateTester(2)
-        printIfExecuted: False
-    - turbine.py:
-        classes:
-            - Turbine:
-                attributes:
-                    - rating:int
-    - aNonExistantModule.py:
-        functions:
-            - aNonExistantFunction(1)
\ No newline at end of file
+dependencies:
+    - matplotlib
\ No newline at end of file
diff --git a/ExampleAssessments/example.yml b/ExampleAssessments/example.yml
index bab1fb0..17ab9b0 100644
--- a/ExampleAssessments/example.yml
+++ b/ExampleAssessments/example.yml
@@ -12,6 +12,15 @@ files:
             - an_undocumented_function(0)
             - aFunctionThatIsntThere(2)
             - greet(2)
+        tests:
+            - |
+                dateOne = example.MyDate(day = 12, month = 8, year = 2001)
+                dateTwo = example.MyDate(day = 12, month = 8, year = 2001)
+                assert dateOne == dateTwo
+            - |
+                dateOne = example.MyDate(day = 12, month = 8, year = 2001)
+                dateTwo = example.MyDate(day = 5, month = 4, year = 1999)
+                assert dateOne == dateTwo
     - aFileThatIsntThere.py:
         functions:
             - hello_world(2)
@@ -19,4 +28,11 @@ files:
         classes:
             - Dog:
             - Cat:
-            - Kitten:
\ No newline at end of file
+            - Kitten:
+        tests:
+            - |
+                nibbles = animals.Kitten()
+                assert nibbles.speak() == "nyaa~~"
+            - |
+                milton = animals.Dog()
+                assert milton.move() == "*moves*"
\ No newline at end of file
diff --git a/ExampleSubmission/example.py b/ExampleSubmission/example.py
index a226adb..cc61122 100644
--- a/ExampleSubmission/example.py
+++ b/ExampleSubmission/example.py
@@ -2,6 +2,7 @@
 # 12-01-21
 
 import tkinter as tk
+from dataclasses import dataclass
 
 class Application(tk.Tk):
     """An example class, which implements a GUI by inheriting from tkinter.Tk
@@ -35,6 +36,19 @@ class Application(tk.Tk):
         """
         return num1 + num2
 
+@dataclass
+class MyDate:
+    year:int
+    month:int
+    day:int
+
+    def __eq__(self, otherDate):
+        return self.year == otherDate.year and self.month == otherDate.month and self.day == otherDate.day
+
+    def __str__(self):
+        "%d-%d-%4d" % (self.day, self.month, self.year)
+    
+
 # hello world!
 def hello_world(times):
     """Prints 'hello world!' to stdout. Prints it out `times` times.
diff --git a/ExampleSubmission/test_dont_test_me.py b/ExampleSubmission/test_dont_test_me.py
new file mode 100644
index 0000000..511c713
--- /dev/null
+++ b/ExampleSubmission/test_dont_test_me.py
@@ -0,0 +1,7 @@
+"""My default pytest will assume that all files prefixed with
+'test' are test files. This file is here to make sure that 
+pytest only runs on the files it should run on.
+"""
+
+def test_1():
+    assert 1 == 2
\ No newline at end of file
diff --git a/mark.py b/mark.py
index c9f0dbd..95bba55 100644
--- a/mark.py
+++ b/mark.py
@@ -25,7 +25,7 @@ def main(**kwargs):
         with open(kwargs["assessment"], "r") as f:
             assessment_struct = yaml.safe_load(f)
 
-        output = reflect.gen_reflection_report(submission_files, assessment_struct)
+        output = reflect.gen_reflection_report(submission_files, assessment_struct, kwargs)
         output_file = kwargs["out"]
         if kwargs["format"] == "yaml":
             strout = yaml.dump(output)
@@ -34,6 +34,7 @@ def main(**kwargs):
 
         if output_file == "stdout":
             print(strout)
+            # input("\n\n[tempdir: %s] Press any key to close..." % tempdir)
             exit()
 
         if output_file == "auto":
@@ -42,6 +43,8 @@ def main(**kwargs):
         with open(output_file, "w") as f:
             f.write(strout)
 
+        # input("\n\n[tempdir: %s] Press any key to close..." % tempdir)
+
 if __name__ == "__main__":
     config = configparser.ConfigParser()
     config.read("smarker.conf")
diff --git a/reflect.py b/reflect.py
index 0ce886f..e5eea48 100644
--- a/reflect.py
+++ b/reflect.py
@@ -1,9 +1,13 @@
+import xml.etree.ElementTree as etree
 from dataclasses import dataclass
 from functools import reduce
 from operator import getitem
+import subprocess
 import importlib
+import tempfile
 import inspect
 import pkgutil
+import shutil
 import sys
 import os
 import re
@@ -17,6 +21,7 @@ class Reflect:
         self.client_code_path = os.path.normpath(self.client_code_path)
         sys.path.insert(1, self.client_code_path)
         self.client_modules = [p for p in pkgutil.iter_modules() if os.path.normpath(str(p[0])[12:-2]) == self.client_code_path]
+        # print("client moduules ", self.client_modules)
 
     def import_module(self, module_name):
         """Imports a module. Before reflection can be conducted, a module
@@ -98,6 +103,15 @@ class Reflect:
         }
 
     def get_class_full_name(self, class_):
+        """Returns the name of a class object as a nice string. e.g. modulename.classname
+        except if it's a builtin there'll be no module name.
+
+        Args:
+            class_ (class): A class to get the name of
+
+        Returns:
+            str: A nicely formatted class name.
+        """
         if class_.__module__ in ['builtins', 'exceptions']:
             return class_.__name__
         return "%s.%s" % (class_.__module__, class_.__name__)
@@ -143,14 +157,48 @@ class Reflect:
         # return inspect.getclasstree(classes)
         return tree
 
-    def get_source_code(self, file_, line_start, line_end):
-        with open(file=file_, mode="r") as f:
-            return f.readlines()[line_start:line_end]
-
-def gen_reflection_report(client_code_path, assessment_struct):
+    def run_tests(self, tests, run_colourful = False):
+        test_results = {}
+        test_results["pytest_report"] = ""
+        for filename, filestests in tests.items():
+            with open(os.path.join(self.client_code_path, "test_" + filename), "a") as f:
+                for m in self.client_modules:
+                    f.write("import %s\n" % m.name)
+                f.write("\n")
+
+                for i, test_code in enumerate(filestests, 1):
+                    f.write("def test_%d():\n" % i)
+                    for line in test_code.split("\n"):
+                        f.write("    %s\n" % line.rstrip())
+                    f.write("\n")
+
+        with tempfile.TemporaryDirectory() as tmp:
+            junitxmlpath = os.path.join(tmp, "report.xml")
+            cmd = ["pytest", "-v"] + [os.path.join(self.client_code_path, "test_%s" % f) for f in tests.keys()] + ["--junitxml=%s" % junitxmlpath]
+            #print(" ".join(cmd))
+            proc = subprocess.Popen(cmd, stdout = subprocess.PIPE)
+            while True:
+                line = proc.stdout.readline()
+                if not line:
+                    break
+                test_results["pytest_report"] += line.decode()
+
+            with open(junitxmlpath, "r") as f:
+                test_results["junitxml"] = f.read()
+        root = etree.fromstring(test_results["junitxml"])
+        test_results["meta"] = root.findall("./testsuite")[0].attrib
+
+        if run_colourful:
+            subprocess.run(cmd)
+
+        return test_results
+            
+def gen_reflection_report(client_code_path, assessment_struct, configuration):
+    print(configuration)
     reflection = Reflect(client_code_path)
     present_module_names = [i.name for i in reflection.client_modules]
     out = assessment_struct
+    tests_to_run = {}
 
     for i, required_file in enumerate(assessment_struct["files"], 0):
         required_file = list(required_file.keys())[0]
@@ -223,8 +271,17 @@ def gen_reflection_report(client_code_path, assessment_struct):
                 out["files"][i][required_file]["functions"][j][required_function]["minimum_arguments"] = present_functions[function_name][-2].count(",") + 1    
                 out["files"][i][required_file]["functions"][j][required_function]["source_code"] = present_functions[function_name][-2]
 
+        if "tests" in required_files_features.keys():
+            filename = list(assessment_struct["files"][i].keys())[0]
+            for j, test in enumerate(assessment_struct["files"][i][required_file]["tests"], 0):
+                try:
+                    tests_to_run[filename].append(test)
+                except KeyError:
+                    tests_to_run[filename] = [test]
+
+    out["test_results"] = reflection.run_tests(tests_to_run, configuration["out"] == "stdout")
     out["class_tree"] = reflection.get_class_tree()
-    return out
+    # return out
 
 if __name__ == "__main__":
     user_code_path = "D:\\Edencloud\\UniStuff\\3.0 - CMP 3rd Year Project\\Smarker\\../ExampleSubmissions/Submission_A"
diff --git a/requirements.txt b/requirements.txt
index bee6c14..8b2a354 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1 +1,2 @@
 PyYAML==6.0
+pytest