Initial commit

runAllTests.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env python3
+
+import os
+import subprocess
+
+from colorama import init as colorama_init
+from colorama import Fore
+from colorama import Style
+
+stats = {}
+
+def runTestNormalLevel(level):
+    print(f"Running test of level {level}")
+    stats[level] = (0,0,0,0)
+    for dirname, dirnames, filenames in os.walk(f'./tests/testlevel{level}'):
+        for filename in filenames:
+            runNormalTest(level, dirname, filename)
+
+def runNormalTest(i, dirname,filename):
+    vslToLLVM = False
+    llvmToBin = False
+    executionCorrect = False
+
+    path = os.path.join(dirname, filename)
+    basename, ext = os.path.splitext(path)
+    # Ignore not .vsl files
+    if ext != ".vsl":
+        return 
+    
+    print(f'\tRunning test {filename}')
+
+    # VSL -> LLVM
+    p = subprocess.run(f"java -jar build/libs/TP2.jar < {path} 1>{basename}.ll", shell=True)
+    if p.returncode == 0 :
+        vslToLLVM = True
+
+
+    # LLVM -> Bin
+    if vslToLLVM :
+        p = subprocess.run(f"clang {basename}.ll -o {basename} 2>/dev/null", shell=True)
+        if p.returncode == 0:
+            llvmToBin = True
+            executionCorrect = True
+
+    # Exe
+    if llvmToBin :
+        try:
+            if os.path.isfile(f"{basename}.test_in"):
+                p = subprocess.run(f"./{basename} < {basename}.test_in", shell=True, stdout=subprocess.PIPE, timeout=10)
+            else:
+                p = subprocess.run(f"./{basename}", shell=True, stdout=subprocess.PIPE, timeout=10)
+        except subprocess.TimeoutExpired:
+            executionCorrect = basename == "./tests/testlevel4/level4diverge"
+        
+        # Check return code
+        if os.path.isfile(f"{basename}.test_ret"):
+            with open(f"{basename}.test_ret", "r") as expected:
+                    executionCorrect = int(expected.read()) == p.returncode
+        # Check stdout
+        if os.path.isfile(f"{basename}.test_out"):
+            with open(f"{basename}.test_out", "rb") as expected:
+                executionCorrect = executionCorrect and expected.read() == p.stdout
+                
+    
+    print(f"{colorFromBool(vslToLLVM)}\t\tVSL  to LLVM      : {'OK' if vslToLLVM else 'Fail'}{Style.RESET_ALL}")
+    print(f"{colorFromBool(llvmToBin)}\t\tLLVM to Bin       : {'OK' if llvmToBin else 'Fail'}{Style.RESET_ALL}")
+    print(f"{colorFromBool(executionCorrect)}\t\tCorrect Execution : {'OK' if executionCorrect else 'Fail'}{Style.RESET_ALL}")
+    
+    x,y,z,t = stats[i]
+    if vslToLLVM :
+        x += 1
+    if llvmToBin :
+        y += 1
+    if executionCorrect :
+        z += 1
+    t += 1
+    stats[i] = (x,y,z,t)
+
+def colorFromBool(b):
+    if b:
+        return Fore.GREEN
+    else: 
+        return Fore.RED
+
+
+def afficheStats(stat):
+    x,y,z,t = stat
+    print(f'\t Nombre de tests : {t}')
+    print(f'{colorFromBool(x==t)}\t vsl to llvm     : {x} / {t}{Style.RESET_ALL}')
+    print(f'{colorFromBool(y==t)}\t llvm to bin     : {y} / {t}{Style.RESET_ALL}')
+    print(f'{colorFromBool(z==t)}\t resultat correct: {z} / {t}{Style.RESET_ALL}')
+
+nbError = 0
+nbErrorTests = 0
+
+def runErrorTest(dirname,filename):
+    path = os.path.join(dirname, filename)
+    basename, ext = os.path.splitext(path)
+    if ext != ".vsl":
+        return
+    
+    print(f'\tRunning test {filename}')
+
+    global nbErrorTests
+    nbErrorTests = nbErrorTests + 1
+
+    p = subprocess.run(f"java -jar build/libs/TP2.jar < {path} 1>{basename}.ll", shell=True, stderr=subprocess.PIPE)
+    if p.returncode != 0 and p.stderr != b'':
+        print(f"\t\t{Fore.GREEN}Error : Yes{Style.RESET_ALL}")
+        global nbError
+        nbError = nbError + 1
+        return 1
+    else:
+        print(f"\t\t{Fore.RED}Error : No{Style.RESET_ALL}")
+        return 0
+
+def runErrorLevelTests():
+    print(f"Running test of level error")
+    for dirname, dirnames, filenames in os.walk(f'./tests/testlevelerror'):
+        # print path to all filenames.
+        for filename in filenames:
+            runErrorTest( dirname, filename)
+
+
+def runTests(testDirname):
+    for i in range(1,5):
+        runTestNormalLevel(i)
+    runErrorLevelTests()
+
+    for i in range(1,5):
+        print(f'Résumé du level {i}')
+        afficheStats(stats[i])
+    print(f'Résumé du level error')
+    print(f'{colorFromBool(nbError==nbErrorTests)}\t Nombre d\'erreurs : {nbError} / {nbErrorTests}{Style.RESET_ALL}')
+
+
+
+
+if __name__ == "__main__" :
+    colorama_init()
+    runTests("tests")