mise à jour des tests

runAllTests.py

@@ -20,14 +20,14 @@ def clangName():
 
 clang = clangName()
 
-def runTestNormalLevel(level):
-    print(f"Running test of level {level}")
-    stats[level] = (0,0,0,0)
-    for dirname, dirnames, filenames in os.walk(f'./tests/testlevel{level}'):
+def runNormalTestSuit(testSuit):
+    print(f"Running test suit : {testSuit}")
+    stats[testSuit] = (0,0,0,0)
+    for dirname, dirnames, filenames in os.walk(testSuit):
         for filename in filenames:
-            runNormalTest(level, dirname, filename)
+            runNormalTest(testSuit, dirname, filename)
 
-def runNormalTest(i, dirname,filename):
+def runNormalTest(testSuit, dirname,filename):
     vslToLLVM = False
     llvmToBin = False
     executionCorrect = False
@@ -55,29 +55,29 @@ def runNormalTest(i, dirname,filename):
 
     # Exe
     if llvmToBin :
+        executionCorrect = True
         try:
+            input = ""
             if os.path.isfile(f"{basename}.test_in"):
-                p = subprocess.run(f"./{basename} < {basename}.test_in", shell=True, stdout=subprocess.PIPE, timeout=10)
-            else:
-                p = subprocess.run(f"./{basename}", shell=True, stdout=subprocess.PIPE, timeout=10)
+                input = f"< {basename}.test_in"
+            p = subprocess.run(f"./{basename} " + input, shell=True, stdout=subprocess.PIPE, timeout=5)
         except subprocess.TimeoutExpired:
-            executionCorrect = basename == "./tests/testlevel4/level4diverge"
+            executionCorrect = "diverge" in basename
         
         # Check return code
         if os.path.isfile(f"{basename}.test_ret"):
             with open(f"{basename}.test_ret", "r") as expected:
-                    executionCorrect = int(expected.read()) == p.returncode
+                    executionCorrect = executionCorrect and int(expected.read()) == p.returncode
         # Check stdout
         if os.path.isfile(f"{basename}.test_out"):
             with open(f"{basename}.test_out", "rb") as expected:
                 executionCorrect = executionCorrect and expected.read() == p.stdout
-                
     
     print(f"{colorFromBool(vslToLLVM)}\t\tVSL  to LLVM      : {'OK' if vslToLLVM else 'Fail'}{Style.RESET_ALL}")
     print(f"{colorFromBool(llvmToBin)}\t\tLLVM to Bin       : {'OK' if llvmToBin else 'Fail'}{Style.RESET_ALL}")
     print(f"{colorFromBool(executionCorrect)}\t\tCorrect Execution : {'OK' if executionCorrect else 'Fail'}{Style.RESET_ALL}")
     
-    x,y,z,t = stats[i]
+    x,y,z,t = stats[testSuit]
     if vslToLLVM :
         x += 1
     if llvmToBin :
@@ -85,7 +85,7 @@ def runNormalTest(i, dirname,filename):
     if executionCorrect :
         z += 1
     t += 1
-    stats[i] = (x,y,z,t)
+    stats[testSuit] = (x,y,z,t)
 
 def colorFromBool(b):
     if b:
@@ -104,7 +104,7 @@ def afficheStats(stat):
 nbError = 0
 nbErrorTests = 0
 
-def runErrorTest(dirname,filename):
+def runErrorTest(testSuit, dirname,filename):
     path = os.path.join(dirname, filename)
     basename, ext = os.path.splitext(path)
     if ext != ".vsl":
@@ -112,37 +112,51 @@ def runErrorTest(dirname,filename):
     
     print(f'\tRunning test {filename}')
 
-    global nbErrorTests
-    nbErrorTests = nbErrorTests + 1
+    hasError = False
 
     p = subprocess.run(f"java -jar build/libs/TP2.jar < {path} 1>{basename}.ll", shell=True, stderr=subprocess.PIPE)
+
     if p.returncode != 0 and p.stderr != b'':
         print(f"\t\t{Fore.GREEN}Error : Yes{Style.RESET_ALL}")
-        global nbError
-        nbError = nbError + 1
-        return 1
+        hasError = True
     else:
         print(f"\t\t{Fore.RED}Error : No{Style.RESET_ALL}")
-        return 0
+        hasError = False
 
-def runErrorLevelTests():
-    print(f"Running test of level error")
-    for dirname, dirnames, filenames in os.walk(f'./tests/testlevelerror'):
-        # print path to all filenames.
+    x,t = stats[testSuit]
+    if hasError :
+        x += 1
+    t += 1
+    stats[testSuit] = (x,t)
+    
+
+def runErrorLevelTests(testSuit):
+    print(f"Running test suit : {testSuit}")
+    stats[testSuit] = (0,0)
+    for dirname, dirnames, filenames in os.walk(testSuit):
         for filename in filenames:
-            runErrorTest( dirname, filename)
+            runErrorTest(testSuit, dirname, filename)
 
 
-def runTests(testDirname):
-    for i in range(1,5):
-        runTestNormalLevel(i)
-    runErrorLevelTests()
+def runTests(testDirName):
+    folderContent = [os.path.join(testDirName, d) for d in os.listdir(testDirName)]
+    testSuits = [d for d in folderContent if os.path.isdir(d)]
+    testSuits.sort()
+    
+    for suit in testSuits:
+        if "error" in suit or "Error" in suit:
+            runErrorLevelTests(suit)
+        else:
+            runNormalTestSuit(suit)
+
+    for suit in testSuits:
+        print(f'Résumé du test {suit}')
+        if "error" in suit or "Error" in suit:
+            nbError, nbErrorTests = stats[suit]
+            print(f'{colorFromBool(nbError==nbErrorTests)}\t Nombre d\'erreurs : {nbError} / {nbErrorTests}{Style.RESET_ALL}')
+        else:
+            afficheStats(stats[suit])
 
-    for i in range(1,5):
-        print(f'Résumé du level {i}')
-        afficheStats(stats[i])
-    print(f'Résumé du level error')
-    print(f'{colorFromBool(nbError==nbErrorTests)}\t Nombre d\'erreurs : {nbError} / {nbErrorTests}{Style.RESET_ALL}')