Skip to content
Snippets Groups Projects
Select Git revision
  • c15586cd24c6f89c70cf123ca73ed4dc82779e6b
  • main default protected
  • NewGraphicSystem
  • LayeredRendering
4 results

imgui.ini

Blame
  • Code owners
    Assign users and groups as approvers for specific file changes. Learn more.
    evaluate.py 19.27 KiB
    #! /usr/bin/python3 -u
    
    import sys
    import os
    import subprocess
    import inspect
    import argparse
    
    # code de sortie du compilateur
    
    NOERROR_CODE = 0
    ERR_TYPE_CODE = 3
    ERR_SYMBOL_TABLE_CODE = 4
    ERR_SYNTAX_CODE = 5
    ERR_LEX_CODE = 6
    
    # coefficients pour le calcul de la note finale
    SC_COEFF = 4
    SA_DIFF_COEFF = 0
    TS_DIFF_COEFF = 0
    SA_COEFF = 4
    C3A_DIFF_COEFF = 0
    C3A_COEFF = 4
    PRE_NASM_DIFF = 0
    PRE_NASM_COEFF = 3
    NASM_DIFF_COEFF = 0
    NASM_COEFF = 3
    EXE_COEFF = 0
    ERR_TS_COEFF = 2
    ERR_TYPE_COEFF = 0
    
    # Paths relative to this file (evaluate.py)
    inputPath = "./input/"
    errInputPathTs = "./input_err_ts/"
    errInputPathType = "./input_err_type/"
    refPath = "./"
    srcPath = "../src/"
    # Keep empty
    classpath = ""
    # Outputs files (don't change this, it's set in the main)
    outErr = sys.stderr
    outScore = sys.stdout
    outVerbose = open(os.devnull,"w")
    outputFilename = "result.txt"
    
    ################################################################################
    def compileCompiler() :
      global classpath
      for file in ["Compiler.java", "SaVM.java", "C3aVM.java", "NasmVM.java"] :
        if not os.path.isfile(srcPath+file) :
          print("Skipping compilation of %s"%file, file=outVerbose)
          continue
        package = file.lower().split('.')[0].replace('vm', '')
        if package in ["c3a", "nasm"] and not os.path.isdir(srcPath+package) :
          print("Skipping compilation of %s"%file, file=outVerbose)
          continue
        print("Compiling %s..."%file, end="", file=outVerbose)
        proc = subprocess.Popen("cd %s && javac %s %s %s"%(srcPath, "-cp " if len(classpath) > 0 else "", classpath, file), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        errMess = proc.stderr.read().decode('utf8')
        returnCode = proc.wait()
        if returnCode == 0 :
          print("Done", file=outVerbose)
        else :
          print("", file=outVerbose)
          print("ERROR !", file=outErr)
          print(errMess, file=outErr)
          exit(1)
        print("", file=outVerbose)
        classpath = "../xerces-2_12_1/*:%s"%srcPath
      classpath = ""
    ################################################################################
    
    ################################################################################
    def deleteClasses() :
    
      for root, subdirs, files in os.walk("%s.."%srcPath) :
        if ".git" in root :
          continue
        for filename in files :
          if os.path.splitext(filename)[1] == ".class" :
            os.remove(root+"/"+filename)
            
      return classpath
    ################################################################################
    
    ################################################################################
    def findClasspath() :
      global classpath
    
      if len(classpath) > 0 :
        return classpath
    
      for root, subdirs, files in os.walk("%s.."%srcPath) :
        if ".git" in root :
          continue
        for filename in files :
          if os.path.splitext(filename)[1] in [".class", ".jar"] :
            classpath += ("" if len(classpath) == 0 else ":") + root
            break
    
      classpath += ("" if len(classpath) == 0 else ":")+"../xerces-2_12_1/*"
      return classpath
    ################################################################################
    
    ################################################################################
    def compiler(verbose="-v 2") :
      return "java -classpath %s Compiler %s"%(findClasspath(), verbose)
    ################################################################################
    
    ################################################################################
    def green(string) :
      return "\033[92m%s\033[0m"%string
    ################################################################################
    
    ################################################################################
    def purple(string) :
      return "\033[95m%s\033[0m"%string
    ################################################################################
    
    ################################################################################
    def red(string) :
      return "\033[91m%s\033[0m"%string
    ################################################################################
    
    ################################################################################
    def changeExtension(filename, newExtension) :
      return os.path.splitext(filename)[0] + newExtension
    ################################################################################
    
    ################################################################################
    def findInputFiles(path) :
      inputFiles = []
      for filename in os.listdir(path) :
        if os.path.splitext(filename)[1] == ".l" :
          inputFiles.append(filename)
      inputFiles.sort()
      return inputFiles
    ################################################################################
    
    ################################################################################
    def deleteCompilationOutputs() :
      outputExtensions = [".exe", ".exeout", ".o", ".out", ".sa", ".saout", ".sc", ".ts", ".nasm", ".nasmout", ".pre-nasm", ".pre-nasmout", ".c3a", ".c3aout", ".fg", ".fgs", ".ig"]
      for filename in os.listdir(inputPath) :
        if os.path.splitext(filename)[1] in outputExtensions :
          os.remove(inputPath+filename)
    ################################################################################
    
    ################################################################################
    def compileInputFiles(inputFiles) :
      evaluation = getNewEvaluationResult(ERR_SYNTAX_CODE)
      for inputFile in inputFiles :
        lightFilename = inputFile.split("/")[-1]
        print("Compiling %s..."%inputFile, file=outVerbose, flush=True, end='')
        returnCode = subprocess.Popen("{} {}{}".format(compiler(), inputPath, inputFile), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).wait()
        if returnCode == NOERROR_CODE :
          evaluation[1]["correct"].append(lightFilename)
          print("Done", file=outVerbose)
        else :
          evaluation[1]["incorrect"].append(lightFilename)
          print("ERROR !", file=outErr)
      print("", file=outVerbose)
      return evaluation
    ################################################################################
    
    ################################################################################
    def getNewEvaluationResult(name) :
      return [name, {"correct" : [], "incorrect" : [], "notfound" : []}]
    ################################################################################
    
    ################################################################################
    def evaluateCompilationErrors(inputFiles, expectedCode, name) :
      evaluation = getNewEvaluationResult(name)
    
      print(file=outVerbose)
    
      for inputFile in inputFiles :
        if not os.path.isfile(inputFile) :
          print("ERROR : could not find '%s'"%inputFile, file=outErr)
          exit(1)
        lightFilename = inputFile.split("/")[-1]
        print("Compiling %s..."%lightFilename, end="", file=outVerbose)
        returnCode = subprocess.Popen("{} {}".format(compiler(verbose=""), inputFile), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).wait()
        print("Done (%d)"%returnCode, file=outVerbose)
        if returnCode == expectedCode :
          evaluation[1]["correct"].append(lightFilename)
        else :
          evaluation[1]["incorrect"].append(lightFilename)
      print("", file=outVerbose)
    
      return evaluation
    ################################################################################
    
    ################################################################################
    def evaluateDiff(inputFiles, extension, extensionRef, path, name) :
      evaluation = getNewEvaluationResult(name)
    
      for filename in inputFiles :
        producedFile = changeExtension(filename, extension)
        if not os.path.isfile(inputPath+producedFile) :
          evaluation[1]["notfound"].append(producedFile)
          continue
        
        ref = refPath+path+changeExtension(producedFile, extensionRef)
        if not os.path.isfile(ref) :
          print("ATTENTION : Fichier non trouvé : %s"%ref, file=sys.stderr)
          continue
    
        refContent = []
        for line in open(ref, "r") :
          line = line.strip()
          if len(line) > 0 :
            refContent.append(line)
        producedContent = []
        for line in open(inputPath+producedFile, "r") :
          line = line.strip()
          if len(line) > 0 :
            producedContent.append(line)
    
        if producedContent == refContent :
          evaluation[1]["correct"].append(producedFile)
        else :
          evaluation[1]["incorrect"].append(producedFile)
    
      return evaluation
    ################################################################################
    
    ################################################################################
    def evaluateSa(inputFiles) :
      errs = []
      for filename in inputFiles :
        saFilename = inputPath+changeExtension(filename, ".sa")
        outFilename = inputPath+changeExtension(filename, ".saout")
        if not os.path.isfile(saFilename) :
          continue
    
        command = "java -classpath %s SaVM -sa %s > %s"%(findClasspath(), saFilename, outFilename)
        out = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).stderr.read().decode()
        if len(out) != 0 :
          errs.append("ERROR %s for input %s : '%s'"%(inspect.stack()[0][3], filename, out))
    
      return evaluateDiff(inputFiles, ".saout", ".out", "out-ref/", "Execution de sa"), errs
    ################################################################################
    
    ################################################################################
    def evaluateC3a(inputFiles) :
      errs = []
      for filename in inputFiles :
        c3aFilename = inputPath+changeExtension(filename, ".c3a")
        tsFilename = inputPath+changeExtension(filename, ".ts")
        outFilename = inputPath+changeExtension(filename, ".c3aout")
        if not os.path.isfile(c3aFilename) or not os.path.isfile(tsFilename) :
          continue
    
        command = "java -classpath %s C3aVM -c3a %s -ts %s > %s"%(findClasspath(), c3aFilename, tsFilename, outFilename)
        out = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).stderr.read().decode()
        if len(out) != 0 :
          errs.append("ERROR %s for input %s : '%s'"%(inspect.stack()[0][3], filename, out))
    
      return evaluateDiff(inputFiles, ".c3aout", ".out", "out-ref/", "Execution du c3a"), errs
    ################################################################################
    
    ################################################################################
    def evaluatePreNasm(inputFiles) :
      errs = []
      for filename in inputFiles :
        nasmFilename = inputPath+changeExtension(filename, ".pre-nasm")
        outFilename = inputPath+changeExtension(filename, ".pre-nasmout")
        if not os.path.isfile(nasmFilename) :
          continue
    
        command = "java -classpath %s NasmVM -nasm %s > %s"%(findClasspath(), nasmFilename, outFilename)
        out = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).stderr.read().decode()
        if len(out) != 0 :
          errs.append("ERROR %s for input %s : '%s'"%(inspect.stack()[0][3], filename, out))
    
      return evaluateDiff(inputFiles, ".pre-nasmout", ".out", "out-ref/", "Execution du pre-nasm"), errs
    ################################################################################
    
    ################################################################################
    def evaluateNasm(inputFiles) :
      errs = []
      for filename in inputFiles :
        nasmFilename = inputPath+changeExtension(filename, ".nasm")
        outFilename = inputPath+changeExtension(filename, ".nasmout")
        if not os.path.isfile(nasmFilename) :
          continue
    
        command = "java -classpath %s NasmVM -nasm %s > %s"%(findClasspath(), nasmFilename, outFilename)
        out = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).stderr.read().decode()
        if len(out) != 0 :
          errs.append("ERROR %s for input %s : '%s'"%(inspect.stack()[0][3], filename, out))
    
      return evaluateDiff(inputFiles, ".nasmout", ".out", "out-ref/", "Execution du nasm"), errs
    ################################################################################
    
    ################################################################################
    def evaluateExecutable(inputFiles) :
      errs = []
      for filename in inputFiles :
        nasmFilename = changeExtension(filename, ".nasm")
        objectFilename = changeExtension(filename, ".o")
        execFilename = changeExtension(filename, ".exe")
        outFilename = changeExtension(filename, ".exeout")
        if not os.path.isfile(inputPath+nasmFilename) :
          continue
    
        out = subprocess.Popen("cd {} && nasm -f elf -dwarf -g {}".format(inputPath+"..","input/"+nasmFilename), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).stderr.read()
        if not os.path.isfile(inputPath+objectFilename) :
          errs.append("ERROR %s for input %s : '%s'"%(inspect.stack()[0][3], filename, out))
          continue
        out = subprocess.Popen("ld -m elf_i386 -o {}{} {}{}".format(inputPath,execFilename,inputPath,objectFilename), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).stderr.read()
        if not os.path.isfile(inputPath+execFilename) :
          errs.append("ERROR %s for input %s : '%s'"%(inspect.stack()[0][3], filename, out))
          continue
        out = subprocess.Popen("{}{} > {}{}".format(inputPath,execFilename,inputPath,outFilename), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).stderr.read()
        if not os.path.isfile(inputPath+outFilename) :
          errs.append("ERROR %s for input %s : '%s'"%(inspect.stack()[0][3], filename, out))
          continue
    
      return evaluateDiff(inputFiles, ".exeout", ".out", "out-ref/", "Execution du binaire"), errs
    ################################################################################
    
    ################################################################################
    def printListElements(destination, elements, colorFunction, useColor, resultStr) :
      if len(elements) == 0 :
        return
      maxColumnSize = len(max(elements, key=len))
      for filename in elements :
        if useColor :
          print("\t{}".format(colorFunction(filename)), file=destination)
        else :
          print("\t{:{}} {}".format(filename, maxColumnSize+2, resultStr), file=destination)
    ################################################################################
    
    ################################################################################
    def printEvaluationResult(destination, evaluationResult, useColor) :
      name = evaluationResult[0]
      correct = evaluationResult[1]["correct"]
      incorrect = evaluationResult[1]["incorrect"]
      notfound = evaluationResult[1]["notfound"]
    
      nbCorrect = len(correct)
      nbTotal = len(correct) + len(incorrect) + len(notfound)
    
      score = 0.0
      if nbTotal > 0 :
        score = 100.0*nbCorrect/nbTotal
      for dest, color in [(destination, useColor), (open(outputFilename, "a"), False)] :
        print("Évaluation de %s :"%name, file=dest)
        print("{}/{} correct ({:6.2f}%)".format(nbCorrect, nbTotal, score), file=dest)
        if nbCorrect+len(incorrect) > 0 :
          printListElements(dest, correct, green, color, "CORRECT")
          printListElements(dest, incorrect, purple, color, "INCORRECT")
          printListElements(dest, notfound, red, color, "NON-EXISTANT")
      return score
    ################################################################################
    
    ################################################################################
    if __name__ == "__main__" :
      parser = argparse.ArgumentParser()
      parser.add_argument("--verbose", "-v", default=False, action="store_true",
        help="Verbose output (obsolete, verbose is default).")
      parser.add_argument("--silent", "-s", default=False, action="store_true",
        help="Less verbose output.")
      parser.add_argument("--noColors", default=False, action="store_true",
        help="Disable colors in output.")
      parser.add_argument("--clean", "-c", default=False, action="store_true",
        help="Clean input dir then exit.")
      parser.add_argument("--number", "-n", default=None, type=int,
        help="Restrict tests to n inputs.")
      parser.add_argument("--files", "-f", default=[], nargs="*",
        help="Specify input files.")
      args = parser.parse_args()
    
      args.verbose = not args.silent
    
      if args.verbose :
        outVerbose = outScore
      if args.clean :
          deleteCompilationOutputs()
          exit(0)
    
      with open(outputFilename, "w") as _ :
        pass
    
      inputFiles = args.files[:args.number]
      if len(inputFiles) == 0 :
        inputFiles = findInputFiles(inputPath)[:args.number]
    
      errInputFilesTs = [errInputPathTs+"/"+f for f in findInputFiles(errInputPathTs)[:args.number]]
      errInputFilesType = [errInputPathType+"/"+f for f in findInputFiles(errInputPathType)[:args.number]]
    
      deleteCompilationOutputs()
    
      compileCompiler()
    
    
      scores = []
      names = []
      errors = []
      names.append("SC           ")
      res = compileInputFiles(inputFiles)
      scores.append(printEvaluationResult(outVerbose, res, not args.noColors))
      msg = "Diff de %s"
      dfSa = lambda files : (evaluateDiff(inputFiles, ".sa", ".sa", "sa-ref/", msg%"sa"), [])
      dfTs = lambda files : (evaluateDiff(inputFiles, ".ts", ".ts", "ts-ref/", msg%"ts"), [])
      dfC3a = lambda files : (evaluateDiff(inputFiles, ".c3a", ".c3a", "c3a-ref/", msg%"c3a"), [])
      dfPreNasm = lambda files : (evaluateDiff(inputFiles, ".pre-nasm", ".pre-nasm", "pre-nasm-ref/", msg%"pre-nasm"), [])
      dfNasm = lambda files : (evaluateDiff(inputFiles, ".nasm", ".nasm", "nasm-ref/", msg%"nasm"), [])
    
      # Evaluation of normal (working) examples
      for evalTarget in [ 
                         ("SA-DIFF      ", dfSa),
                         ("TS-DIFF      ", dfTs),
                         ("SA           ", evaluateSa),
                         ("C3A-DIFF     ", dfC3a),
                         ("C3A          ", evaluateC3a),
                         ("PRE-NASM-DIFF", dfPreNasm),
                         ("PRE-NASM     ", evaluatePreNasm),
                         ("NASM-DIFF    ", dfNasm),
                         ("NASM         ", evaluateNasm),
                         ("EXE          ", evaluateExecutable),
                        ] :
        names.append(evalTarget[0])
        res, err = evalTarget[1](inputFiles)
        scores.append(printEvaluationResult(outVerbose, res, not args.noColors))
        errors += err
    
      # Evaluation of ill formed examples, that should trigger a compilation error
      for evalTarget in [
                         ("ERR_TS       ", evaluateCompilationErrors, errInputFilesTs, ERR_SYMBOL_TABLE_CODE),
                         ("ERR_TYPE     ", evaluateCompilationErrors, errInputFilesType, ERR_TYPE_CODE),
                        ] :
        names.append(evalTarget[0])
        res = evalTarget[1](evalTarget[2], evalTarget[3], evalTarget[0])
        scores.append(printEvaluationResult(outVerbose, res, not args.noColors))
    
      deleteClasses()
    
      # The rest of the script will now print the results
    
      if not args.noColors :
        print("Légende : {}  {}  {}".format(green("CORRECT"), purple("INCORRECT"), red("NON-EXISTANT")), file=outVerbose)
    
      errorsStr = ""
    
      if len(errors) > 0 :
        errorsStr = "%s\n%s"%((30*"-")+("EVALUATION ERRORS")+(30*"-"), "\n\n".join(errors))
    
      print(errorsStr)
      print(errorsStr, file=open(outputFilename, "a"))
    
      for i in range(len(scores)):
        print(names[i], "\t", scores[i])
      
      print("Copy-pastable scores :", file=outScore)
      print("\t".join(names), file=outScore)
      print("\t".join(["%6.2f"%scores[i] for i in range(len(scores))]), file=outScore)
    
      print("\nSauvegardé dans le fichier %s"%outputFilename)
    
    # les coefficients associés à chaque partie pour le calcul de la note finale
      coeffs = [SC_COEFF, SA_DIFF_COEFF, TS_DIFF_COEFF, SA_COEFF, C3A_DIFF_COEFF, C3A_COEFF, PRE_NASM_DIFF, PRE_NASM_COEFF, NASM_DIFF_COEFF, NASM_COEFF, EXE_COEFF, ERR_TS_COEFF, ERR_TYPE_COEFF]
    
    # calcul de la note finale
    
      note = 0
      for i in range(len(scores)):
        note += scores[i] / 100 * coeffs[i]
      print ("note = ",  note)
    
    ################################################################################