qubo_lab/comparisonStats.py

#!/usr/bin/env python3

import argparse
import os
import glob
import json
import numpy as np
import matplotlib.pyplot as plt
import configparser
import scriptUtils

def main():
    args = __parseArguments()
    
    __stats(args["comparisonDir"], args["outputDir"])


def __parseArguments():
    parser = scriptUtils.ArgParser()
    
    parser.addInstanceDirArg()
    
    parser.addArg(alias="comparisonDir", shortFlag="c", longFlag="comparison_dir",
                  help="the direcotry with all comparison files", type=str)
    
    parser.addArg(alias="outputDir", shortFlag="s", longFlag="comparison_stats_dir",
                  help="Directory to store the stats", type=str)
                  
    arguments = parser.parse()
    
    arguments["datasetDir"] = os.path.abspath(arguments["datasetDir"])
    
    arguments["comparisonDir"] = os.path.join(arguments["datasetDir"],
                                              arguments["comparisonDir"])
                                              
    arguments["outputDir"] = os.path.join(arguments["datasetDir"],
                                          arguments["outputDir"])
    
    return arguments

def __stats(comparisonDir, outputDir):
    runs = glob.glob(os.path.join(comparisonDir, "run*"))
    
    for run in runs:
        
        stats = __collectStats(run)
    
        print(stats)
    
        runOutputDir = os.path.join(outputDir, os.path.basename(run))
        
        __writeStats(stats, runOutputDir)
    
def __collectStats(comparisonDir):
    files = glob.glob(os.path.join(comparisonDir, "*.cmp"))
    
    stats = {}
    stats["match"] = {"count": 0,
                      "instances": []}
    stats["false_positive"] = {"count": 0,
                               "instances": []}
    stats["false_negative"] = {"count": 0,
                               "instances": []}
    stats["unsat"] = {"count": 0,
                      "instances": []}
    
    for path in files:
        
        comparison = __readComparison(path)
        
        minisat_satisfiable = comparison["minisat_satisfiable"]
        qubo_satisfiable = comparison["qubo_satisfiable"]
        
        instanceName = str(os.path.basename(path)).split(".")[0]
        
        if minisat_satisfiable == qubo_satisfiable:
            stats["match"]["count"] += 1
            stats["match"]["instances"].append(instanceName)
            
        elif minisat_satisfiable == False and qubo_satisfiable == True:
            stats["false_positive"]["count"] += 1
            stats["false_positive"]["instances"].append(instanceName)
            
        elif minisat_satisfiable == True and qubo_satisfiable == False:
            stats["false_negative"]["count"] += 1
            stats["false_negative"]["instances"].append(instanceName)
            
        if not minisat_satisfiable:
            stats["unsat"]["count"] += 1
            stats["unsat"]["instances"].append(instanceName)
            
    return stats
        
def __readComparison(path):
    cmpFile = open(path, "r")
    comparison = json.load(cmpFile)
    cmpFile.close()
    
    return comparison
    
    
def __writeStats(stats, outputDir):
    if not os.path.exists(outputDir):
        os.makedirs(outputDir)
    
    with open(os.path.join(outputDir,"statusCollection"), "w+") as statusFile:
        statusFile.write(json.dumps(stats))
    
    fig = plt.figure()
    ax = fig.add_subplot(111)
    
    matchCount = stats["match"]["count"]
    falseNegativeCount = stats["false_negative"]["count"]
    falsePositiveCount = stats["false_positive"]["count"]
    
    numInstances = matchCount + falseNegativeCount + falsePositiveCount
    
    matchBar = ax.bar(x=0, height=matchCount)
    
    falsePositiveBar = ax.bar(x=1, height=falsePositiveCount)
    falseNegativeBar = ax.bar(x=1,
                               height=falseNegativeCount,
                               bottom=falsePositiveCount)
                               
    ax.axhline(y=matchCount, linestyle="--", color="gray")
    ax.axhline(y=falseNegativeCount, linestyle="--", color="gray")
    
    
    plt.ylabel("SAT Instanzen")
    plt.title("Verlgeich Minisat / WMIS qubo mit qbsolv")
    plt.xticks([0, 1], ("Gleiches Ergebnis", "Unterschiedliches Ergebnis"))
    plt.yticks([0, matchCount, falseNegativeCount, numInstances])
    plt.legend((matchBar, falsePositiveBar, falseNegativeBar),
               ("Gleiches Ergebnis",
                "False Positive",
                "False Negative"))
    
    plt.savefig(os.path.join(outputDir, "stats.png"))
    
if __name__ == "__main__":
    main()