queries: switched to lazy loding

queryInstancesFromScope.js

@@ -1,6 +1,6 @@
 db.getCollection("experiment_scopes").aggregate([
 {
-    $match: {_id: "test"}
+    $match: {_id: "c45_v[5-45]_1"}
 },
 {
     $unwind : "$instances"

query_instances_from_scope_ids.js

@@ -0,0 +1,23 @@
+db.getCollection("experiment_scopes").aggregate([
+{
+    $match: {_id: "c45_v[5-45]_1"}
+},
+{
+    $unwind : "$instances"
+},
+{
+    $lookup:
+    {
+        from: "instances",
+        localField: "instances",
+        foreignField: "_id",
+        as: "instance"
+    }
+},
+{
+    $unwind: "$instance"
+},
+{
+    $project: {instance_id: "$instance._id"}
+}
+])

query_req_data_for_solving.js

@@ -1,7 +1,7 @@
 db.getCollection('experiment_scopes').aggregate(
 [
 {
-    $match: {_id: "c50_v[5, 50]_1"}
+    $match: {_id: "c45_v[5-45]_1"}
 },
 {
     $unwind: "$instances"
@@ -48,10 +48,6 @@ db.getCollection('experiment_scopes').aggregate(
         {
             $project:
             {
-                /*_id: false,
-                qubo: false,
-                solver_graph: false,
-                description: false*/
                 embeddings: true
             }
         }       
@@ -74,4 +70,5 @@ db.getCollection('experiment_scopes').aggregate(
         embeddings: "$embeddings.embeddings"
     }
 }
+*/
 ])

query_req_data_for_solving_ids.js

@@ -0,0 +1,83 @@
+db.getCollection('experiment_scopes').aggregate(
+[
+{
+    $match: {_id: "c42_v[5-42]_1"}
+},
+{
+    $unwind: "$instances"
+},
+{
+    $project: {"instance_id": "$instances"}
+},
+{
+    $lookup:
+    {
+        from: "wmis_qubos",
+        localField: "instance_id",  
+        foreignField: "instance",
+        as: "wmis_qubo"
+    }
+},
+{
+    $unwind: "$wmis_qubo"
+},
+{
+    $project:
+    {
+        instance_id: true,
+        wmis_qubo_id: "$wmis_qubo._id"
+    }
+},
+{
+    $lookup:
+    {
+        from: "embeddings",
+        let:
+        {
+            qubo_id: "$wmis_qubo_id",
+            solver_graph_id: ObjectId("5c9e13168c6fe628c16eb361")
+        },
+        pipeline:
+        [
+        {
+            $project:
+            {
+                qubo: true,
+                solver_graph: true,
+            }
+        },
+        {
+            $match:
+            {
+                $expr:
+                {
+                    $and:
+                    [
+                    {$eq: ["$qubo", "$$qubo_id"]},
+                    {$eq: ["$solver_graph", "$$solver_graph_id"]}
+                    ]
+                }
+            }
+        },
+        {
+            $project:
+            {
+                _id: true
+            }
+        }       
+        ],
+        as: "embeddings_id"
+    }
+},
+{
+    $unwind: "$embeddings_id"
+},
+{
+    $project:
+    {
+        instance_id: true,
+        wmis_qubo_id: true,
+        embeddings_id: "$embeddings_id._id"
+    }
+}
+])

test_data_extraction.py

@@ -6,14 +6,15 @@ import dimod
 
 def main():
     wmis_siman_results_alpha_num_of_assignments()
+    #wmis_siman_results()
     
 def wmis_siman_results():
     db = script.connect_to_instance_pool()
     
     q = queries.WMIS_result_scope_query_raw(db)
-    q.query("c50_v[5, 50]_1", "wmis_siman_results")
+    q.query("c45_v[5-45]_1", "wmis_siman_results")
     
-    for i in range(30):
+    for i in range(2):
         result = q.__next__()
         
         sample_set = queries.read_raw_wmis_sample_set(result["data"])
@@ -29,14 +30,17 @@ def wmis_siman_results_alpha_num_of_assignments():
     idb = script.connect_to_instance_pool()
     
     q = queries.WMIS_result_scope_query_raw(idb)
-    q.query("c50_v[5, 50]_1", "wmis_siman_results")
+    q.query("c42_v[5-42]_1", "wmis_siman_results")
     
-    insert_row = ("INSERT INTO c50_v5to50_1_wmis_alpha_number_of_assignments "
+    insert_row = ("INSERT INTO c42_v5to42_1_wmis_results "
                   "(result_id, "
                   " number_of_clauses, "
                   " number_of_variables, "
-                  " number_of_found_assignments) "
-                  "VALUES (%s, %s, %s, %s) ")
+                  " number_of_found_assignments, "
+                  " chain_break_fraction, "
+                  " num_occurrences, "
+                  " energy) "
+                  "VALUES (%s, %s, %s, %s, %s, %s, %s) ")
     
     for result in q:
         sample_set = queries.read_raw_wmis_sample_set(result["data"])
@@ -48,7 +52,10 @@ def wmis_siman_results_alpha_num_of_assignments():
         edb_cursor.execute(insert_row, (str(result["_id"]),
                                         int(sat.getNumberOfClauses()),
                                         int(sat.getNumberOfVariables()),
-                                        int(data["number_of_assignments"])))
+                                        int(data["number_of_assignments"]),
+                                        float(data["chain_break_fraction"]),
+                                        int(data["num_occurrences"]),
+                                        int(data["energy"])))
         
         print(data)
         

test_sat_to_qubo_workflow.py

@@ -57,13 +57,13 @@ def __wmis2():
     print(res.first)
     
 def __wmis3():    
-    db = script.connect_to_instance_pool("dbc")
+    db = script.connect_to_instance_pool()
     target_graph =  dnx.chimera_graph(16, 16, 4)
     target_graph_id = queries.get_id_of_solver_graph(db["solver_graphs"],
                                                      nx.node_link_data(target_graph))
     
     solver_input_query = queries.WMIS_solver_input_scope_query(db)
-    solver_input_query.query("c50_v[5, 50]_1", target_graph_id)
+    solver_input_query.query("c42_v[5-42]_1", target_graph_id)
     
     base_sampler = SimulatedAnnealingSampler()
     chimera_sampler = dimod.StructureComposite(base_sampler,

util/queries.py

@@ -7,6 +7,8 @@ class Instance_scope_query:
     def __init__(self, database):
         self.__database = database
         self.__query = None
+        self.__instance_ids = []
+        self.__instance_id_iterator = None;
         
     def query(self, scope):
         self.__query = self.__database["experiment_scopes"].aggregate([
@@ -29,15 +31,23 @@ class Instance_scope_query:
                 "$unwind": "$instance"
             },
             {
-                "$replaceRoot": {"newRoot": "$instance"}
+                "$project": {"instance_id": "$instance._id"}
             }
         ])
+        
+        self.__instance_ids = []
+        for doc in self.__query:
+            self.__instance_ids.append(doc["instance_id"])
+        
+        self.__instance_id_iterator = iter(self.__instance_ids)
     
     def __iter__(self):
         return self
     
     def __next__(self):
-        document = self.__query.next()
+        instance_id = self.__instance_id_iterator.__next__()
+        
+        document = self.__database["instances"].find_one({"_id": instance_id})
         
         return self.__document_to_sat(document)
     
@@ -54,6 +64,8 @@ class WMIS_scope_query_raw:
     def __init__(self, database):
         self.__database = database
         self.__query = None
+        self.__qubo_ids = []
+        self.__qubo_id_iterator = None
     
     def query(self, scope):
         self.__query = self.__database["experiment_scopes"].aggregate([
@@ -76,15 +88,26 @@ class WMIS_scope_query_raw:
                 "$unwind": "$qubo"
             },
             {
-                "$replaceRoot": {"newRoot": "$qubo"}
+                "$project": {"qubo_id": "$qubo._id"}
             }
         ])
+        
+        self.__qubo_ids = []
+        for doc in self.__query:
+            self.__qubo_ids.append(doc["qubo_id"])
+        
+        self.__qubo_id_iterator = iter(self.__qubo_ids)
+            
+    def __len__(self):
+        return self.query.count_documents({})
             
     def __iter__(self):
         return self
     
     def __next__(self):
-        return self.__query.next()
+        qubo_filter = {"_id": self.__qubo_id_iterator.__next__()}
+        
+        return self.__database["wmis_qubos"].find_one(qubo_filter)
     
 class WMIS_scope_query (WMIS_scope_query_raw):
     
@@ -99,6 +122,8 @@ class WMIS_solver_input_scope_query_raw:
     def __init__(self, database):
         self.__database = database
         self.__query = None
+        self.__ids = []
+        self.__id_iterator = None;
         
     def query(self, scope, solver_graph_id):
         self.__query = self.__database["experiment_scopes"].aggregate([
@@ -123,17 +148,31 @@ class WMIS_solver_input_scope_query_raw:
             {
                 "$unwind": "$wmis_qubo"
             },
+            {
+                "$project":
+                {
+                    "instance_id": True,
+                    "wmis_qubo_id": "$wmis_qubo._id"
+                }
+            },
             {
                 "$lookup":
                 {
                     "from": "embeddings",
                     "let":
                     {
-                        "qubo_id": "$wmis_qubo._id",
+                        "qubo_id": "$wmis_qubo_id",
                         "solver_graph_id": bson.ObjectId(solver_graph_id)
                     },
                     "pipeline":
                     [
+                    {
+                        "$project":
+                        {
+                            "qubo": True,
+                            "solver_graph": True,
+                        }
+                    },
                     {
                         "$match":
                         {
@@ -148,34 +187,55 @@ class WMIS_solver_input_scope_query_raw:
                         }
                     },
                     {
-                        "$project": { "list": "$embeddings" }
-                    }
+                        "$project":
+                        {
+                            "_id": True
+                        }
+                    }       
                     ],
-                    "as": "embeddings"
+                    "as": "embeddings_id"
                 }
             },
             {
-                "$match": {"embeddings": {"$exists": True, "$not": {"$size": 0}}}
-            },
-            {
-                "$unwind": "$embeddings"
+                "$unwind": "$embeddings_id"
             },
             {
                 "$project":
                 {
-                    "_id": False,
                     "instance_id": True,
-                    "wmis_qubo": True,
-                    "embeddings": True
+                    "wmis_qubo_id": True,
+                    "embeddings_id": "$embeddings_id._id"
                 }
             }
         ])
+                
+        self.__ids = []
+        for doc in self.__query:
+            self.__ids.append({
+                "instance_id": doc["instance_id"],
+                "wmis_qubo_id": doc["wmis_qubo_id"],
+                "embeddings_id": doc["embeddings_id"]
+            })
+        
+        self.__id_iterator = iter(self.__ids)
             
     def __iter__(self):
         return self
     
     def __next__(self):
-        return self.__query.next()
+        ids = self.__id_iterator.__next__()
+        
+        doc = {}
+        
+        doc["instance_id"] = ids["instance_id"]
+        
+        qubo_filter = {"_id": ids["wmis_qubo_id"]}
+        doc["wmis_qubo"] = self.__database["wmis_qubos"].find_one(qubo_filter)
+        
+        embeddings_filter = {"_id": ids["embeddings_id"]}
+        doc["embeddings"] = self.__database["embeddings"].find_one(embeddings_filter)
+        
+        return doc
     
 class WMIS_solver_input_scope_query (WMIS_solver_input_scope_query_raw):
     
@@ -192,7 +252,7 @@ class WMIS_solver_input_scope_query (WMIS_solver_input_scope_query_raw):
         data["embeddings_id"] = doc["embeddings"]["_id"]
         
         data["embeddings"] = []
-        for raw_emb in doc["embeddings"]["list"]:
+        for raw_emb in doc["embeddings"]["embeddings"]:
             data["embeddings"].append(read_raw_embedding(raw_emb))
             
         return data

util/random_instance_pool.py

@@ -11,11 +11,17 @@ class Random_instance_pool:
         return self
             
     def __next__(self):
-        params = self.__parameter_range.next()
+        return self.Random(self.__parameter_range.next())
+    
+    class Random:
+        def __init__(self, parameters):
+            self.__params = parameters
+        
+        def random(self):
+            return randomSAT.generateRandomKSAT(self.__params.number_of_clauses,
+                                                self.__params.number_of_variables,
+                                                self.__params.variables_per_clause)
         
-        return randomSAT.generateRandomKSAT(params.number_of_clauses,
-                                            params.number_of_variables,
-                                            params.variables_per_clause)
     
     
 class Instance_parameters:

util/script.py

@@ -241,19 +241,43 @@ def find_wmis_embeddings_for_scope(db, scope, solver_graph):
     qubos = queries.WMIS_scope_query(db)
     qubos.query(scope)
     
+    new_embeddings_found = 0
+    already_found = 0
+    total_count = 0
     for qubo, qubo_id in tqdm(qubos):
-        if not __embedding_entry_exists(db["embeddings"], qubo_id, solver_graph_id):
-            nx_qubo = graph.qubo_to_nx_graph(qubo)
-            
-            emb = minorminer.find_embedding(nx_qubo.edges(),
-                                            solver_graph.edges(),
-                                            return_overlap=True)
-            
-            if emb[1] == 1:
-                write_wmis_embedding_to_pool_db(db["embeddings"],
-                                                qubo_id,
-                                                solver_graph_id,
-                                                emb[0])
+        total_count += 1
+        
+        max_no_improvement = 10
+        for i in range(5):
+            if __embedding_entry_exists(db["embeddings"], qubo_id, solver_graph_id):
+                already_found += 1
+                break;
+            else: 
+                nx_qubo = graph.qubo_to_nx_graph(qubo)
+                
+                emb = minorminer.find_embedding(nx_qubo.edges(),
+                                                solver_graph.edges(),
+                                                return_overlap=True,
+                                                max_no_improvement=max_no_improvement)
+                
+                if emb[1] == 1:
+                    write_wmis_embedding_to_pool_db(db["embeddings"],
+                                                    qubo_id,
+                                                    solver_graph_id,
+                                                    emb[0])
+                    new_embeddings_found += 1
+                    
+            max_no_improvement *= 1.5
+    
+    percentage = 0
+    
+    if total_count > 0:
+        percentage = round(((new_embeddings_found + already_found) / total_count) * 100)
+    
+    print("found {} of {} embeddigns ({}%)".format(new_embeddings_found + already_found,
+                                                   total_count,
+                                                   percentage))
+    print("{} new embeddigns found".format(new_embeddings_found))
 
 def save_simulated_annealing_result(collection, result, solver_input, emb_list_index):
     doc = {}
@@ -271,5 +295,8 @@ def analyze_wmis_sample(sample):
     data = {}
     
     data["number_of_assignments"] = np.count_nonzero(list(sample.sample.values()))
+    data["chain_break_fraction"] = sample.chain_break_fraction
+    data["num_occurrences"] = sample.num_occurrences
+    data["energy"] = sample.energy
     
     return data