Merge pull request #128 from dongxinEric/bugfix/fix-discrepancies-between-MM-and-DocLayer

Fixed several bugs in the MM code around index and unique index.

Author: apkar

test/correctness/document-correctness.py

@@ -333,23 +333,31 @@ def _run_operation_(op1, op2):
         transactional_shim.remove(collection2)
 
         indexes = []
+        num_of_indexes = 5;
         indexes_first = gen.global_prng.choice([True, False])
         if indexes_enabled:
-            for i in range(0, 5):
+            for i in range(0, num_of_indexes):
                 index_obj = gen.random_index_spec()
                 indexes.append(index_obj)
 
+        # 0.5% likelyhood to allow using unique index in this iteration, assuming a uniform distribution
+        useUnique = (gen.global_prng.randint(1,200) == 1)
+        # only allow one out of $num_of_indexes to be unique.
+        allowed_ii = gen.global_prng.randint(1,num_of_indexes)
         if indexes_first:
+            ii = 1
             for i in indexes:
-                # When we do enable make sure we don't enable for 50% cases. That way unique indexes will cloud everything else. Make it something like 5-10%.
-                # uniqueIndex = gen.global_prng.choice([True, False])
-                uniqueIndex = False
+                if ii == allowed_ii:
+                    uniqueIndex = useUnique
+                else:
+                    uniqueIndex = False
                 okay = _run_operation_(
                     (transactional_shim.ensure_index, (collection1, i), {"unique":uniqueIndex}),
                     (transactional_shim.ensure_index, (collection2, i), {"unique":uniqueIndex})
                     )
                 if not okay:
                     return (okay, fname, None)
+                ii += 1
         docs = []
         for i in range(0, num_doc):
             doc = gen.random_document(True)
@@ -364,17 +372,20 @@ def _run_operation_(op1, op2):
             return (okay, fname, None)
 
         if not indexes_first:
+            ii = 1
             for i in indexes:
-                # When we do enable make sure we don't enable for 50% cases. That way unique indexes will cloud everything else. Make it something like 5-10%.
-                # uniqueIndex = gen.global_prng.choice([True, False])
-                uniqueIndex = False
+                if ii == allowed_ii:
+                    uniqueIndex = useUnique
+                else:
+                    uniqueIndex = False
                 okay = _run_operation_(
                     (transactional_shim.ensure_index, (collection1, i), {"unique":uniqueIndex}),
                     (transactional_shim.ensure_index, (collection2, i), {"unique":uniqueIndex})
                     )
                 if not okay:
                     print "Failed when adding index after insert"
                     return (okay, fname, None)
+                ii += 1
 
         okay = check_query(dict(), collection1, collection2)
         if not okay:

test/correctness/mongo_model.py

@@ -546,16 +546,23 @@ def _create_index(self, keys, kwargs):
         def _get_index_name(keys):
             return "_".join(["%s_%s" % item for item in keys])
         kwargs.setdefault("name", _get_index_name(keys))
+        seen = set()
+        deduplicatedKeys = []
+        for key in keys:
+            if key[0] not in seen:
+                deduplicatedKeys.append(key)
+                seen.add(key[0])
         for i in self.indexes:
-            if i.keys == keys:
+            if i.keys == deduplicatedKeys:
+                # There was an index that indexes on the same field(s), abort here.
                 return None
             if i.name == kwargs["name"]:
                 raise MongoModelException("There is an index with this name and a different key spec", code=29993)
 
         if "unique" in kwargs.keys() and kwargs["unique"]:
-            newIndex = MongoUniqueIndex(keys, kwargs)
+            newIndex = MongoUniqueIndex(deduplicatedKeys, kwargs)
         else:
-            newIndex = MongoIndex(keys, kwargs)
+            newIndex = MongoIndex(deduplicatedKeys, kwargs)
 
         self.indexes.append(newIndex) # insert first, since an index can be added but in error state if its constraints are violated.
         newIndex.build(self.data.values())
@@ -1095,10 +1102,29 @@ class MongoIndex(object):
     def __init__(self, indexKeys, kwargs):
         self.name = kwargs["name"]
         if len(indexKeys) == 1:
+            if indexKeys[0][1] == "text" :
+                raise MongoModelException("Document Layer does not support this index type, yet.", code=29969)
+            elif (type(indexKeys[0][1]) is int and abs(indexKeys[0][1]) != 1) or (type(indexKeys[0][1]) is not int):
+                raise MongoModelException("Index must be 1 for ascending or -1 for descending", code=29991)
+        else :
+            first = True
+            ascending = True
+            for key in indexKeys:
+                if key[1] == "text":
+                    raise MongoModelException("Document Layer does not support this index type, yet.", code=29969)
+                elif (type(key[1]) is int and abs(key[1]) != 1) or (type(key[1]) is not int):
+                    raise MongoModelException("Index must be 1 for ascending or -1 for descending", code=29991)
+                if first:
+                    first = False
+                    ascending = (key[1] == 1)
+                else:
+                    if ascending != (key[1] == 1):
+                        raise MongoModelException("Mixed order compound indexes are not supported", code=29990)
+        self.keys = indexKeys
+        if len(self.keys) == 1:
             self.isSimple = True
         else:
             self.isSimple = False
-        self.keys = indexKeys
         # self check
         self.validate_self()
 
@@ -1119,7 +1145,7 @@ def validate_and_build_entry(self, documents):
                 nValues = 1
                 for key in self.keys:
                     values = expand(
-                        field=key,
+                        field=key[0],
                         document=document,
                         check_last_array=True,
                         expand_array=True,
@@ -1144,20 +1170,23 @@ def build(self, documents):
     def validate_and_build_entry(self, documents):
         seen = set()
         for document in documents:
+            entry = ""
             for key in self.keys:
-                entry = ""
                 _values = expand(
-                    field=key,
+                    field=key[0],
                     document=document,
                     check_last_array=True,
                     expand_array=True,
                     add_last_array=False,
                     check_none_at_all=True,
                     check_none_next=True,
                     debug=False)
-
-                entry = entry + reduce((lambda acc, x: acc + x), map((lambda x: str(x)), _values), "None")
-                if entry in seen:
-                    raise MongoModelException("Duplicated value not allowed by unique index", code=11000)
-                else:
-                    seen.add(entry)
+                entry = entry + reduce((lambda acc, x: acc + x), map((lambda x: str(x)), _values), "")
+            if entry in seen:
+                # print "[{}] in {} ? {}".format(entry, seen, entry in seen)
+                # print "Violating keys " + str(key)
+                # print "Duplicated value: " + entry
+                raise MongoModelException("Duplicated value not allowed by unique index", code=11000)
+            else:
+                # print "Inserting {} for key {}".format(entry, key)
+                seen.add(entry)