prevents duplicates in simstring database

Georgetown-IR-Lab · Sep 9, 2016 · e4e0c11 · e4e0c11
1 parent 021f914
commit e4e0c11
Showing 1 changed file with 6 additions and 2 deletions.
diff --git a/install.py b/install.py
@@ -92,11 +92,15 @@ def parse_and_encode_ngrams(extracted_it, simstring_dir, cuisty_dir):
     mkdir(cuisty_dir)
 
     ss_db = SimstringDBWriter(simstring_dir)
-
     cuisty_db = CuiSemTypesDB(cuisty_dir)
 
+    simstring_terms = set()
+
     for i, (term, cui, stys, preferred) in enumerate(extracted_it, start=1):
-        ss_db.insert(term)
+        if term not in simstring_terms:
+            ss_db.insert(term)
+            simstring_terms.add(term)
+
         cuisty_db.insert(term, cui, stys, preferred)