Compute entropy in chunks

src/chunk_utils.py

@@ -0,0 +1,28 @@
+from src.disk_utils import load_pattern_dict
+
+CHUNK_SIZE = 5000
+
+
+def chunks(lst, n=CHUNK_SIZE):
+    """Yield successive n-sized chunks from lst.
+    Reference: https://stackoverflow.com/a/312464/376454
+    """
+    for i in range(0, len(lst), n):
+        yield lst[i: i + n]
+
+
+def get_num_chunks(lst):
+    return int(len(lst) / CHUNK_SIZE) + 1
+
+
+def find_chunk_no(large_list, element):
+    for chunk_no, small_list in enumerate(chunks(large_list), start=1):
+        if element in small_list:
+            return chunk_no
+    return None
+
+
+def get_pattern(all_dictionary, guess_word):
+    chunk_no = find_chunk_no(all_dictionary, guess_word)
+    pattern_dict = load_pattern_dict(chunk_no)
+    return pattern_dict[guess_word]

src/disk_utils.py

@@ -0,0 +1,22 @@
+import pickle
+
+
+def get_pattern_dict_fname(chunk_no=None):
+    if chunk_no is None:
+        suffix = ''
+    else:
+        suffix = f'_{chunk_no}'
+    return f'pattern_dict{suffix}.p'
+
+
+def load_pattern_dict(chunk_no=None):
+    """Load the cache.
+    """
+    fname = get_pattern_dict_fname(chunk_no)
+    return pickle.load(open(fname, 'rb'))
+
+
+def save_pattern_dict(pattern_dict, chunk_no=None):
+    fname = get_pattern_dict_fname(chunk_no)
+    pickle.dump(pattern_dict, open(fname, 'wb+'))
+    return

wordle.py

@@ -2,10 +2,11 @@
 import os
 import itertools
 import random
-import pickle
+from src.disk_utils import get_pattern_dict_fname, load_pattern_dict, save_pattern_dict
 from tqdm import tqdm
 from scipy.stats import entropy
 from collections import defaultdict, Counter
+from src.chunk_utils import chunks, get_num_chunks, get_pattern
 
 N_GUESSES = 10
 DICT_FILE_all = 'all_words.txt'
@@ -67,6 +68,20 @@ def calculate_entropies(words, possible_words, pattern_dict, all_patterns):
     return entropies
 
 
+def calculate_entropies_in_chunks(all_words, all_patterns, num_chunks, filter_candidates = False):
+    entropies = {}
+    for chunk_no in range(1, num_chunks + 1):
+        pattern_dict = load_pattern_dict(chunk_no)
+
+        candidates = list(pattern_dict.keys())
+        if filter_candidates:
+            candidates = list(set(candidates).intersection(all_words))
+        chunk_entropies = calculate_entropies(candidates, all_words, pattern_dict, all_patterns)
+        entropies.update(chunk_entropies)
+
+    return entropies
+
+
 def main():
     # load all 5-letter-words for making patterns 
     with open(DICT_FILE_all) as ifp:
@@ -84,12 +99,14 @@ def main():
     # Generate the possible patterns of information we can get
     all_patterns = list(itertools.product([0, 1, 2], repeat=WORD_LEN))
 
-    # Calculate the pattern_dict and cache it, or load the cache.
-    if 'pattern_dict.p' in os.listdir('.'):
-        pattern_dict = pickle.load(open('pattern_dict.p', 'rb'))
-    else:
-        pattern_dict = generate_pattern_dict(all_dictionary)
-        pickle.dump(pattern_dict, open('pattern_dict.p', 'wb+'))
+    num_chunks = get_num_chunks(all_dictionary)
+    for chunk_no, dictionary_chunk in enumerate(chunks(all_dictionary), start=1):
+        fname = get_pattern_dict_fname(chunk_no)
+        print(f'[{chunk_no}/{num_chunks}] Processing {fname}')
+        if not os.path.exists(fname):
+            # Calculate the pattern_dict and cache it
+            pattern_dict = generate_pattern_dict(dictionary_chunk)
+            save_pattern_dict(pattern_dict, chunk_no)
 
     # Simulate games
     stats = defaultdict(list)
@@ -100,21 +117,18 @@ def main():
             guess_word = 'tares'
             all_words = set(all_dictionary)
             info = calculate_pattern(guess_word, WORD_TO_GUESS)
-            words = pattern_dict[guess_word][info]
+            words = get_pattern(all_dictionary, guess_word)[info]
             all_words = all_words.intersection(words)
             init_round = 1
         else:
             all_words = set(all_dictionary)
             init_round = 0
 
         for n_round in range(init_round, N_GUESSES):
-
-            candidates = all_dictionary
-            entropies = calculate_entropies(candidates, all_words, pattern_dict, all_patterns)
+            entropies = calculate_entropies_in_chunks(all_words, all_patterns, num_chunks, filter_candidates=False)
 
             if max(entropies.values()) < 0.1:
-                candidates = all_words
-                entropies = calculate_entropies(candidates, all_words, pattern_dict, all_patterns)
+                entropies = calculate_entropies_in_chunks(all_words, all_patterns, num_chunks, filter_candidates=True)
 
             # Guess the candiate with highest entropy
             guess_word = max(entropies.items(), key=lambda x: x[1])[0]
@@ -128,7 +142,7 @@ def main():
                 break
 
             # Filter our list of remaining possible words
-            words = pattern_dict[guess_word][info]
+            words = get_pattern(all_dictionary, guess_word)[info]
             all_words = all_words.intersection(words)
 
 if __name__ == "__main__":