Merge remote-tracking branch 'origin/master'

combined_bot_main.py

@@ -12,6 +12,7 @@ from fruit import Fruit
 from pickaxe import Pickaxe_Field
 from litris import Litris
 from menti_words import MentiWords
+from flappy import Flappy
 
 
 def run():
@@ -73,6 +74,10 @@ def run():
         elif overlay.rb_int.get() == 11:
             menti = MentiWords(overlay)
             menti.execute_main_loop()
+        elif overlay.rb_int.get() == 12:
+            flappy = Flappy(overlay)
+            flappy.execute_main_loop()
+            flappy.flappy_pos_disc.run_mode = 'stop'
 
 if __name__ == "__main__":
     run()

combined_user_interface.py

@@ -33,7 +33,7 @@ class PrimaryOverlay(threading.Thread):
         self.Emitter_Box = ttk.Combobox
 
         self.RadioButtons = dict
-        self.RadioButtonNames = ["Equip", "Crops", "Farm", "Magic", "Craft", "Mine", "Fruit", "Sodo", "PAxe", "Ltris", "Menti"]
+        self.RadioButtonNames = ["Equip", "Crops", "Farm", "Magic", "Craft", "Mine", "Fruit", "Sodo", "PAxe", "Ltris", "Menti", "Flapp"]
         self.RadioButton1 = tk.Radiobutton
         self.RadioButton2 = tk.Radiobutton
         self.RadioButton3 = tk.Radiobutton
@@ -45,12 +45,13 @@ class PrimaryOverlay(threading.Thread):
         self.RadioButton9 = tk.Radiobutton
         self.RadioButton10 = tk.Radiobutton
         self.RadioButton11 = tk.Radiobutton
+        self.RadioButton12 = tk.Radiobutton
 
         self.StartButton = tk.Button
         self.StopButton = tk.Button
         self.PauseButton = tk.Button
         self.QuitButton = tk.Button
-        self.TkPosition = '133x429+60+600'
+        self.TkPosition = '133x454+60+600'
         self.setDaemon(True)
         self.StatusLabel = tk.Label
 
@@ -78,7 +79,7 @@ class PrimaryOverlay(threading.Thread):
         self.rb_int = tk.IntVar(self.root, value=1)
         self.RadioButtons = dict()
         # var = tk.IntVar(value=1)
-        for i in range(1, 12):
+        for i in range(1, 13):
             self.RadioButtons[i] = tk.Radiobutton(self.rb_frame, text=self.RadioButtonNames[i - 1],
                                                   variable=self.rb_int,
                                                   value=i, command=self.radio_button_callback)
@@ -338,6 +339,17 @@ class PrimaryOverlay(threading.Thread):
             self.SpawnLabel.configure(text="")
             self.EnergyLabel.configure(text="")
             self.hide_mining_overlay()
+        elif self.rb_int.get() == 12:
+            self.EnergyEntry.configure(state=tk.DISABLED)
+            self.energy_use.set('')
+            self.SpawnEntry.configure(state=tk.DISABLED)
+            self.spawn_use.set('')
+            self.Emitter_Box.configure(state=tk.DISABLED)
+            self.emitter_use.set('')
+            self.EmitterLabel.configure(text="")
+            self.SpawnLabel.configure(text="")
+            self.EnergyLabel.configure(text="")
+            self.hide_mining_overlay()
     def get_run_mode(self):
         return self.run_mode
 

create_mask_from_png.py

@@ -10,7 +10,7 @@ from config_file import UserConfigs
 # load the original input image and display it to our screen
 
 #filename = "equip/chests/chest_23_32"
-path = "equip/wands/"
+path = "flappy/"
 os.chdir(path)
 
 for entry in os.listdir():

equipment.py

@@ -112,6 +112,9 @@ class Equipment(GameBase):
 
                 spawn_1 = self.find_emitter(emitter, screenshot, 1)
                 if len(spawn_1) == 1:
+                    #output_by_area = self.vision_stun.draw_rectangles(screenshot, spawn_1)
+                    #cv.imshow("spawn_1", output_by_area)
+                    #cv.waitKey(150)
                     points = self.vision_stun.get_click_points(spawn_1)
                     for i in range(0, self.SPAWN_COUNT, 1):
                         self.click_point(points[0][0], points[0][1])
@@ -316,7 +319,7 @@ class Equipment(GameBase):
             else:
                 needle = cv.imread("equip/emitters/ring_e2_32.jpg", cv.IMREAD_UNCHANGED)
                 mask = cv.imread("equip/emitters/ring_e2_32-mask.png", cv.IMREAD_COLOR)
-                return self.vision_stun.find(screen, needle, 0.95, 1, True, mask)
+                return self.vision_stun.find(screen, needle, 0.95, 1, False, mask)
         elif emitter_to_use == EMITTER_WAND:
             if layer == 0:
                 needle = cv.imread("equip/emitters/wand_e1_32.jpg", cv.IMREAD_UNCHANGED)

field.py

@@ -14,7 +14,6 @@ class Field():
         else:
             self.state = [[' ' for cols in range(Field.WIDTH)]
                           for rows in range(Field.HEIGHT)]
-        #self.rotate_90_degree_clckwise(self.state)
         self.cleared_rows = 1
 
     def __str__(self):
@@ -24,7 +23,8 @@ class Field():
             '{:2d} |'.format(i) + ' '.join(row) + '|'
                 for i, row in enumerate(self.state)]) + '\n' + BAR
 
-    def matrixflip(self, m, d):
+    @staticmethod
+    def matrixflip(m, d):
         tempm = m.copy()
         if d == 'h':
             for i in range(0, len(tempm), 1):
@@ -33,14 +33,16 @@ class Field():
             tempm.reverse()
         return (tempm)
 
-    def rotate_90_degree_anticlckwise(self, matrix):
+    @staticmethod
+    def rotate_90_degree_anticlckwise(matrix):
         new_matrix = []
         for i in range(len(matrix[0]), 0, -1):
             new_matrix.append(list(map(lambda x: x[i - 1], matrix)))
 
         return new_matrix
 
-    def rotate_90_degree_clckwise(self, matrix):
+    @staticmethod
+    def rotate_90_degree_clckwise(matrix):
         new_matrix = []
         for i in range(len(matrix[0])):
             li = list(map(lambda x: x[i], matrix))
@@ -58,9 +60,8 @@ class Field():
             self.state[row] = [' ' for cols in range(Field.WIDTH)]
 
     def rotate_state(self):
-        self.state = self.rotate_90_degree_anticlckwise(self.state)
+        self.state = Field.rotate_90_degree_anticlckwise(self.state)
-        #self.state = self.matrixflip(self.state, 'v')
+
-        #print(self.state)
 
     def get_line_count(self):
         return self.cleared_rows
@@ -151,9 +152,14 @@ class Field():
 
 
 
+    def check_crucial_pos_to_be_free(self):
+        if self.state[19][9] == ' ' and self.state[19][10] == ' ' and self.state[18][9] == ' ' and self.state[18][10] == ' ':
+            return True
+        return False
+
     def predict_gaps_in_next_rotation(self):
         tmp_state = copy(self.state)
-        tmp_state = self.rotate_90_degree_anticlckwise(tmp_state)
+        tmp_state = Field.rotate_90_degree_anticlckwise(tmp_state)
         for row in range(int(self.HEIGHT/2)):
             tmp_state[row] = [' ' for cols in range(Field.WIDTH)]
         return sum(

flappy.py

@@ -0,0 +1,49 @@
+import cv2 as cv
+import pydirectinput
+from game_base_class import GameBase
+from flappy_pos_discovery_thread import FlappyPosDiscovery
+from keyboard_thread import KeyboardEvent
+
+class Flappy(GameBase):
+
+    def __init__(self, overlay):
+        super().__init__(overlay)
+
+        self.litris_reset_board = cv.imread("control_elements/sodoku_reset_button.jpg", cv.IMREAD_COLOR)
+
+        self.flappy_pos_disc = FlappyPosDiscovery()
+        self.keyboard_listener = KeyboardEvent()
+
+    def assess_playfield_and_make_move(self):
+        #last_letter_received = time()
+
+        while True:
+            if self.overlay.run_mode == 'stopped' or self.overlay.run_mode == 'paused':
+                return
+            wait_timer = 380 + self.keyboard_listener.offset_value
+            self.click()
+            cv.waitKey(wait_timer)
+            '''
+            if self.flappy_pos_disc.next_gate_height[0] is not 0:
+                offset = (self.flappy_pos_disc.next_gate_height[0] + 120) - self.flappy_pos_disc.get_actual_pet_height()
+                if offset > 50:
+                    self.click()
+            cv.waitKey(int(wait_timer -100))
+            print("pet_pos: ", self.flappy_pos_disc.get_actual_pet_height())
+
+            if self.flappy_pos_disc.next_gate_height[0] is not 0:
+                offset = (self.flappy_pos_disc.next_gate_height[0] + 120) - self.flappy_pos_disc.current_pet_height
+                if offset > 50:
+                    offset = 50
+                elif offset < -50:
+                    offset = -50
+                print(offset)
+                wait_timer = 380 + offset
+                print(wait_timer)
+                print("pet_pos: ", self.flappy_pos_disc.current_pet_height)
+                print("next gate: ", self.flappy_pos_disc.next_gate_height[0] + 120) '''
+
+    def click(self):
+        pydirectinput.mouseDown()
+        cv.waitKey(50)
+        pydirectinput.mouseUp()

flappy_pos_discovery_thread.py

@@ -0,0 +1,83 @@
+import threading
+import cv2 as cv
+from window_capture import WindowCapture
+from vision import Vision
+from config_file import UserConfigs
+
+class FlappyPosDiscovery(threading.Thread):
+
+    def __init__(self):
+        threading.Thread.__init__(self)
+
+        self.config = UserConfigs()
+        self.capture_window = WindowCapture(None, None, self.config)
+        self.vision_stun = Vision()
+
+        self.needle_f = self.scale_picture(cv.imread("flappy/flappy.jpg"), 50)
+        self.mask_f = self.scale_picture(cv.imread("flappy/flappy-mask.png"), 50)
+
+        self.needle_g = self.scale_picture(cv.imread("flappy/gate.jpg"), 50)
+        self.mask_g = self.scale_picture(cv.imread("flappy/gate-mask.png"), 50)
+
+        self.run_mode = 'run'
+        self.current_pet_height = []
+        self.next_gate_height = [0,0,0]
+
+        self.offset_down = 90
+
+        self.start()
+
+    def run(self):
+        while self.run_mode == 'run':
+            screenshot_g = self.scale_picture(self.capture_window.get_screenshot_by_area([700, 1200, 1860, 90]), 50)
+            screenshot_f = self.scale_picture(self.capture_window.get_screenshot_by_area([200, 1250, 610, 90]), 50)
+
+            #cv.imshow("screenshot_g", screenshot_f)
+            #cv.waitKey(150)
+
+            rectangles_g = self.vision_stun.find(screenshot_g, self.needle_g, 0.8, 1)
+            rectangles_f = self.vision_stun.find(screenshot_f, self.needle_f, 0.7, 1) #, True, self.mask_f)
+
+            if len(rectangles_g) is not 0:
+                height = rectangles_g[0][1] + rectangles_g[0][3] + (self.offset_down /2)
+                if self.next_gate_height[2] is not height:
+                    self.next_gate_height.append(height)
+                if len(self.next_gate_height) > 3:
+                    self.next_gate_height.pop(0)
+                #pro_screen = self.vision_stun.draw_rectangles(screenshot_g, rectangles_g)
+                #cv.imshow("screenshot_g", pro_screen)
+                #cv.waitKey(150)
+            else:
+                pass
+
+            if len(rectangles_f) is not 0:
+                self.current_pet_height.append(rectangles_f[0][1] + rectangles_f[0][3] + self.offset_down)
+                if len(self.current_pet_height) > 5:
+                    self.current_pet_height.pop(0)
+
+            #print("pet_pos: ", self.current_pet_height)
+            #print("next gate: ", self.next_gate_height)
+
+    def scale_picture(self, img, scale_percent):
+        width = int(img.shape[1] * scale_percent / 100)
+        height = int(img.shape[0] * scale_percent / 100)
+
+        resized_img = cv.resize(img, (width, height), interpolation=4)
+
+        gray = cv.cvtColor(resized_img, cv.COLOR_BGR2GRAY)
+        thresh = cv.threshold(gray, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1]
+
+        return thresh
+
+    def get_actual_pet_height(self):
+        return sum(self.current_pet_height) / len(self.current_pet_height)
+
+
+    def get_next_gate_height(self):
+        return self.next_gate_height
+
+
+
+
+
+

keyboard_thread.py

@@ -0,0 +1,32 @@
+import threading
+import cv2 as cv
+import keyboard
+
+
+class KeyboardEvent(threading.Thread):
+
+    def __init__(self):
+        threading.Thread.__init__(self)
+
+        self.run_mode = 'run'
+        self.offset_value = 0
+        self.start()
+    def run(self):
+        while self.run_mode == 'run':
+            if keyboard.is_pressed('up'):  # if key 'q' is pressed
+                self.offset_value = -75
+            elif keyboard.is_pressed('down'):
+                self.offset_value = 75
+            elif keyboard.is_pressed('left') or keyboard.is_pressed('right'):
+                self.offset_value = 0
+            cv.waitKey(10)
+
+    def callback(self):
+        pass
+
+    def destroy(self):
+        self.destroy()
+
+    def get_run_mode(self):
+        return self.run_mode
+

litris.py

@@ -1,3 +1,5 @@
+from copy import copy
+
 import cv2 as cv
 import numpy as np
 from time import time
@@ -15,43 +17,37 @@ class Litris(GameBase):
 
         self.keyboard = Controller()
 
-        self.data_coordinates = np.zeros((20, 20), dtype=object)
-        self.observation = np.zeros((20, 20), dtype=int)
-
-        self.colors = [1, 2, 3, 4, 5, 6, 7, 8, 9]
-
         self.offset_left = 610
         self.offset_down = 40
 
-        self.fill_data_coordinates()
-
         self.field = Field()
-        self.field_mem = Field()
 
         self.litris_reset_board = cv.imread("control_elements/sodoku_reset_button.jpg", cv.IMREAD_COLOR)
 
         self.stone_id_thread = NewStoneID()
 
         self.move_mode = 1
+        self.moved_around_full = False
 
-    def fill_data_coordinates(self):
+        self.field_state_storage = {
-        # 610 to 1950 = 1340 - 76 / 20 = 63
+            1: self.field,
-        # 40 to 1380 = 1340 - 76 / 20 = 63
+            2: self.field,
-        # spacing 19 * 4
+            3: self.field,
-        dim = 63
+            4: self.field
-        e_spacing = 4
+        }
-        i_spacing = 4
+
-        for e in range(0, 20, 1):
+    def reset(self):
-            for i in range(0, 20, 1):
+        self.field.reset_field()
-                self.data_coordinates[e][i] = [(i * dim) + (i * i_spacing), (e * dim) + (e * e_spacing), dim, dim]
+        self.field.cleared_rows = 1
+        self.move_mode = 1
+        self.moved_around_full = False
 
     def assess_playfield_and_make_move(self):
         last_letter_received = time()
         while True:
             if self.stone_id_thread.get_pick_up_status() == False:
                 if (time() - last_letter_received) >= 5:
-                    self.field.reset_field()
+                    self.reset()
-                    self.field.cleared_rows = 1
                     last_letter_received = time()
                     self.dig_point(1500, 980, 100)
                     if self.overlay.run_mode == 'stopped' or self.overlay.run_mode == 'paused':
@@ -59,7 +55,7 @@ class Litris(GameBase):
                     continue
                 if self.overlay.run_mode == 'stopped' or self.overlay.run_mode == 'paused':
                     return
-                cv.waitKey(50)
+                cv.waitKey(25)
                 continue
 
 
@@ -89,21 +85,46 @@ class Litris(GameBase):
             self.move_stone(column - offset_col, rotation)
             self.drop_down()
             print(self.field)
-            if self.field.get_line_count() >= 6 and self.field.height() <= 2:
+            if self.field.get_line_count() >= 6 and self.field.height() <= 2 and self.field.check_crucial_pos_to_be_free():
-                if self.field.predict_gaps_in_next_rotation() <= 5:
+                if self.field.predict_gaps_in_next_rotation() <= 3:
+                    self.field_state_storage[self.move_mode] = copy(self.field)
                     self.update_move_mode()
+                    #self.field.state = self.stone_id_thread.get_current_board_state()
                     self.field.rotate_state()
                     #self.update_move_mode()
                     self.field.reset_half_field()
+                    self.update_field_with_stored_edges()
                     #field_mem = copy(self.field)
                     #self.field = copy(self.field_mem)
                     #self.field_mem = copy(field_mem)
 
                     self.field.cleared_rows = 1
 
-                    cv.waitKey(200)
+                    cv.waitKey(100)
             self.stone_id_thread.set_pick_up_status(False)
 
+    def update_field_with_stored_edges(self):
+        if self.move_mode == 1 and self.moved_around_full:
+            state_copy = Field.rotate_90_degree_clckwise(copy(self.field_state_storage[2].state))
+            for i in range(11,20,1):
+                for e in range(0,2,1):
+                    self.field.state[i][e] = state_copy[i][e]
+        elif self.move_mode == 2 and self.moved_around_full:
+            state_copy = Field.rotate_90_degree_clckwise(copy(self.field_state_storage[3].state))
+            for i in range(11,20,1):
+                for e in range(0,2,1):
+                    self.field.state[i][e] = state_copy[i][e]
+        elif self.move_mode == 3 and self.moved_around_full:
+            state_copy = Field.rotate_90_degree_clckwise(copy(self.field_state_storage[4].state))
+            for i in range(11, 20, 1):
+                for e in range(0, 2, 1):
+                    self.field.state[i][e] = state_copy[i][e]
+        elif self.move_mode == 4:
+            state_copy = Field.rotate_90_degree_clckwise(copy(self.field_state_storage[1].state))
+            for i in range(11,20,1):
+                for e in range(0,2,1):
+                    self.field.state[i][e] = state_copy[i][e]
+            self.moved_around_full = True
 
     def drop_down(self):
         if self.move_mode == 1:
@@ -126,7 +147,7 @@ class Litris(GameBase):
             self.keyboard.press(down)
             self.keyboard.release(down)
             print("drop down pressed:", down)
-            cv.waitKey(50)
+            cv.waitKey(40)
 
     def update_move_mode(self):
         if self.move_mode <=3:
@@ -159,13 +180,13 @@ class Litris(GameBase):
         self.keyboard.press(down)
         self.keyboard.release(down)
         print("direction pressed: ", down)
-        cv.waitKey(120)
+        cv.waitKey(60)
 
         if rotation == 3:
             self.keyboard.press('e')
             self.keyboard.release('e')
             print("rotation 1 pressed: e")
-            cv.waitKey(40)
+            cv.waitKey(30)
         elif rotation == 2:
             self.keyboard.press('e')
             self.keyboard.release('e')
@@ -179,28 +200,28 @@ class Litris(GameBase):
             self.keyboard.press('e')
             self.keyboard.release('e')
             print("rotation 3 pressed: e 1")
-            cv.waitKey(20)
+            cv.waitKey(30)
             self.keyboard.press('e')
             self.keyboard.release('e')
             print("rotation 3 pressed: e 2")
-            cv.waitKey(20)
+            cv.waitKey(30)
             self.keyboard.press('e')
             self.keyboard.release('e')
             print("rotation 3 pressed: e 3")
-            cv.waitKey(20)
+            cv.waitKey(30)
 
         if col_movement < 0:
             for i in range(0, col_movement, - 1):
                 self.keyboard.press(left)
                 self.keyboard.release(left)
                 print("move left 3 pressed:", left)
-                cv.waitKey(40)
+                cv.waitKey(30)
         else:
             for i in range(0, col_movement, 1):
                 self.keyboard.press(right)
                 self.keyboard.release(right)
                 print("move right 3 pressed:", right)
-                cv.waitKey(40)
+                cv.waitKey(30)
 
     def point_in_rect(self, point):
         for e in range(0, 20, 1):

litris_stone_id_thread.py

@@ -26,9 +26,10 @@ class NewStoneID(threading.Thread):
         self.vision_stun = Vision()
 
         self.stone_coordinates = np.zeros((4, 4), dtype=object)
+        self.data_coordinates = np.zeros((20, 20), dtype=object)
         self.fill_data_coordinates()
 
-        self.needles = {1: cv.imread("litris/blue_needle.jpg", cv.IMREAD_UNCHANGED)}
+        self.needles = {'Q': cv.imread("litris/blue_needle.jpg", cv.IMREAD_UNCHANGED)}
 
         self.run_mode = 'run'
         self.actual_letter = ""
@@ -75,7 +76,7 @@ class NewStoneID(threading.Thread):
         screenshot = self.capture_window.get_screenshot()
         screenshot = screenshot[580:845, 1148:1412]
 
-        rectangles = self.vision_stun.find(screenshot, self.needles[1], 0.85, 16)
+        rectangles = self.vision_stun.find(screenshot, self.needles['Q'], 0.85, 16)
         if len(rectangles) == 0:
             return None
 
@@ -133,6 +134,50 @@ class NewStoneID(threading.Thread):
                         return e, i
         return None, None
 
+    def get_current_board_state(self):
+
+        # get an updated image of the game
+        screenshot = self.capture_window.get_screenshot()
+        #screenshot = cv.imread("litris/main_playfield.jpg")
+        screenshot = screenshot[40:1380, 610:1950]  # 1000,1000
+
+        # cv.imshow("screenshot", screenshot)
+        # cv.waitKey(150)
+        # continue
+        #data_coords = np.zeros((20, 20), dtype=object)
+        data_coords = np.full((20, 20), ' ', dtype=object)
+        #field = Pickaxe_Field()
+        for needle_key in self.needles.keys():
+            #gray_needle = cv.cvtColor(self.needles[needle_key], cv.COLOR_BGR2GRAY)
+            #thresh_needle = cv.threshold(gray_needle, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1]
+            rectangles = self.vision_stun.find(screenshot, self.needles[needle_key], 0.8, 200)
+            if len(rectangles) == 0:
+                continue
+            points = self.vision_stun.get_click_points(rectangles)
+
+            for point in points:
+                x, y = self.point_in_rect(point)
+                if x is not None and y is not None:
+                    data_coords[x][y] = needle_key
+                    #self.change_value(x, y, int(needle_key))
+                    # print(field.data_value_grid)
+                    # cv.circle(screenshot, points[0], 7, (0, 255, 0), -1)
+                    #output_image = self.vision_stun.draw_rectangles(screenshot, rectangles)
+                    #cv.imshow("output_image", output_image)
+                    #cv.waitKey(150)
+        return data_coords.tolist()
+
+    def point_in_rect(self, point):
+        for e in range(0, 20, 1):
+            for i in range(0, 20, 1):
+                x1, y1, w, h = self.data_coordinates[e][i]
+                x2, y2 = x1 + w, y1 + h
+                x, y = point
+                if x1 < x and x < x2:
+                    if y1 < y and y < y2:
+                        return e, i
+        return None, None
+
     def fill_data_coordinates(self):
         # 610 to 1950 = 1340 - 76 / 20 = 63
         # 40 to 1380 = 1340 - 76 / 20 = 63
@@ -143,3 +188,6 @@ class NewStoneID(threading.Thread):
         for e in range(0, 4, 1):
             for i in range(0, 4, 1):
                 self.stone_coordinates[e][i] = [(i * dim) + (i * i_spacing), (e * dim) + (e * e_spacing), dim, dim]
+        for e in range(0, 20, 1):
+            for i in range(0, 20, 1):
+                self.data_coordinates[e][i] = [(i * dim) + (i * i_spacing), (e * dim) + (e * e_spacing), dim, dim]

menti_words.py

@@ -1,7 +1,10 @@
 import cv2 as cv
 import numpy as np
 import pydirectinput
+from nltk.corpus import words
+from pytesseract import pytesseract
 
+import utils
 from utils import mse
 from game_base_class import GameBase
 import random
@@ -18,6 +21,8 @@ class MentiWords(GameBase):
     def __init__(self, overlay):
         super().__init__(overlay)
 
+        pytesseract.tesseract_cmd = r'C:\Program Files\Tesseract-OCR\tesseract.exe'
+
         self.observation = np.zeros((9, 9), dtype=int)
 
         self.colors = [1, 2, 3, 4, 5, 6, 7, 8, 9]
@@ -27,13 +32,55 @@ class MentiWords(GameBase):
 
         self.menti_play_button = cv.imread("control_elements/play.jpg", cv.IMREAD_COLOR)
 
-        self.needles = {'E': cv.imread("menti_words/e.jpg", cv.IMREAD_COLOR),
+        self.needles = {
+            'A': cv.imread("menti_words/a.jpg", cv.IMREAD_COLOR),
+            'B': cv.imread("menti_words/b.jpg", cv.IMREAD_COLOR),
+            'C': cv.imread("menti_words/c.jpg", cv.IMREAD_COLOR),
+            'D': cv.imread("menti_words/d.jpg", cv.IMREAD_COLOR),
+            'E': cv.imread("menti_words/e.jpg", cv.IMREAD_COLOR),
+            'F': cv.imread("menti_words/f.jpg", cv.IMREAD_COLOR),
+
             'H': cv.imread("menti_words/h.jpg", cv.IMREAD_COLOR),
+            'I': cv.imread("menti_words/i.jpg", cv.IMREAD_COLOR),
+
+            'L': cv.imread("menti_words/l.jpg", cv.IMREAD_COLOR),
+            'M': cv.imread("menti_words/m.jpg", cv.IMREAD_COLOR),
+            'N': cv.imread("menti_words/n.jpg", cv.IMREAD_COLOR),
             'O': cv.imread("menti_words/o.jpg", cv.IMREAD_COLOR),
-                        'R': cv.imread("menti_words/r.jpg", cv.IMREAD_COLOR),
-                        'T': cv.imread("menti_words/t.jpg", cv.IMREAD_COLOR),
             'P': cv.imread("menti_words/p.jpg", cv.IMREAD_COLOR),
-                        'I': cv.imread("menti_words/i.jpg", cv.IMREAD_COLOR)
+
+            'R': cv.imread("menti_words/r.jpg", cv.IMREAD_COLOR),
+            'S': cv.imread("menti_words/s.jpg", cv.IMREAD_COLOR),
+            'T': cv.imread("menti_words/t.jpg", cv.IMREAD_COLOR),
+            'U': cv.imread("menti_words/u.jpg", cv.IMREAD_COLOR),
+
+            'W': cv.imread("menti_words/w.jpg", cv.IMREAD_COLOR)
+
+                        }
+        self.masks = {
+            'A': cv.imread("menti_words/a-mask.png", cv.IMREAD_COLOR),
+            'B': cv.imread("menti_words/b-mask.png", cv.IMREAD_COLOR),
+            'C': cv.imread("menti_words/c-mask.png", cv.IMREAD_COLOR),
+            'D': cv.imread("menti_words/d-mask.png", cv.IMREAD_COLOR),
+            'E': cv.imread("menti_words/e-mask.png", cv.IMREAD_COLOR),
+            'F': cv.imread("menti_words/f-mask.png", cv.IMREAD_COLOR),
+
+            'H': cv.imread("menti_words/h-mask.png", cv.IMREAD_COLOR),
+            'I': cv.imread("menti_words/i-mask.png", cv.IMREAD_COLOR),
+
+            'L': cv.imread("menti_words/l-mask.png", cv.IMREAD_COLOR),
+            'M': cv.imread("menti_words/m-mask.png", cv.IMREAD_COLOR),
+            'N': cv.imread("menti_words/n-mask.png", cv.IMREAD_COLOR),
+            'O': cv.imread("menti_words/o-mask.png", cv.IMREAD_COLOR),
+            'P': cv.imread("menti_words/p-mask.png", cv.IMREAD_COLOR),
+
+            'R': cv.imread("menti_words/r-mask.png", cv.IMREAD_COLOR),
+            'S': cv.imread("menti_words/s-mask.png", cv.IMREAD_COLOR),
+            'T': cv.imread("menti_words/t-mask.png", cv.IMREAD_COLOR),
+            'U': cv.imread("menti_words/u-mask.png", cv.IMREAD_COLOR),
+
+            'W': cv.imread("menti_words/w-mask.png", cv.IMREAD_COLOR)
+
                         }
 
         self.current_letters = []
@@ -41,6 +88,7 @@ class MentiWords(GameBase):
 
         with open("menti_dic") as file:
             self.word_list = [line.rstrip() for line in file]
+        #self.word_list2 = words.words()
 
     def reset_lists(self):
         self.current_letters = []
@@ -79,7 +127,7 @@ class MentiWords(GameBase):
 
         #screenshot = cv.imread("menti_words/screenshot.jpg")
         screenshot = self.capture_window.get_screenshot()
-        screenshot = screenshot[870:1240, 1080:1480]
+        screenshot = utils.scale_screenshot(screenshot[870:1270, 1080:1480], 200, False)
 
         #cv.imshow("screenshot", screenshot)
         #cv.waitKey(150)
@@ -87,19 +135,35 @@ class MentiWords(GameBase):
         for needle_key in self.needles.keys():
             # gray_needle = cv.cvtColor(self.needles[needle_key], cv.COLOR_BGR2GRAY)
             # thresh_needle = cv.threshold(gray_needle, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1]
-            rectangles = self.vision_stun.find(screenshot, self.needles[needle_key], 0.85, 56)
+
+            thresh_needle = utils.scale_screenshot(self.needles[needle_key], 200, False)
+            rectangles = self.vision_stun.find(screenshot, thresh_needle , 0.85, 2)
+            #rectangles = self.vision_stun.find(screenshot, self.needles[needle_key], 0.95, 1 ,True, self.masks[needle_key])
             if len(rectangles) == 0:
                 continue
             points = self.vision_stun.get_click_points(rectangles)
+            for point in points:
                 self.current_letters.append(needle_key)
-            self.letter_coords[needle_key] = points[0]
+                self.letter_coords[needle_key] = (int(point[0]/2), int(point[1]/2))
+            '''
+            cropped1 = self.vision_stun.draw_display_picture(screenshot, rectangles, 10)
+            #cropped1 = utils.scale_screenshot(cropped1)
+            cv.imshow("cropped1", cropped1)
+            cv.waitKey(150)
+            text_1 = pytesseract.image_to_string(cropped1, lang='eng', config='--psm 6').strip()
+            if str.isalpha(text_1):
+                #cv.imshow("cropped1", cropped1)
+                #cv.waitKey(150)
+                points = self.vision_stun.get_click_points(rectangles)
+                self.current_letters.append(text_1)
+                self.letter_coords[text_1] = points[0]'''
             if self.overlay.run_mode == 'stopped' or self.overlay.run_mode == 'paused':
                 return
 
     def possible_words(self, lwords, charSet):
         lst = []
         for word in lwords:
-            if len(word) <= 2:
+            if len(word) <= 1:
                 continue
             flag = 1
             chars = self.charCount(word)

utils.py

@@ -61,18 +61,20 @@ def check_for_bl_line(line_item, black_list):
     return False
 
 
-def scale_screenshot(screenshot):
+def scale_screenshot(screenshot, percent = 200, thresh_mode = True):
-    scale_percent = 200  # percent of original size
+    scale_percent = percent  # percent of original size
     width = int(screenshot.shape[1] * scale_percent / 100)
     height = int(screenshot.shape[0] * scale_percent / 100)
     dim = (width, height)
     resized_img = cv.resize(screenshot, dim, interpolation=4)
 
+    if thresh_mode:
         gray = cv.cvtColor(resized_img, cv.COLOR_BGR2GRAY)
         thresh = cv.threshold(gray, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1]
+        return thresh
     # cv.imshow("Tresh", thresh)
     # cv.waitKey(1000)
-    return thresh
+    return resized_img
 
 
 def save_line_item_npy_jpg(short_pic_list):

vision.py

@@ -158,10 +158,11 @@ class Vision:
 
         if mask is not None:
             result = cv.matchTemplate(haystack_img, needle_img, cv.TM_CCORR_NORMED, None, mask)
-            _minVal, _maxVal, minLoc, maxLoc = cv.minMaxLoc(result, None)
         else:
             result = cv.matchTemplate(haystack_img, needle_img, self.method)
 
+        _minVal, _maxVal, minLoc, maxLoc = cv.minMaxLoc(result, None)
+
         if normalize:
             cv.normalize(result, result, 0, 1, cv.NORM_MINMAX, -1)
         # Get the all the positions from the match result that exceed our threshold
@@ -246,11 +247,11 @@ class Vision:
 
         return haystack_img
 
-    def draw_display_picture(self, haystack_img, rectangles):
+    def draw_display_picture(self, haystack_img, rectangles, border = 0):
 
         pic = None
         for (x, y, w, h) in rectangles:
-            pic = haystack_img[y:y + h, x:x + w]
+            pic = haystack_img[y-border:y + h +border, x-border:x + w + border]
 
         # scale_percent = 500  # percent of original size
         # width = int(pic.shape[1] * scale_percent / 100)