Merge remote-tracking branch 'origin/master'

.idea/lc-py.iml

@@ -4,7 +4,7 @@
     <content url="file://$MODULE_DIR$">
       <excludeFolder url="file://$MODULE_DIR$/venv" />
     </content>
-    <orderEntry type="inheritedJdk" />
+    <orderEntry type="jdk" jdkName="Python 3.7 (lc-py-b)" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
 </module>

.idea/misc.xml

@@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
-  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.7 (lc_py_b)" project-jdk-type="Python SDK" />
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.7 (lc-py-b)" project-jdk-type="Python SDK" />
 </project>

combined_bot_main.py

@@ -9,6 +9,10 @@ from craft import Craft
 from mine import Mine
 from sodoku import Sodoku
 from fruit import Fruit
+from pickaxe import Pickaxe_Field
+from litris import Litris
+from menti_words import MentiWords
+from flappy import Flappy
 
 
 def run():
@@ -32,7 +36,6 @@ def run():
             overlay.run_mode = 'init'
             continue
         elif overlay.run_mode == 'quit':
-            overlay.destroy()
             return
         else:
             break
@@ -61,7 +64,20 @@ def run():
         elif overlay.rb_int.get() == 8:
             sodo = Sodoku(overlay)
             sodo.execute_main_loop()
-
+        elif overlay.rb_int.get() == 9:
+            paxe = Pickaxe_Field(overlay)
+            paxe.execute_main_loop()
+        elif overlay.rb_int.get() == 10:
+            ltris = Litris(overlay)
+            ltris.execute_main_loop()
+            ltris.stone_id_thread.run_mode = 'stop'
+        elif overlay.rb_int.get() == 11:
+            menti = MentiWords(overlay)
+            menti.execute_main_loop()
+        elif overlay.rb_int.get() == 12:
+            flappy = Flappy(overlay)
+            flappy.execute_main_loop()
+            flappy.flappy_pos_disc.run_mode = 'stop'
 
 if __name__ == "__main__":
     run()

combined_user_interface.py

@@ -33,7 +33,7 @@ class PrimaryOverlay(threading.Thread):
         self.Emitter_Box = ttk.Combobox
 
         self.RadioButtons = dict
-        self.RadioButtonNames = ["Equip", "Crops", "Farm", "Magic", "Craft", "Mine", "Fruit", "Sodo"]
+        self.RadioButtonNames = ["Equip", "Crops", "Farm", "Magic", "Craft", "Mine", "Fruit", "Sodo", "PAxe", "Ltris", "Menti", "Flapp"]
         self.RadioButton1 = tk.Radiobutton
         self.RadioButton2 = tk.Radiobutton
         self.RadioButton3 = tk.Radiobutton
@@ -42,12 +42,16 @@ class PrimaryOverlay(threading.Thread):
         self.RadioButton6 = tk.Radiobutton
         self.RadioButton7 = tk.Radiobutton
         self.RadioButton8 = tk.Radiobutton
+        self.RadioButton9 = tk.Radiobutton
+        self.RadioButton10 = tk.Radiobutton
+        self.RadioButton11 = tk.Radiobutton
+        self.RadioButton12 = tk.Radiobutton
 
         self.StartButton = tk.Button
         self.StopButton = tk.Button
         self.PauseButton = tk.Button
         self.QuitButton = tk.Button
-        self.TkPosition = '133x354+60+600'
+        self.TkPosition = '133x454+60+600'
         self.setDaemon(True)
         self.StatusLabel = tk.Label
 
@@ -75,7 +79,7 @@ class PrimaryOverlay(threading.Thread):
         self.rb_int = tk.IntVar(self.root, value=1)
         self.RadioButtons = dict()
         # var = tk.IntVar(value=1)
-        for i in range(1, 9):
+        for i in range(1, 13):
             self.RadioButtons[i] = tk.Radiobutton(self.rb_frame, text=self.RadioButtonNames[i - 1],
                                                   variable=self.rb_int,
                                                   value=i, command=self.radio_button_callback)
@@ -176,6 +180,7 @@ class PrimaryOverlay(threading.Thread):
 
     def destroy(self):
         self.hide_mining_overlay()
+        del self.MiningOverlay
         self.root.destroy()
 
     def start_button_callback(self):
@@ -301,7 +306,50 @@ class PrimaryOverlay(threading.Thread):
             self.SpawnLabel.configure(text="")
             self.EnergyLabel.configure(text="")
             self.hide_mining_overlay()
-
+        elif self.rb_int.get() == 9:
+            self.EnergyEntry.configure(state=tk.DISABLED)
+            self.energy_use.set('')
+            self.SpawnEntry.configure(state=tk.DISABLED)
+            self.spawn_use.set('')
+            self.Emitter_Box.configure(state=tk.DISABLED)
+            self.emitter_use.set('')
+            self.EmitterLabel.configure(text="")
+            self.SpawnLabel.configure(text="")
+            self.EnergyLabel.configure(text="")
+            self.hide_mining_overlay()
+        elif self.rb_int.get() == 10:
+            self.EnergyEntry.configure(state=tk.DISABLED)
+            self.energy_use.set('')
+            self.SpawnEntry.configure(state=tk.DISABLED)
+            self.spawn_use.set('')
+            self.Emitter_Box.configure(state=tk.DISABLED)
+            self.emitter_use.set('')
+            self.EmitterLabel.configure(text="")
+            self.SpawnLabel.configure(text="")
+            self.EnergyLabel.configure(text="")
+            self.hide_mining_overlay()
+        elif self.rb_int.get() == 11:
+            self.EnergyEntry.configure(state=tk.DISABLED)
+            self.energy_use.set('')
+            self.SpawnEntry.configure(state=tk.DISABLED)
+            self.spawn_use.set('')
+            self.Emitter_Box.configure(state=tk.DISABLED)
+            self.emitter_use.set('')
+            self.EmitterLabel.configure(text="")
+            self.SpawnLabel.configure(text="")
+            self.EnergyLabel.configure(text="")
+            self.hide_mining_overlay()
+        elif self.rb_int.get() == 12:
+            self.EnergyEntry.configure(state=tk.DISABLED)
+            self.energy_use.set('')
+            self.SpawnEntry.configure(state=tk.DISABLED)
+            self.spawn_use.set('')
+            self.Emitter_Box.configure(state=tk.DISABLED)
+            self.emitter_use.set('')
+            self.EmitterLabel.configure(text="")
+            self.SpawnLabel.configure(text="")
+            self.EnergyLabel.configure(text="")
+            self.hide_mining_overlay()
     def get_run_mode(self):
         return self.run_mode
 

create_mask_from_png.py

@@ -10,7 +10,7 @@ from config_file import UserConfigs
 # load the original input image and display it to our screen
 
 #filename = "equip/chests/chest_23_32"
-path = "equip/wands/"
+path = "flappy/"
 os.chdir(path)
 
 for entry in os.listdir():

crops.py

@@ -163,7 +163,7 @@ class Crops(GameBase):
         # cv.waitKey(150)
         # continue
         data_coords = np.zeros((8, 14), dtype=object)
-        # field = Field()
+        # field = Pickaxe_Field()
 
         for needle_key in self.needles.keys():
             # gray_needle = cv.cvtColor(self.needles[needle_key], cv.COLOR_BGR2GRAY)

equipment.py

@@ -112,6 +112,9 @@ class Equipment(GameBase):
 
                 spawn_1 = self.find_emitter(emitter, screenshot, 1)
                 if len(spawn_1) == 1:
+                    #output_by_area = self.vision_stun.draw_rectangles(screenshot, spawn_1)
+                    #cv.imshow("spawn_1", output_by_area)
+                    #cv.waitKey(150)
                     points = self.vision_stun.get_click_points(spawn_1)
                     for i in range(0, self.SPAWN_COUNT, 1):
                         self.click_point(points[0][0], points[0][1])
@@ -316,7 +319,7 @@ class Equipment(GameBase):
             else:
                 needle = cv.imread("equip/emitters/ring_e2_32.jpg", cv.IMREAD_UNCHANGED)
                 mask = cv.imread("equip/emitters/ring_e2_32-mask.png", cv.IMREAD_COLOR)
-                return self.vision_stun.find(screen, needle, 0.95, 1, True, mask)
+                return self.vision_stun.find(screen, needle, 0.95, 1, False, mask)
         elif emitter_to_use == EMITTER_WAND:
             if layer == 0:
                 needle = cv.imread("equip/emitters/wand_e1_32.jpg", cv.IMREAD_UNCHANGED)

farm.py

@@ -116,7 +116,7 @@ class Farm(GameBase):
         #return
         #continue
         data_coords = np.zeros((8, 14), dtype=object)
-        # field = Field()
+        # field = Pickaxe_Field()
         for needle_key in self.needles.keys():
             # gray_needle = cv.cvtColor(self.needles[needle_key], cv.COLOR_BGR2GRAY)
             # thresh_needle = cv.threshold(gray_needle, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1]

field.py

@@ -0,0 +1,213 @@
+#!/usr/bin/python
+from copy import copy
+
+from tetromino import Tetromino
+
+class Field():
+
+    WIDTH = 20
+    HEIGHT = 20
+
+    def __init__(self, state=None):
+        if state:
+            self.state = state
+        else:
+            self.state = [[' ' for cols in range(Field.WIDTH)]
+                          for rows in range(Field.HEIGHT)]
+        self.cleared_rows = 1
+
+    def __str__(self):
+        BAR = '   ' + '-' * (Field.WIDTH * 2 + 1) + '\n    ' + \
+            ' '.join(map(str, range(Field.WIDTH))) + '\n'
+        return BAR + '\n'.join([
+            '{:2d} |'.format(i) + ' '.join(row) + '|'
+                for i, row in enumerate(self.state)]) + '\n' + BAR
+
+    @staticmethod
+    def matrixflip(m, d):
+        tempm = m.copy()
+        if d == 'h':
+            for i in range(0, len(tempm), 1):
+                tempm[i].reverse()
+        elif d == 'v':
+            tempm.reverse()
+        return (tempm)
+
+    @staticmethod
+    def rotate_90_degree_anticlckwise(matrix):
+        new_matrix = []
+        for i in range(len(matrix[0]), 0, -1):
+            new_matrix.append(list(map(lambda x: x[i - 1], matrix)))
+
+        return new_matrix
+
+    @staticmethod
+    def rotate_90_degree_clckwise(matrix):
+        new_matrix = []
+        for i in range(len(matrix[0])):
+            li = list(map(lambda x: x[i], matrix))
+            li.reverse()
+            new_matrix.append(li)
+
+        return new_matrix
+
+    def reset_field(self):
+        self.state = [[' ' for cols in range(Field.WIDTH)]
+                      for rows in range(Field.HEIGHT)]
+
+    def reset_half_field(self):
+        for row in range(int(self.HEIGHT/2)):
+            self.state[row] = [' ' for cols in range(Field.WIDTH)]
+
+    def rotate_state(self):
+        self.state = Field.rotate_90_degree_anticlckwise(self.state)
+
+
+    def get_line_count(self):
+        return self.cleared_rows
+
+    def _test_tetromino(self, tetromino, row, column):
+        """
+        Tests to see if a tetromino can be placed at the specified row and
+        column. It performs the test with the bottom left corner of the
+        tetromino at the specified row and column.
+        """
+        assert column >= 0
+        assert column + tetromino.width() <= Field.WIDTH
+        assert row - tetromino.height() + 1 >= 0
+        assert row < Field.HEIGHT
+        for ti, si in list(enumerate(range(row - tetromino.height() + 1,
+                                           row + 1)))[::-1]:
+            for tj, sj in enumerate(range(column, column + tetromino.width())):
+                if tetromino[ti][tj] != ' ' and self.state[si][sj] != ' ':
+                    return False
+        return True
+
+    def _place_tetromino(self, tetromino, row, column):
+        """
+        Place a tetromino at the specified row and column.
+        The bottom left corner of the tetromino will be placed at the specified
+        row and column. This function does not perform checks and will overwrite
+        filled spaces in the field.
+        """
+        assert column >= 0
+        assert column + tetromino.width() <= Field.WIDTH
+        assert row - tetromino.height() + 1 >= 0
+        assert row < Field.HEIGHT
+        for ti, si in list(enumerate(range(row - tetromino.height() + 1,
+                                           row + 1)))[::-1]:
+            for tj, sj in enumerate(range(column, column + tetromino.width())):
+                if tetromino[ti][tj] != ' ':
+                    self.state[si][sj] = tetromino[ti][tj]
+
+    def _get_tetromino_drop_row(self, tetromino, column):
+        """
+        Given a tetromino and a column, return the row that the tetromino
+        would end up in if it were dropped in that column.
+        Assumes the leftmost column of the tetromino will be aligned with the
+        specified column.
+        """
+        assert isinstance(tetromino, Tetromino)
+        assert column >= 0
+        assert column + tetromino.width() <= Field.WIDTH
+        last_fit = -1
+        for row in range(tetromino.height(), Field.HEIGHT):
+            if self._test_tetromino(tetromino, row, column):
+                last_fit = row
+            else:
+                return last_fit
+        return last_fit
+
+    def _line_clear(self):
+        """
+        Checks and removes all filled lines.
+        """
+        self.state = list(filter(lambda row: row.count(' ') != 0, self.state))
+        while len(self.state) < Field.HEIGHT:
+            self.state.insert(0, [' ' for col in range(Field.WIDTH)])
+            self.cleared_rows = self.cleared_rows + 1
+            print("cleared rows: ", self.cleared_rows)
+
+    def copy(self):
+        """
+        Returns a shallow copy of the field.
+        """
+        return Field([row[:] for row in self.state])
+
+    def drop(self, tetromino, column):
+        """
+        Drops a tetromino in the specified column.
+        The leftmost column of the tetromino will be aligned with the specified
+        column.
+        Returns the row it was dropped in for computations.
+        """
+        assert isinstance(tetromino, Tetromino)
+        assert column >= 0
+        assert column + tetromino.width() <= Field.WIDTH
+        row = self._get_tetromino_drop_row(tetromino, column)
+        assert row != -1
+        self._place_tetromino(tetromino, row, column)
+        self._line_clear()
+        return row
+
+
+
+    def check_crucial_pos_to_be_free(self):
+        if self.state[19][9] == ' ' and self.state[19][10] == ' ' and self.state[18][9] == ' ' and self.state[18][10] == ' ':
+            return True
+        return False
+
+    def predict_gaps_in_next_rotation(self):
+        tmp_state = copy(self.state)
+        tmp_state = Field.rotate_90_degree_anticlckwise(tmp_state)
+        for row in range(int(self.HEIGHT/2)):
+            tmp_state[row] = [' ' for cols in range(Field.WIDTH)]
+        return sum(
+            ["".join([row[col] for row in tmp_state]).lstrip().count(' ')
+             for col in range(Field.WIDTH)])
+
+
+    def count_gaps(self):
+        """
+        Check each column one by one to make sure there are no gaps in the
+        column.
+        """
+        return sum(
+            ["".join([row[col] for row in self.state]).lstrip().count(' ')
+             for col in range(Field.WIDTH)])
+
+    def height(self):
+        """
+        Returns the height on the field of the highest placed tetromino on the
+        field.
+        """
+        for i, row in enumerate(self.state):
+            if ''.join(row).strip():
+                return Field.HEIGHT - i
+
+if __name__ == '__main__':
+    import sys
+    f = Field()
+    if len(sys.argv) > 1 and sys.argv[1] == 'sim':
+        from optimizer import Optimizer
+        i = input()
+        while i != 'q':
+            t = Tetromino.create(i)
+            opt = Optimizer.get_optimal_drop(f, t)
+            t.rotate(opt['orientation'])
+            f.drop(t, opt['column'])
+            print(f)
+            i = input()
+    t = Tetromino.JTetromino().rotate_right()
+    print(t)
+    f.drop(t, 0)
+    print(f)
+    # f.drop(Tetromino.LTetromino(), 2)
+    # print(f)
+    # f.drop(Tetromino.JTetromino().rotate_left(), 5)
+    # print(f)
+    # t = Tetromino.LTetromino().flip()
+    # f.drop(t, 0)
+    # f.drop(Tetromino.TTetromino().flip(), 0)
+    # f.drop(Tetromino.JTetromino(), 4)
+    # print(f)

flappy.py

@@ -0,0 +1,49 @@
+import cv2 as cv
+import pydirectinput
+from game_base_class import GameBase
+from flappy_pos_discovery_thread import FlappyPosDiscovery
+from keyboard_thread import KeyboardEvent
+
+class Flappy(GameBase):
+
+    def __init__(self, overlay):
+        super().__init__(overlay)
+
+        self.litris_reset_board = cv.imread("control_elements/sodoku_reset_button.jpg", cv.IMREAD_COLOR)
+
+        self.flappy_pos_disc = FlappyPosDiscovery()
+        self.keyboard_listener = KeyboardEvent()
+
+    def assess_playfield_and_make_move(self):
+        #last_letter_received = time()
+
+        while True:
+            if self.overlay.run_mode == 'stopped' or self.overlay.run_mode == 'paused':
+                return
+            wait_timer = 380 + self.keyboard_listener.offset_value
+            self.click()
+            cv.waitKey(wait_timer)
+            '''
+            if self.flappy_pos_disc.next_gate_height[0] is not 0:
+                offset = (self.flappy_pos_disc.next_gate_height[0] + 120) - self.flappy_pos_disc.get_actual_pet_height()
+                if offset > 50:
+                    self.click()
+            cv.waitKey(int(wait_timer -100))
+            print("pet_pos: ", self.flappy_pos_disc.get_actual_pet_height())
+
+            if self.flappy_pos_disc.next_gate_height[0] is not 0:
+                offset = (self.flappy_pos_disc.next_gate_height[0] + 120) - self.flappy_pos_disc.current_pet_height
+                if offset > 50:
+                    offset = 50
+                elif offset < -50:
+                    offset = -50
+                print(offset)
+                wait_timer = 380 + offset
+                print(wait_timer)
+                print("pet_pos: ", self.flappy_pos_disc.current_pet_height)
+                print("next gate: ", self.flappy_pos_disc.next_gate_height[0] + 120) '''
+
+    def click(self):
+        pydirectinput.mouseDown()
+        cv.waitKey(50)
+        pydirectinput.mouseUp()

flappy_pos_discovery_thread.py

@@ -0,0 +1,83 @@
+import threading
+import cv2 as cv
+from window_capture import WindowCapture
+from vision import Vision
+from config_file import UserConfigs
+
+class FlappyPosDiscovery(threading.Thread):
+
+    def __init__(self):
+        threading.Thread.__init__(self)
+
+        self.config = UserConfigs()
+        self.capture_window = WindowCapture(None, None, self.config)
+        self.vision_stun = Vision()
+
+        self.needle_f = self.scale_picture(cv.imread("flappy/flappy.jpg"), 50)
+        self.mask_f = self.scale_picture(cv.imread("flappy/flappy-mask.png"), 50)
+
+        self.needle_g = self.scale_picture(cv.imread("flappy/gate.jpg"), 50)
+        self.mask_g = self.scale_picture(cv.imread("flappy/gate-mask.png"), 50)
+
+        self.run_mode = 'run'
+        self.current_pet_height = []
+        self.next_gate_height = [0,0,0]
+
+        self.offset_down = 90
+
+        self.start()
+
+    def run(self):
+        while self.run_mode == 'run':
+            screenshot_g = self.scale_picture(self.capture_window.get_screenshot_by_area([700, 1200, 1860, 90]), 50)
+            screenshot_f = self.scale_picture(self.capture_window.get_screenshot_by_area([200, 1250, 610, 90]), 50)
+
+            #cv.imshow("screenshot_g", screenshot_f)
+            #cv.waitKey(150)
+
+            rectangles_g = self.vision_stun.find(screenshot_g, self.needle_g, 0.8, 1)
+            rectangles_f = self.vision_stun.find(screenshot_f, self.needle_f, 0.7, 1) #, True, self.mask_f)
+
+            if len(rectangles_g) is not 0:
+                height = rectangles_g[0][1] + rectangles_g[0][3] + (self.offset_down /2)
+                if self.next_gate_height[2] is not height:
+                    self.next_gate_height.append(height)
+                if len(self.next_gate_height) > 3:
+                    self.next_gate_height.pop(0)
+                #pro_screen = self.vision_stun.draw_rectangles(screenshot_g, rectangles_g)
+                #cv.imshow("screenshot_g", pro_screen)
+                #cv.waitKey(150)
+            else:
+                pass
+
+            if len(rectangles_f) is not 0:
+                self.current_pet_height.append(rectangles_f[0][1] + rectangles_f[0][3] + self.offset_down)
+                if len(self.current_pet_height) > 5:
+                    self.current_pet_height.pop(0)
+
+            #print("pet_pos: ", self.current_pet_height)
+            #print("next gate: ", self.next_gate_height)
+
+    def scale_picture(self, img, scale_percent):
+        width = int(img.shape[1] * scale_percent / 100)
+        height = int(img.shape[0] * scale_percent / 100)
+
+        resized_img = cv.resize(img, (width, height), interpolation=4)
+
+        gray = cv.cvtColor(resized_img, cv.COLOR_BGR2GRAY)
+        thresh = cv.threshold(gray, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1]
+
+        return thresh
+
+    def get_actual_pet_height(self):
+        return sum(self.current_pet_height) / len(self.current_pet_height)
+
+
+    def get_next_gate_height(self):
+        return self.next_gate_height
+
+
+
+
+
+

fruit.py

@@ -93,7 +93,7 @@ class Fruit(GameBase):
         #cv.waitKey(150)
         # continue
         data_coords = np.zeros((7, 11), dtype=object)
-        # field = Field()
+        # field = Pickaxe_Field()
         for needle_key in self.needles.keys():
             # gray_needle = cv.cvtColor(self.needles[needle_key], cv.COLOR_BGR2GRAY)
             # thresh_needle = cv.threshold(gray_needle, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1]

keyboard_thread.py

@@ -0,0 +1,32 @@
+import threading
+import cv2 as cv
+import keyboard
+
+
+class KeyboardEvent(threading.Thread):
+
+    def __init__(self):
+        threading.Thread.__init__(self)
+
+        self.run_mode = 'run'
+        self.offset_value = 0
+        self.start()
+    def run(self):
+        while self.run_mode == 'run':
+            if keyboard.is_pressed('up'):  # if key 'q' is pressed
+                self.offset_value = -75
+            elif keyboard.is_pressed('down'):
+                self.offset_value = 75
+            elif keyboard.is_pressed('left') or keyboard.is_pressed('right'):
+                self.offset_value = 0
+            cv.waitKey(10)
+
+    def callback(self):
+        pass
+
+    def destroy(self):
+        self.destroy()
+
+    def get_run_mode(self):
+        return self.run_mode
+

litris.py

@@ -0,0 +1,235 @@
+from copy import copy
+
+import cv2 as cv
+import numpy as np
+from time import time
+from game_base_class import GameBase
+from pynput.keyboard import Key, Controller
+from field import Field
+from tetromino import Tetromino
+from optimizer import Optimizer
+from litris_stone_id_thread import NewStoneID
+
+class Litris(GameBase):
+
+    def __init__(self, overlay):
+        super().__init__(overlay)
+
+        self.keyboard = Controller()
+
+        self.offset_left = 610
+        self.offset_down = 40
+
+        self.field = Field()
+
+        self.litris_reset_board = cv.imread("control_elements/sodoku_reset_button.jpg", cv.IMREAD_COLOR)
+
+        self.stone_id_thread = NewStoneID()
+
+        self.move_mode = 1
+        self.moved_around_full = False
+
+        self.field_state_storage = {
+            1: self.field,
+            2: self.field,
+            3: self.field,
+            4: self.field
+        }
+
+    def reset(self):
+        self.field.reset_field()
+        self.field.cleared_rows = 1
+        self.move_mode = 1
+        self.moved_around_full = False
+
+    def assess_playfield_and_make_move(self):
+        last_letter_received = time()
+        while True:
+            if self.stone_id_thread.get_pick_up_status() == False:
+                if (time() - last_letter_received) >= 5:
+                    self.reset()
+                    last_letter_received = time()
+                    self.dig_point(1500, 980, 100)
+                    if self.overlay.run_mode == 'stopped' or self.overlay.run_mode == 'paused':
+                        return
+                    continue
+                if self.overlay.run_mode == 'stopped' or self.overlay.run_mode == 'paused':
+                    return
+                cv.waitKey(25)
+                continue
+
+
+            current_letter = self.stone_id_thread.get_actual_letter()
+            print("current_letter: ", current_letter)
+            last_letter_received = time()
+
+
+            current_tetromino = Tetromino.create(current_letter)
+            if self.move_mode == 2:
+                current_tetromino.rotate(3)
+            elif self.move_mode == 3:
+                current_tetromino.rotate(2)
+            elif self.move_mode == 4:
+                current_tetromino.rotate(1)
+
+
+            opt = Optimizer.get_optimal_drop(self.field, current_tetromino)
+            rotation = opt['tetromino_rotation']
+            column = opt['tetromino_column']
+            print("Rota:", rotation)
+            print("column:", column)
+            current_tetromino.rotate(rotation)
+            offset_col = current_tetromino.get_offset_column(rotation, self.move_mode)
+            print("offset column:", offset_col)
+            self.field.drop(current_tetromino, column)
+            self.move_stone(column - offset_col, rotation)
+            self.drop_down()
+            print(self.field)
+            if self.field.get_line_count() >= 6 and self.field.height() <= 2 and self.field.check_crucial_pos_to_be_free():
+                if self.field.predict_gaps_in_next_rotation() <= 3:
+                    self.field_state_storage[self.move_mode] = copy(self.field)
+                    self.update_move_mode()
+                    #self.field.state = self.stone_id_thread.get_current_board_state()
+                    self.field.rotate_state()
+                    #self.update_move_mode()
+                    self.field.reset_half_field()
+                    self.update_field_with_stored_edges()
+                    #field_mem = copy(self.field)
+                    #self.field = copy(self.field_mem)
+                    #self.field_mem = copy(field_mem)
+
+                    self.field.cleared_rows = 1
+
+                    cv.waitKey(100)
+            self.stone_id_thread.set_pick_up_status(False)
+
+    def update_field_with_stored_edges(self):
+        if self.move_mode == 1 and self.moved_around_full:
+            state_copy = Field.rotate_90_degree_clckwise(copy(self.field_state_storage[2].state))
+            for i in range(11,20,1):
+                for e in range(0,2,1):
+                    self.field.state[i][e] = state_copy[i][e]
+        elif self.move_mode == 2 and self.moved_around_full:
+            state_copy = Field.rotate_90_degree_clckwise(copy(self.field_state_storage[3].state))
+            for i in range(11,20,1):
+                for e in range(0,2,1):
+                    self.field.state[i][e] = state_copy[i][e]
+        elif self.move_mode == 3 and self.moved_around_full:
+            state_copy = Field.rotate_90_degree_clckwise(copy(self.field_state_storage[4].state))
+            for i in range(11, 20, 1):
+                for e in range(0, 2, 1):
+                    self.field.state[i][e] = state_copy[i][e]
+        elif self.move_mode == 4:
+            state_copy = Field.rotate_90_degree_clckwise(copy(self.field_state_storage[1].state))
+            for i in range(11,20,1):
+                for e in range(0,2,1):
+                    self.field.state[i][e] = state_copy[i][e]
+            self.moved_around_full = True
+
+    def drop_down(self):
+        if self.move_mode == 1:
+            down = Key.down
+            left = Key.left
+            right = Key.right
+        elif self.move_mode == 2:
+            down = Key.left
+            left = Key.up
+            right = Key.down
+        elif self.move_mode == 3:
+            down = Key.up
+            left = Key.right
+            right = Key.left
+        elif self.move_mode == 4:
+            down = Key.right
+            left = Key.down
+            right = Key.up
+        for i in range(1, 10, 1):
+            self.keyboard.press(down)
+            self.keyboard.release(down)
+            print("drop down pressed:", down)
+            cv.waitKey(40)
+
+    def update_move_mode(self):
+        if self.move_mode <=3:
+            self.move_mode = self.move_mode + 1
+        elif self.move_mode == 4:
+            self.move_mode = 1
+
+    def move_stone(self, col_movement, rotation):
+        if col_movement is None:
+            return
+
+        if self.move_mode == 1:
+            down = Key.down
+            left = Key.left
+            right = Key.right
+        elif self.move_mode == 2:
+            down = Key.left
+            left = Key.up
+            right = Key.down
+        elif self.move_mode == 3:
+            down = Key.up
+            left = Key.right
+            right = Key.left
+        elif self.move_mode == 4:
+            down = Key.right
+            left = Key.down
+            right = Key.up
+
+        # Press and release space
+        self.keyboard.press(down)
+        self.keyboard.release(down)
+        print("direction pressed: ", down)
+        cv.waitKey(60)
+
+        if rotation == 3:
+            self.keyboard.press('e')
+            self.keyboard.release('e')
+            print("rotation 1 pressed: e")
+            cv.waitKey(30)
+        elif rotation == 2:
+            self.keyboard.press('e')
+            self.keyboard.release('e')
+            print("rotation 2 pressed: e 1")
+            cv.waitKey(30)
+            self.keyboard.press('e')
+            self.keyboard.release('e')
+            print("rotation 2 pressed: e 2")
+            cv.waitKey(30)
+        elif rotation == 1:
+            self.keyboard.press('e')
+            self.keyboard.release('e')
+            print("rotation 3 pressed: e 1")
+            cv.waitKey(30)
+            self.keyboard.press('e')
+            self.keyboard.release('e')
+            print("rotation 3 pressed: e 2")
+            cv.waitKey(30)
+            self.keyboard.press('e')
+            self.keyboard.release('e')
+            print("rotation 3 pressed: e 3")
+            cv.waitKey(30)
+
+        if col_movement < 0:
+            for i in range(0, col_movement, - 1):
+                self.keyboard.press(left)
+                self.keyboard.release(left)
+                print("move left 3 pressed:", left)
+                cv.waitKey(30)
+        else:
+            for i in range(0, col_movement, 1):
+                self.keyboard.press(right)
+                self.keyboard.release(right)
+                print("move right 3 pressed:", right)
+                cv.waitKey(30)
+
+    def point_in_rect(self, point):
+        for e in range(0, 20, 1):
+            for i in range(0, 20, 1):
+                x1, y1, w, h = self.data_coordinates[e][i]
+                x2, y2 = x1 + w, y1 + h
+                x, y = point
+                if x1 < x and x < x2:
+                    if y1 < y and y < y2:
+                        return e, i
+        return None, None

litris_stone_id_thread.py

@@ -0,0 +1,193 @@
+import threading
+import cv2 as cv
+import numpy as np
+from window_capture import WindowCapture
+from vision import Vision
+from config_file import UserConfigs
+
+O_FULL = [[0, 0, 0, 0], [0, 1, 1, 0], [0, 1, 1, 0], [0, 0, 0, 0]]
+D_FULL = [[0, 0, 0, 0], [0, 1, 1, 0], [0, 0, 1, 0], [0, 0, 0, 0]]
+L_FULL = [[0, 0, 0, 0], [0, 0, 1, 0], [1, 1, 1, 0], [0, 0, 0, 0]]
+J_FULL = [[0, 0, 0, 0], [1, 1, 1, 0], [0, 0, 1, 0], [0, 0, 0, 0]]
+I_FULL = [[0, 0, 0, 0], [1, 1, 1, 1], [0, 0, 0, 0], [0, 0, 0, 0]]
+C_FULL = [[0, 0, 0, 0], [0, 0, 1, 0], [0, 0, 0, 0], [0, 0, 0, 0]]
+B_FULL = [[0, 0, 0, 0], [0, 1, 1, 0], [0, 0, 0, 0], [0, 0, 0, 0]]
+A_FULL = [[0, 0, 0, 0], [0, 1, 1, 1], [0, 0, 0, 0], [0, 0, 0, 0]]
+S_FULL = [[0, 0, 0, 0], [0, 0, 1, 1], [0, 1, 1, 0], [0, 0, 0, 0]]
+Z_FULL = [[0, 0, 0, 0], [0, 1, 1, 0], [0, 0, 1, 1], [0, 0, 0, 0]]
+T_FULL = [[0, 0, 0, 0], [0, 1, 1, 1], [0, 0, 1, 0], [0, 0, 0, 0]]
+class NewStoneID(threading.Thread):
+
+    def __init__(self):
+        threading.Thread.__init__(self)
+
+        self.config = UserConfigs()
+        self.capture_window = WindowCapture(None, None, self.config)
+        self.vision_stun = Vision()
+
+        self.stone_coordinates = np.zeros((4, 4), dtype=object)
+        self.data_coordinates = np.zeros((20, 20), dtype=object)
+        self.fill_data_coordinates()
+
+        self.needles = {'Q': cv.imread("litris/blue_needle.jpg", cv.IMREAD_UNCHANGED)}
+
+        self.run_mode = 'run'
+        self.actual_letter = ""
+        self.to_pick_up = False
+
+        self.start()
+
+    def run(self):
+        while self.run_mode == 'run':
+            current_stone = self.new_stone_detection()
+            if current_stone is None:
+                cv.waitKey(50)
+                continue
+            else:
+                current_letter = self.get_letter_for_stone(current_stone)
+                if current_letter is None:
+                    continue
+                else:
+                    self.actual_letter = current_letter
+                    self.to_pick_up = True
+                    while self.to_pick_up:
+                        cv.waitKey(10)
+                    self.actual_letter = ""
+
+    def get_actual_letter(self):
+        return self.actual_letter
+    def set_pick_up_status(self, status):
+        self.to_pick_up = status
+    def get_pick_up_status(self):
+        return self.to_pick_up
+
+    def callback(self):
+        pass
+
+    def destroy(self):
+        self.destroy()
+
+    def get_run_mode(self):
+        return self.run_mode
+
+
+    def new_stone_detection(self):
+
+        screenshot = self.capture_window.get_screenshot()
+        screenshot = screenshot[580:845, 1148:1412]
+
+        rectangles = self.vision_stun.find(screenshot, self.needles['Q'], 0.85, 16)
+        if len(rectangles) == 0:
+            return None
+
+        points = self.vision_stun.get_click_points(rectangles)
+
+        stone_coords = np.zeros((4, 4), dtype=object)
+        for point in points:
+            x, y = self.point_in_small_rect(point)
+            if x is not None and y is not None:
+                stone_coords[x][y] = 1
+                # self.change_value(x, y, int(needle_key))
+                # print(field.data_value_grid)
+                #cv.circle(screenshot, points[0], 7, (0, 255, 0), -1)
+                #output_image = self.vision_stun.draw_rectangles(screenshot, rectangles)
+                #cv.imshow("output_image", output_image)
+                #cv.waitKey(150)
+
+        return stone_coords
+
+    def get_letter_for_stone(self, stone):
+
+        if np.array_equal(stone, O_FULL):
+            return "O"
+        elif np.array_equal(stone, D_FULL):
+            return "D"
+        elif np.array_equal(stone, L_FULL):
+            return "L"
+        elif np.array_equal(stone, J_FULL):
+            return "J"
+        elif np.array_equal(stone, I_FULL):
+            return "I"
+        elif np.array_equal(stone, C_FULL):
+            return "C"
+        elif np.array_equal(stone, B_FULL):
+            return "B"
+        elif np.array_equal(stone, A_FULL):
+            return "A"
+        elif np.array_equal(stone, S_FULL):
+            return "S"
+        elif np.array_equal(stone, Z_FULL):
+            return "Z"
+        elif np.array_equal(stone, T_FULL):
+            return "T"
+        else:
+         return None
+
+    def point_in_small_rect(self, point):
+        for e in range(0, 4, 1):
+            for i in range(0, 4, 1):
+                x1, y1, w, h = self.stone_coordinates[e][i]
+                x2, y2 = x1 + w, y1 + h
+                x, y = point
+                if x1 < x and x < x2:
+                    if y1 < y and y < y2:
+                        return e, i
+        return None, None
+
+    def get_current_board_state(self):
+
+        # get an updated image of the game
+        screenshot = self.capture_window.get_screenshot()
+        #screenshot = cv.imread("litris/main_playfield.jpg")
+        screenshot = screenshot[40:1380, 610:1950]  # 1000,1000
+
+        # cv.imshow("screenshot", screenshot)
+        # cv.waitKey(150)
+        # continue
+        #data_coords = np.zeros((20, 20), dtype=object)
+        data_coords = np.full((20, 20), ' ', dtype=object)
+        #field = Pickaxe_Field()
+        for needle_key in self.needles.keys():
+            #gray_needle = cv.cvtColor(self.needles[needle_key], cv.COLOR_BGR2GRAY)
+            #thresh_needle = cv.threshold(gray_needle, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1]
+            rectangles = self.vision_stun.find(screenshot, self.needles[needle_key], 0.8, 200)
+            if len(rectangles) == 0:
+                continue
+            points = self.vision_stun.get_click_points(rectangles)
+
+            for point in points:
+                x, y = self.point_in_rect(point)
+                if x is not None and y is not None:
+                    data_coords[x][y] = needle_key
+                    #self.change_value(x, y, int(needle_key))
+                    # print(field.data_value_grid)
+                    # cv.circle(screenshot, points[0], 7, (0, 255, 0), -1)
+                    #output_image = self.vision_stun.draw_rectangles(screenshot, rectangles)
+                    #cv.imshow("output_image", output_image)
+                    #cv.waitKey(150)
+        return data_coords.tolist()
+
+    def point_in_rect(self, point):
+        for e in range(0, 20, 1):
+            for i in range(0, 20, 1):
+                x1, y1, w, h = self.data_coordinates[e][i]
+                x2, y2 = x1 + w, y1 + h
+                x, y = point
+                if x1 < x and x < x2:
+                    if y1 < y and y < y2:
+                        return e, i
+        return None, None
+
+    def fill_data_coordinates(self):
+        # 610 to 1950 = 1340 - 76 / 20 = 63
+        # 40 to 1380 = 1340 - 76 / 20 = 63
+        # spacing 19 * 4
+        dim = 63
+        e_spacing = 4
+        i_spacing = 4
+        for e in range(0, 4, 1):
+            for i in range(0, 4, 1):
+                self.stone_coordinates[e][i] = [(i * dim) + (i * i_spacing), (e * dim) + (e * e_spacing), dim, dim]
+        for e in range(0, 20, 1):
+            for i in range(0, 20, 1):
+                self.data_coordinates[e][i] = [(i * dim) + (i * i_spacing), (e * dim) + (e * e_spacing), dim, dim]

menti_dic

@@ -0,0 +1,144 @@
+HIT
+IT
+HI
+POT
+TOP
+TO
+HAS
+ASH
+HA
+AS
+AH
+NOW
+NO
+ON
+OWN
+WON
+WHEN
+NEW
+HE
+WE
+FORM
+OR
+OF
+FROM
+FOR
+HIST
+HIS
+HITS
+THIS
+SIT
+ITS
+FIND
+IF
+FIN
+IN
+BEST
+BET
+SET
+BE
+HAT
+NAT
+TAN
+ANT
+AN
+THAN
+RACE
+ACE
+ACER
+CARE
+ERA
+ARC
+CAR
+EAR
+TEA
+LATE
+LET
+ATE
+EAT
+TALE
+HIRE
+TIRE
+THEIR
+TIE
+TIER
+TAB
+BOAT
+BUT
+OUT
+ABOUT
+AUTO
+HER
+HERO
+OTHER
+TOE
+HOT
+RICE
+PRICE
+RIP
+PIE
+RIPE
+ICE
+FIRST
+FIST
+SIR
+FIT
+FITS
+SAT
+TASTE
+STATE
+SEA
+EAST
+TEST
+MAIL
+EMAIL
+MALE
+LIE
+AIM
+MILE
+FEAR
+ART
+RATE
+AFTER
+FAR
+ARE
+DEAF
+FACE
+CAFE
+FED
+FACED
+INK
+LINK
+LINKS
+SIN
+SKIN
+SKI
+HERO
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

menti_words.py

@@ -0,0 +1,185 @@
+import cv2 as cv
+import numpy as np
+import pydirectinput
+from nltk.corpus import words
+from pytesseract import pytesseract
+
+import utils
+from utils import mse
+from game_base_class import GameBase
+import random
+
+GREEN = 1
+YELLOW = 2
+RED = 3
+BLUE = 4
+ORANGE = 5
+
+
+class MentiWords(GameBase):
+
+    def __init__(self, overlay):
+        super().__init__(overlay)
+
+        pytesseract.tesseract_cmd = r'C:\Program Files\Tesseract-OCR\tesseract.exe'
+
+        self.observation = np.zeros((9, 9), dtype=int)
+
+        self.colors = [1, 2, 3, 4, 5, 6, 7, 8, 9]
+
+        self.offset_left = 1080
+        self.offset_down = 870
+
+        self.menti_play_button = cv.imread("control_elements/play.jpg", cv.IMREAD_COLOR)
+
+        self.needles = {
+            'A': cv.imread("menti_words/a.jpg", cv.IMREAD_COLOR),
+            'B': cv.imread("menti_words/b.jpg", cv.IMREAD_COLOR),
+            'C': cv.imread("menti_words/c.jpg", cv.IMREAD_COLOR),
+            'D': cv.imread("menti_words/d.jpg", cv.IMREAD_COLOR),
+            'E': cv.imread("menti_words/e.jpg", cv.IMREAD_COLOR),
+            'F': cv.imread("menti_words/f.jpg", cv.IMREAD_COLOR),
+
+            'H': cv.imread("menti_words/h.jpg", cv.IMREAD_COLOR),
+            'I': cv.imread("menti_words/i.jpg", cv.IMREAD_COLOR),
+
+            'L': cv.imread("menti_words/l.jpg", cv.IMREAD_COLOR),
+            'M': cv.imread("menti_words/m.jpg", cv.IMREAD_COLOR),
+            'N': cv.imread("menti_words/n.jpg", cv.IMREAD_COLOR),
+            'O': cv.imread("menti_words/o.jpg", cv.IMREAD_COLOR),
+            'P': cv.imread("menti_words/p.jpg", cv.IMREAD_COLOR),
+
+            'R': cv.imread("menti_words/r.jpg", cv.IMREAD_COLOR),
+            'S': cv.imread("menti_words/s.jpg", cv.IMREAD_COLOR),
+            'T': cv.imread("menti_words/t.jpg", cv.IMREAD_COLOR),
+            'U': cv.imread("menti_words/u.jpg", cv.IMREAD_COLOR),
+
+            'W': cv.imread("menti_words/w.jpg", cv.IMREAD_COLOR)
+
+                        }
+        self.masks = {
+            'A': cv.imread("menti_words/a-mask.png", cv.IMREAD_COLOR),
+            'B': cv.imread("menti_words/b-mask.png", cv.IMREAD_COLOR),
+            'C': cv.imread("menti_words/c-mask.png", cv.IMREAD_COLOR),
+            'D': cv.imread("menti_words/d-mask.png", cv.IMREAD_COLOR),
+            'E': cv.imread("menti_words/e-mask.png", cv.IMREAD_COLOR),
+            'F': cv.imread("menti_words/f-mask.png", cv.IMREAD_COLOR),
+
+            'H': cv.imread("menti_words/h-mask.png", cv.IMREAD_COLOR),
+            'I': cv.imread("menti_words/i-mask.png", cv.IMREAD_COLOR),
+
+            'L': cv.imread("menti_words/l-mask.png", cv.IMREAD_COLOR),
+            'M': cv.imread("menti_words/m-mask.png", cv.IMREAD_COLOR),
+            'N': cv.imread("menti_words/n-mask.png", cv.IMREAD_COLOR),
+            'O': cv.imread("menti_words/o-mask.png", cv.IMREAD_COLOR),
+            'P': cv.imread("menti_words/p-mask.png", cv.IMREAD_COLOR),
+
+            'R': cv.imread("menti_words/r-mask.png", cv.IMREAD_COLOR),
+            'S': cv.imread("menti_words/s-mask.png", cv.IMREAD_COLOR),
+            'T': cv.imread("menti_words/t-mask.png", cv.IMREAD_COLOR),
+            'U': cv.imread("menti_words/u-mask.png", cv.IMREAD_COLOR),
+
+            'W': cv.imread("menti_words/w-mask.png", cv.IMREAD_COLOR)
+
+                        }
+
+        self.current_letters = []
+        self.letter_coords = {}
+
+        with open("menti_dic") as file:
+            self.word_list = [line.rstrip() for line in file]
+        #self.word_list2 = words.words()
+
+    def reset_lists(self):
+        self.current_letters = []
+        self.letter_coords = {}
+
+    def assess_playfield_and_make_move(self):
+        if self.check_for_button_and_execute(self.capture_window.get_screenshot(), self.menti_play_button):
+            cv.waitKey(2000)
+
+        self.reset_lists()
+        self.get_current_board_state()
+        if self.overlay.run_mode == 'stopped' or self.overlay.run_mode == 'paused':
+            return
+        matches = self.possible_words(self.word_list, self.current_letters)
+        matches.sort()
+        new_matches = list(map(list,matches))
+        for letter_list in new_matches:
+            self.type_word_into_ui(letter_list)
+            if self.overlay.run_mode == 'stopped' or self.overlay.run_mode == 'paused':
+                return
+
+    def type_word_into_ui(self, letters):
+
+        for letter in letters:
+            pydirectinput.moveTo(self.letter_coords[letter][0] + self.offset_left, self.letter_coords[letter][1] + self.offset_down)
+            pydirectinput.mouseDown()
+            w = random.randint(25, 50)
+            cv.waitKey(100 + w)
+            if self.overlay.run_mode == 'stopped' or self.overlay.run_mode == 'paused':
+                return
+        pydirectinput.mouseUp()
+        cv.waitKey(400)
+
+
+    def get_current_board_state(self):
+
+        #screenshot = cv.imread("menti_words/screenshot.jpg")
+        screenshot = self.capture_window.get_screenshot()
+        screenshot = utils.scale_screenshot(screenshot[870:1270, 1080:1480], 200, False)
+
+        #cv.imshow("screenshot", screenshot)
+        #cv.waitKey(150)
+        #continue
+        for needle_key in self.needles.keys():
+            # gray_needle = cv.cvtColor(self.needles[needle_key], cv.COLOR_BGR2GRAY)
+            # thresh_needle = cv.threshold(gray_needle, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1]
+
+            thresh_needle = utils.scale_screenshot(self.needles[needle_key], 200, False)
+            rectangles = self.vision_stun.find(screenshot, thresh_needle , 0.85, 2)
+            #rectangles = self.vision_stun.find(screenshot, self.needles[needle_key], 0.95, 1 ,True, self.masks[needle_key])
+            if len(rectangles) == 0:
+                continue
+            points = self.vision_stun.get_click_points(rectangles)
+            for point in points:
+                self.current_letters.append(needle_key)
+                self.letter_coords[needle_key] = (int(point[0]/2), int(point[1]/2))
+            '''
+            cropped1 = self.vision_stun.draw_display_picture(screenshot, rectangles, 10)
+            #cropped1 = utils.scale_screenshot(cropped1)
+            cv.imshow("cropped1", cropped1)
+            cv.waitKey(150)
+            text_1 = pytesseract.image_to_string(cropped1, lang='eng', config='--psm 6').strip()
+            if str.isalpha(text_1):
+                #cv.imshow("cropped1", cropped1)
+                #cv.waitKey(150)
+                points = self.vision_stun.get_click_points(rectangles)
+                self.current_letters.append(text_1)
+                self.letter_coords[text_1] = points[0]'''
+            if self.overlay.run_mode == 'stopped' or self.overlay.run_mode == 'paused':
+                return
+
+    def possible_words(self, lwords, charSet):
+        lst = []
+        for word in lwords:
+            if len(word) <= 1:
+                continue
+            flag = 1
+            chars = self.charCount(word)
+            for key in chars:
+                if key not in charSet:
+                    flag = 0
+                else:
+                    if charSet.count(key) != chars[key]:
+                        flag = 0
+            if flag == 1:
+                lst.append(word)
+                #print(word)
+        return list(set(lst))
+
+    def charCount(self, word):
+        dict = {}
+        for i in word:
+            dict[i] = dict.get(i, 0) + 1
+        return dict