added crop game first shot

crop/Field_Representation_crop.py

@@ -0,0 +1,661 @@
+import random
+import cv2 as cv
+import numpy as np
+import pydirectinput
+from window_capture import WindowCapture
+from vision import Vision
+from config_file import UserConfigs
+from utils import mse
+from utils import get_click_point
+
+GREEN = 1
+YELLOW = 2
+BLUE = 3
+RED = 4
+PURPLE = 5
+RAINBOW = 6
+BIGBOMB = 7
+BOMB = 8
+ARROW_DOWN = 9
+ARROW_RIGHT = 10
+ROCK_1 = 11
+ROCK_2 = 12
+ROCK_3 = 13
+BURGER = 14
+PAB1 = 15
+GOLDBAR = 16
+MAGINENT = 21
+CHEMTRANT = 22
+TENESENT = 23
+CIBUTRANT = 24
+ARTISENT = 25
+
+STATUS_FOUND_CONTINUATION = "1"
+STATUS_FOUND_DEADEND = "2"
+STATUS_FOUND_ENDING = "3"
+
+
+class Field:
+    data_value_grid = []
+    data_coordinates = []
+    next_level = cv.imread("next_level.jpg", cv.IMREAD_COLOR)
+    next_level_x = cv.imread("next_level_x.jpg", cv.IMREAD_COLOR)
+    explosives = []
+    explosives.append(RAINBOW)
+    explosives.append(ARROW_RIGHT)
+    explosives.append(ARROW_DOWN)
+    explosives.append(BIGBOMB)
+    explosives.append(BOMB)
+    colors = []
+    colors.append(GREEN)
+    colors.append(YELLOW)
+    colors.append(BLUE)
+    colors.append(RED)
+    colors.append(PURPLE)
+
+    # colors.append(MAGINENT)
+    # colors.append(CHEMTRANT)
+    # colors.append(TENESENT)
+    # colors.append(CIBUTRANT)
+    # colors.append(ARTISENT)
+
+    def __init__(self):
+        self.data_value_grid = np.zeros((8, 14), dtype=int)
+        self.data_coordinates = np.zeros((8, 14), dtype=object)
+        self.data_score_map = np.zeros((8, 14), dtype=int)
+        self.observation = np.zeros((8, 14), dtype=int)
+
+        # 230 to 2110 = 1883 / 14 = 134.5
+        # 60 to 1130 = 1076 / 8 =  134.5
+        dim = 134.5
+        for e in range(0, 8, 1):
+            for i in range(0, 14, 1):
+                self.data_coordinates[e][i] = [i * dim, e * dim, dim, dim]
+
+        # initialize the user-class
+        self.config = UserConfigs()
+
+        # initialize the StunWindowCapture class
+        self.capture_window = WindowCapture(None, None, self.config)
+        # initialize the StunVision class
+        self.vision_stun = Vision()
+
+        self.needles = {GREEN: cv.imread("green.jpg", cv.IMREAD_COLOR),
+                        YELLOW: cv.imread("yellow.jpg", cv.IMREAD_COLOR),
+                        PURPLE: cv.imread("purple.jpg", cv.IMREAD_COLOR),
+                        BLUE: cv.imread("blue.jpg", cv.IMREAD_COLOR),
+                        RED: cv.imread("red.jpg", cv.IMREAD_COLOR)
+
+                        # RAINBOW: cv.imread("rainbow.jpg", cv.IMREAD_COLOR),
+                        # BIGBOMB: cv.imread("bigbomb.jpg", cv.IMREAD_COLOR),
+                        # BOMB: cv.imread("bomb.jpg", cv.IMREAD_COLOR),
+                        # ARROW_DOWN: cv.imread("arrow_down.jpg", cv.IMREAD_COLOR),
+                        # ARROW_RIGHT: cv.imread("arrow_right.jpg", cv.IMREAD_COLOR),
+                        # ROCK_1: cv.imread("rock1.jpg", cv.IMREAD_COLOR),
+                        # ROCK_2: cv.imread("rock2.jpg", cv.IMREAD_COLOR),
+                        # ROCK_3: cv.imread("rock3.jpg", cv.IMREAD_COLOR),
+                        # BURGER: cv.imread("burger.jpg", cv.IMREAD_COLOR),
+                        # GOLDBAR: cv.imread("burger.jpg", cv.IMREAD_COLOR),
+                        # PAB1: cv.imread("pab1.jpg", cv.IMREAD_COLOR),
+                        # MAGINENT: cv.imread("maginent.jpg", cv.IMREAD_COLOR),
+                        # CHEMTRANT: cv.imread("chemtrant.jpg", cv.IMREAD_COLOR),
+                        # TENESENT: cv.imread("tenesent.jpg", cv.IMREAD_COLOR),
+                        # CIBUTRANT: cv.imread("cibutrant.jpg", cv.IMREAD_COLOR),
+                        # ARTISENT: cv.imread("artisent.jpg", cv.IMREAD_COLOR)
+                        }
+
+    def reset(self):
+        self.observation = []
+
+    def assess_playfield_and_make_move(self):
+
+        new_observation, new_screenshot = self.get_current_board_state()
+
+        # wrong movement detection
+        # last board state is same as actual
+        if mse(new_observation, self.observation) == 0.0:
+            # no movement detected -> blow explosives or reset
+            #self.detonate_explosive_when_stuck(new_observation)
+            cv.waitKey(500)
+
+        self.find_patterns_and_valid_moves(new_observation)
+        self.observation = new_observation
+        return new_observation
+
+    def move_to(self, x, y):
+        point_src = (1113, 598)
+        pydirectinput.moveTo(point_src[0], point_src[1])
+        pydirectinput.mouseDown()
+        w = random.randint(1, 100)
+        cv.waitKey(150 + w)
+        pydirectinput.moveTo(x, y)
+        pydirectinput.mouseUp()
+        cv.waitKey(500 + w)
+
+    def change_value(self, x, y, val):
+        self.data_value_grid[x][y] = val
+
+    def pointInRect(self, point):
+        for e in range(0, 8, 1):
+            for i in range(0, 14, 1):
+                x1, y1, w, h = self.data_coordinates[e][i]
+                x2, y2 = x1 + w, y1 + h
+                x, y = point
+                if (x1 < x and x < x2):
+                    if (y1 < y and y < y2):
+                        return e, i
+        return None, None
+
+    def check_for_next_level(self, screen, needle):
+        offset_left = 230
+        offset_down = 58
+        rectangles = self.vision_stun.find(screen, needle, 0.70, 1)
+        if len(rectangles) == 0:
+            return False
+        point = self.vision_stun.get_click_points(rectangles)[0]
+        self.dig_point(point[0] + offset_left, point[1] + offset_down, 500)
+        return True
+
+    def get_current_board_state(self):
+        try:
+            # get an updated image of the game
+            screenshot = self.capture_window.get_screenshot()
+            #screenshot = cv.imread("playfield.jpg")
+            screenshot = screenshot[58:1134, 230:2113]  # 1883,1076
+            # gray = cv.cvtColor(screenshot, cv.COLOR_BGR2GRAY)
+            # thresh = cv.threshold(gray, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1]
+            '''
+            if self.check_for_next_level(screenshot, self.next_level):
+                cv.waitKey(500)
+                screenshot = self.capture_window.get_screenshot()
+                screenshot = screenshot[58:1134, 230:2113]
+            if self.check_for_next_level(screenshot, self.next_level_x):
+                cv.waitKey(500)
+                screenshot = self.capture_window.get_screenshot()
+                screenshot = screenshot[58:1134, 230:2113]
+            '''
+        except:
+            # self.capture_window.release()
+            # print("Game window not available - shutting down application")
+            # return None
+            pass
+        # cv.imshow("screenshot", screenshot)
+        # cv.waitKey(150)
+        # continue
+        data_coords = np.zeros((8, 14), dtype=object)
+        # field = Field()
+        for needle_key in self.needles.keys():
+            # gray_needle = cv.cvtColor(self.needles[needle_key], cv.COLOR_BGR2GRAY)
+            # thresh_needle = cv.threshold(gray_needle, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1]
+            rectangles = self.vision_stun.find(screenshot, self.needles[needle_key], 0.70, 56)
+            if len(rectangles) == 0:
+                continue
+            points = self.vision_stun.get_click_points(rectangles)
+
+            for point in points:
+                x, y = self.pointInRect(point)
+                if x is not None and y is not None:
+                    data_coords[x][y] = int(needle_key)
+                    # self.change_value(x, y, int(needle_key))
+                    # print(field.data_value_grid)
+                    # cv.circle(screenshot, points[0], 7, (0, 255, 0), -1)
+                    # output_image = vision_stun.draw_rectangles(screenshot, rectangles)
+                    # cv.imshow("output_image", output_image)
+                    # cv.waitKey(150)
+
+        #score_map = np.zeros((8, 14), dtype=object)
+        #for x in range(0, 8, 1):
+        #    for y in range(0, 14, 1):
+        #        score_map[x, y] = self.score_for_attached_same_color_all_directions(data_coords, x, y)
+        #        cv.putText(screenshot, str(score_map[x, y]),
+        #                   self.get_click_point(self.data_coordinates[x, y]), cv.FONT_HERSHEY_SIMPLEX,
+        #                   1, (0, 0, 0), 3, 2)
+        #cv.imwrite('screenshot_scored.jpg', screenshot)
+
+        return data_coords, screenshot
+
+    def find_patterns_and_valid_moves(self, state):
+
+        #score_map = np.zeros((8, 14), dtype=object)
+        for x in range(0, 8, 1):
+            for y in range(0, 14, 1):
+                self.data_score_map[x, y] = self.score_for_attached_same_color_all_directions(state, x, y)
+
+        full_moves = []
+        reserve_moves = []
+        for x in range(0, 8, 1):
+            for y in range(0, 14, 1):
+                if self.data_score_map[x, y] >= 4:
+                    path_option = []
+                    path_option.append([x, y])
+                    recursion_reminder = []
+                    recursion_reminder.append([x, y])
+                    result = self.find_next_same_color_all_directions_recursion(state, x, y, path_option, recursion_reminder, True)
+                    if result == STATUS_FOUND_ENDING:
+                        path_option.append([x, y])
+                    full_moves.append(path_option)
+                if self.data_score_map[x, y] >= 3:
+                    path_option = []
+                    path_option.append([x, y])
+                    recursion_reminder = []
+                    recursion_reminder.append([x, y])
+                    self.find_next_same_color_all_directions_recursion(state, x, y, path_option, recursion_reminder, False)
+                    if len(path_option) >= 3:
+                        reserve_moves.append(path_option)
+        #print(self.data_score_map)
+
+        #screenshot = cv.imread("screenshot_scored.jpg")
+
+
+        full_moves.sort(key=len, reverse=True)
+        reserve_moves.sort(key=len, reverse=True)
+        '''
+        for move in full_moves:
+            mp = []
+            for point in move:
+                cp = self.get_click_point(self.data_coordinates[point[0], point[1]])
+                mp.append(cp)
+            cv.polylines(screenshot,
+                              [np.array(mp)],
+                              isClosed=False,
+                              color=(0, 255, 0),
+                              thickness=3)
+
+        cv.imwrite('screenshot_pathed.jpg', screenshot)
+        '''
+
+        if len(full_moves) >= 1:
+            self.execute_move(full_moves)
+        elif len(reserve_moves) >= 1:
+            self.execute_move(reserve_moves)
+        else:
+            # boelk
+            pass
+
+    def execute_move(self, moves):
+        offset_left = 230
+        offset_down = 58
+        if len(moves) >= 1:
+            first = True
+            for cords in moves[0]:
+                square = self.data_coordinates[cords[0], cords[1]]
+                pydirectinput.moveTo(get_click_point(square)[0] + offset_left, get_click_point(square)[1]+offset_down)
+                cv.waitKey(50)
+                if first:
+                    pydirectinput.mouseDown()
+                    cv.waitKey(100)
+                    first = False
+            pydirectinput.mouseUp()
+            cv.waitKey(50)
+
+    def get_directions_array(self, current_x, current_y):
+        left_x = current_x
+        left_y = current_y - 1
+
+        right_x = current_x
+        right_y = current_y + 1
+
+        upper_x = current_x - 1
+        upper_y = current_y
+
+        lower_x = current_x + 1
+        lower_y = current_y
+
+        lower_left_x = current_x + 1
+        lower_left_y = current_y - 1
+
+        lower_right_x = current_x + 1
+        lower_right_y = current_y + 1
+
+        upper_left_x = current_x - 1
+        upper_left_y = current_y - 1
+
+        upper_right_x = current_x - 1
+        upper_right_y = current_y + 1
+
+        directions = []
+        directions.append([left_x, left_y])
+        directions.append([lower_left_x, lower_left_y])
+        directions.append([lower_x, lower_y])
+        directions.append([lower_right_x, lower_right_y])
+        directions.append([right_x, right_y])
+        directions.append([upper_right_x, upper_right_y])
+        directions.append([upper_x, upper_y])
+        directions.append([upper_left_x, upper_left_y])
+
+        return directions
+
+    def find_next_same_color_all_directions_recursion(self, state, current_x, current_y, path_store, recursion_reminder, look_for_ending):
+        directions = self.get_directions_array(current_x, current_y)
+        color = state[current_x, current_y]
+
+        for direction in directions:
+            if self.is_direction_in_bounce_and_same_color(state, direction, color) == 1:
+                if (self.data_score_map[direction[0], direction[1]] >= 2) and (direction not in recursion_reminder):
+                    recursion_reminder.append(direction)
+                    result = self.find_next_same_color_all_directions_recursion(state, direction[0], direction[1], path_store , recursion_reminder, look_for_ending)
+                    if result == STATUS_FOUND_CONTINUATION:
+                        path_store.append(direction)
+                        return STATUS_FOUND_CONTINUATION
+                    elif result == STATUS_FOUND_DEADEND:
+                        if look_for_ending:
+                            continue
+                        else:
+                            path_store.append(direction)
+                            return STATUS_FOUND_DEADEND
+                    elif result == STATUS_FOUND_ENDING:
+                        path_store.append(direction)
+                        return STATUS_FOUND_ENDING
+
+        if look_for_ending:
+            for direction in directions:
+                if direction == path_store[0]:
+                    #path_store.append(direction)
+                    return STATUS_FOUND_ENDING
+
+        return STATUS_FOUND_DEADEND
+
+
+    def score_for_attached_same_color_all_directions(self, state, current_x, current_y):
+        left_x = current_x - 1
+        left_y = current_y
+
+        right_x = current_x + 1
+        right_y = current_y
+
+        upper_x = current_x
+        upper_y = current_y - 1
+
+        lower_x = current_x
+        lower_y = current_y + 1
+
+        lower_left_x = current_x + 1
+        lower_left_y = current_y - 1
+
+        lower_right_x = current_x + 1
+        lower_right_y = current_y + 1
+
+        upper_left_x = current_x - 1
+        upper_left_y = current_y - 1
+
+        upper_right_x = current_x - 1
+        upper_right_y = current_y + 1
+
+        directions = []
+        directions.append([left_x, left_y])
+        directions.append([right_x, right_y])
+        directions.append([lower_x, lower_y])
+        directions.append([upper_x, upper_y])
+        directions.append([lower_left_x, lower_left_y])
+        directions.append([lower_right_x, lower_right_y])
+        directions.append([upper_left_x, upper_left_y])
+        directions.append([upper_right_x, upper_right_y])
+
+        score = 0
+        color = state[current_x, current_y]
+        if color == 0:
+            return score
+        for direction in directions:
+            score = score + self.is_direction_in_bounce_and_same_color(state, direction, color)
+
+        return score
+
+
+    def is_direction_in_bounce_and_same_color(self, state, loc, color):
+        x, y = loc
+        if x <= 7 and x >= 0 and y <= 13 and y >= 0:
+            if state[x, y] == color:
+                return 1
+        return 0
+
+    def local_pos_check(self, state, e, i, e_check, i_check, needle):
+        if e + e_check >= 0 and e + e_check <= 7 and i + i_check >= 0 and i + i_check <= 13:
+            if state[e + e_check, i + i_check] == needle:
+                return True
+            else:
+                return False
+
+    def local_pos_checks(self, state, e, i, e_check, i_check, needles):
+        if e + e_check >= 0 and e + e_check <= 7 and i + i_check >= 0 and i + i_check <= 13:
+            for needle in needles:
+                if state[e + e_check, i + i_check] == needle:
+                    return True
+                else:
+                    continue
+        return False
+
+    def check_5_horizontal(self, state, e, i, color):
+        try:
+            #
+            if state[e, i] == color and state[e, i + 1] == color and state[e, i + 3] == color and state[
+                e, i + 4] == color:
+                if state[e, i + 2] <= 0 or state[e, i + 2] >= 6:
+                    return False
+                #
+                if e - 1 >= 0 and i + 2 <= 13:
+                    if state[e - 1, i + 2] == color:
+                        return True
+            return False
+        except:
+            return False
+
+    def check_5_vertical(self, state, e, i, color):
+        try:
+            #
+            if state[e, i] == color and state[e + 1, i] == color and state[e + 3, i] == color and state[
+                e + 4, i] == color:
+                if state[e + 2, i] <= 0 or state[e + 2, i] >= 6:
+                    return False
+                # third left upper
+                if e + 2 <= 7 and i - 1 >= 0:
+                    if state[e + 2, i - 1] == color:
+                        print("upper left", color, e, i)
+                        src_pt = self.get_click_point(self.data_coordinates[e + 2, i - 1])
+                        dest_pt = self.get_click_point(self.data_coordinates[e + 2, i])
+                        self.move_tile(src_pt, dest_pt)
+                        return True
+                if e + 2 <= 7 and i + 1 <= 13:
+                    if state[e + 2, i + 1] == color:
+                        print("upper left", color, e, i)
+                        src_pt = self.get_click_point(self.data_coordinates[e + 2, i + 1])
+                        dest_pt = self.get_click_point(self.data_coordinates[e + 2, i])
+                        self.move_tile(src_pt, dest_pt)
+                        return True
+            return False
+        except:
+            return False
+
+    def check_3_with_gap(self, state, e, i, color):
+        try:
+            # second color next to starting point
+            if i + 2 <= 13:
+                if state[e, i] == color and state[e, i + 2] == color:
+                    # third upper
+                    if e - 1 >= 0 and i + 1 <= 13:
+                        # if state[e - 1, i - 1] == color and (state[e, i - 1] >= 1 and state[e, i - 1] <= 5):
+                        if state[e - 1, i + 1] == color and (state[e, i + 1] in self.colors):
+                            src_pt = self.get_click_point(self.data_coordinates[e - 1, i + 1])
+                            dest_pt = self.get_click_point(self.data_coordinates[e, i + 1])
+                            self.move_tile(src_pt, dest_pt)
+                            return True
+                    # third left lower
+                    if e + 1 <= 7 and i + 1 <= 13:
+                        if state[e + 1, i + 1] == color and (state[e, i + 1] in self.colors):
+                            src_pt = self.get_click_point(self.data_coordinates[e + 1, i + 1])
+                            dest_pt = self.get_click_point(self.data_coordinates[e, i + 1])
+                            self.move_tile(src_pt, dest_pt)
+                            return True
+            if e + 2 <= 7:
+                if state[e, i] == color and state[e + 2, i] == color:
+                    # third upper
+                    if e + 1 >= 0 and i + 1 <= 13:
+                        # if state[e - 1, i - 1] == color and (state[e, i - 1] >= 1 and state[e, i - 1] <= 5):
+                        if state[e + 1, i + 1] == color and (state[e + 1, i] in self.colors):
+                            src_pt = self.get_click_point(self.data_coordinates[e + 1, i + 1])
+                            dest_pt = self.get_click_point(self.data_coordinates[e + 1, i])
+                            self.move_tile(src_pt, dest_pt)
+                            return True
+                    # third left lower
+                    if e + 1 <= 7 and i - 1 >= 0:
+                        if state[e + 1, i - 1] == color and (state[e + 1, i] in self.colors):
+                            src_pt = self.get_click_point(self.data_coordinates[e + 1, i - 1])
+                            dest_pt = self.get_click_point(self.data_coordinates[e + 1, i])
+                            self.move_tile(src_pt, dest_pt)
+                            return True
+        except:
+            return False
+
+    def check_3_horizontal(self, state, e, i, color):
+        try:
+            # second color next to starting point
+            if state[e, i] == color and state[e, i + 1] == color:
+                # third left upper
+                if e - 1 >= 0 and i - 1 >= 0:
+                    if state[e - 1, i - 1] == color and (state[e, i - 1] in self.colors):
+                        print("upper left", color, e, i)
+                        src_pt = self.get_click_point(self.data_coordinates[e - 1, i - 1])
+                        dest_pt = self.get_click_point(self.data_coordinates[e, i - 1])
+                        self.move_tile(src_pt, dest_pt)
+                        return True
+                # third left lower
+                if e + 1 <= 7 and i - 1 >= 0:
+                    if state[e + 1, i - 1] == color and (state[e, i - 1] in self.colors):
+                        print("lower left", color, e, i)
+                        src_pt = self.get_click_point(self.data_coordinates[e + 1, i - 1])
+                        dest_pt = self.get_click_point(self.data_coordinates[e, i - 1])
+                        self.move_tile(src_pt, dest_pt)
+                        return True
+                # third left with gap
+                if i - 2 >= 0:
+                    if state[e, i - 2] == color and (state[e, i - 1] in self.colors):
+                        print("left gap ", color, e, i)
+                        src_pt = self.get_click_point(self.data_coordinates[e, i - 2])
+                        dest_pt = self.get_click_point(self.data_coordinates[e, i - 1])
+                        self.move_tile(src_pt, dest_pt)
+                        return True
+                # third right upper
+                if e - 1 >= 0 and i + 2 <= 13:
+                    if state[e - 1, i + 2] == color and (state[e, i + 2] in self.colors):
+                        print("upper right", color, e, i)
+                        src_pt = self.get_click_point(self.data_coordinates[e - 1, i + 2])
+                        dest_pt = self.get_click_point(self.data_coordinates[e, i + 2])
+                        self.move_tile(src_pt, dest_pt)
+                        return True
+                # third right lower
+                if e + 1 <= 7 and i + 2 <= 13:
+                    if state[e + 1, i + 2] == color and (state[e, i + 2] in self.colors):
+                        print("upper lower", color, e, i)
+                        src_pt = self.get_click_point(self.data_coordinates[e + 1, i + 2])
+                        dest_pt = self.get_click_point(self.data_coordinates[e, i + 2])
+                        self.move_tile(src_pt, dest_pt)
+                        return True
+
+                # third right with gap
+                if i + 3 <= 13:
+                    if state[e, i + 3] == color and (state[e, i + 2] in self.colors):
+                        print("right gap ", color, e, i)
+                        src_pt = self.get_click_point(self.data_coordinates[e, i + 3])
+                        dest_pt = self.get_click_point(self.data_coordinates[e, i + 2])
+                        self.move_tile(src_pt, dest_pt)
+                        return True
+        except:
+            return False
+
+    def check_3_vertical(self, state, e, i, color):
+        try:
+            # second color next to starting point o
+            if state[e, i] == color and state[e + 1, i] == color:
+                # third left upper
+                if e - 1 >= 0 and i - 1 >= 0:
+                    if state[e - 1, i - 1] == color and (state[e - 1, i] in self.colors):
+                        print("upper left", color, e, i)
+                        src_pt = self.get_click_point(self.data_coordinates[e - 1, i - 1])
+                        dest_pt = self.get_click_point(self.data_coordinates[e - 1, i])
+                        self.move_tile(src_pt, dest_pt)
+                        return True
+                # third left lower
+                if e + 2 <= 7 and i - 1 >= 0:
+                    if state[e + 2, i - 1] == color and (state[e + 2, i] in self.colors):
+                        print("lower left", color, e, i)
+                        src_pt = self.get_click_point(self.data_coordinates[e + 2, i - 1])
+                        dest_pt = self.get_click_point(self.data_coordinates[e + 2, i])
+                        self.move_tile(src_pt, dest_pt)
+                        return True
+                # third right upper
+                if e - 1 >= 0 and i + 1 <= 13:
+                    if state[e - 1, i + 1] == color and (state[e - 1, i] in self.colors):
+                        print("upper right", color, e, i)
+                        src_pt = self.get_click_point(self.data_coordinates[e - 1, i + 1])
+                        dest_pt = self.get_click_point(self.data_coordinates[e - 1, i])
+                        self.move_tile(src_pt, dest_pt)
+                        return True
+                # third right lower
+                if e + 2 <= 7 and i + 1 <= 13:
+                    if state[e + 2, i + 1] == color and (state[e + 2, i] in self.colors):
+                        print("upper lower", color, e, i)
+                        src_pt = self.get_click_point(self.data_coordinates[e + 2, i + 1])
+                        dest_pt = self.get_click_point(self.data_coordinates[e + 2, i])
+                        self.move_tile(src_pt, dest_pt)
+                        return True
+                # third upper with gap
+                if e - 2 >= 0:
+                    if state[e - 2, i] == color and (state[e - 1, i] in self.colors):
+                        print("upper gap ", color, e, i)
+                        src_pt = self.get_click_point(self.data_coordinates[e - 2, i])
+                        dest_pt = self.get_click_point(self.data_coordinates[e - 1, i])
+                        self.move_tile(src_pt, dest_pt)
+                        return True
+                # third lower with gap
+                if e + 3 <= 7:
+                    if state[e + 3, i] == color and (state[e + 2, i] in self.colors):
+                        print("lower gap ", color, e, i)
+                        src_pt = self.get_click_point(self.data_coordinates[e + 3, i])
+                        dest_pt = self.get_click_point(self.data_coordinates[e + 2, i])
+                        self.move_tile(src_pt, dest_pt)
+                        return True
+        except:
+            return False
+
+    def move_tile(self, point_source, point_dest):
+        offset_left = 230
+        offset_down = 58
+        pydirectinput.moveTo(point_source[0] + offset_left, point_source[1] + offset_down)
+        # pydirectinput.moveTo(0,0)
+        pydirectinput.mouseDown()
+        w = random.randint(25, 50)
+        cv.waitKey(100 + w)
+        pydirectinput.moveTo(point_dest[0] + offset_left, point_dest[1] + offset_down)
+        pydirectinput.mouseUp()
+        cv.waitKey(400 + w)
+
+    def check_for_button_and_click_it(self, button_url):
+        screenshot = self.capture_window.get_screenshot()
+        # gray = cv.cvtColor(screenshot, cv.COLOR_BGR2GRAY)
+        # thresh = cv.threshold(gray, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1]
+        # gray_needle = cv.cvtColor(cv.imread(button_url, cv.IMREAD_UNCHANGED), cv.COLOR_BGR2GRAY)
+        # thresh_needle = cv.threshold(gray_needle, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1]
+        needle = cv.imread(button_url, cv.IMREAD_UNCHANGED)
+        # rectangles = self.vision_stun.find(thresh, thresh_needle, 0.4, 1)
+        rectangles = self.vision_stun.find(screenshot, needle, 0.7, 1)
+
+        if len(rectangles) == 1:
+            pointis = self.vision_stun.get_click_points(rectangles)
+            for pointi in pointis:
+                self.dig_point(pointi[0], pointi[1], 150)
+
+    def dig_point(self, point1, point2, dig_time):
+        pydirectinput.moveTo(point1, point2)
+        cv.waitKey(dig_time)
+        pydirectinput.mouseDown()
+        w = random.randint(50, 100)
+        cv.waitKey(w)
+        pydirectinput.mouseUp()
+
+    def get_click_point(self, rectangle):
+        # Loop over all the rectangles
+        x, y, w, h = rectangle
+        # Determine the center position
+        center_x = x + int(w / 2)
+        center_y = y + int(h / 2)
+        # Save the points
+        return int(center_x), int(center_y)

crop/crop_overlay.py

@@ -0,0 +1,66 @@
+# Run tkinter code in another thread
+import threading
+import tkinter as tk
+from tkinter import ttk
+
+
+class CropOverlay(threading.Thread):
+
+    def __init__(self):
+        threading.Thread.__init__(self)
+
+        self.run_mode = 'init'
+
+        self.root = tk.Tk
+
+        self.ButtonFrame = tk.Frame
+
+        self.StartButton = tk.Button
+        self.StopButton = tk.Button
+        self.TkPosition = '133x52+60+600'
+
+        self.StatusLabel = tk.Label
+
+        self.start()
+
+    def run(self):
+        self.root = tk.Tk()
+        self.ButtonFrame = tk.Frame(self.root)
+        self.StartButton = tk.Button(self.ButtonFrame, text="Start", command=self.start_button_callback, width='8')
+        self.StartButton.grid(row=0, column=0)
+        self.StopButton = tk.Button(self.ButtonFrame, text="Stop", command=self.stop_button_callback, width='8',
+                                    state=tk.DISABLED)
+        self.StopButton.grid(row=0, column=1)
+
+        self.ButtonFrame.grid(row=3, column=0, columnspan=2)
+
+        self.StatusLabel = tk.Label(self.root, text="", font=("Helvetica", 10, "bold"),
+                                    background="grey")
+        self.StatusLabel.grid(row=4, column=0, columnspan=2)
+
+        # self.ClearButton.pack(side="top")
+        self.root.geometry(self.TkPosition)
+        self.root.overrideredirect(1)  # fenster ohne aussen rum :-)
+        # self.root.attributes('-alpha', 0.7)  # fenster transparent
+        self.root.attributes('-topmost', 1)  # fenster immer im vordergrund
+        # self.root.wm_attributes("-disabled", True)
+        self.root.configure(background='black')
+        self.root.mainloop()
+
+    def start_button_callback(self):
+        self.StartButton.configure(state=tk.DISABLED)
+        self.StopButton.configure(state=tk.NORMAL)
+        self.StatusLabel.configure(text='')
+        self.run_mode = 'started'
+
+    def stop_button_callback(self):
+        self.StartButton.configure(state=tk.NORMAL)
+        self.StopButton.configure(state=tk.DISABLED)
+        self.run_mode = 'stopped'
+
+    def get_run_mode(self):
+        return self.run_mode
+
+    def update_status_label(self, mode, energy_to_go):
+        text = mode + ": " + str(energy_to_go)
+        self.StatusLabel.configure(text=text)

crop/main_crop_ui.py

@@ -0,0 +1,35 @@
+from Field_Representation_crop import Field
+from time import time
+import cv2 as cv
+import keyboard
+from crop_overlay import CropOverlay
+
+
+def run():
+    field = Field()
+    overlay = CropOverlay()
+
+    while True:
+        if overlay.run_mode == 'stopped':
+            overlay.update_status_label("stopped", "0")
+            overlay.run_mode = 'init'
+            continue
+        elif overlay.run_mode == 'started':
+            cv.waitKey(500)
+        elif overlay.run_mode == 'init':
+            continue
+        else:
+            # boelk
+            pass
+
+        while True:
+            if overlay.run_mode == 'stopped':
+                break
+
+            cords = field.assess_playfield_and_make_move()
+            print(cords)
+            cv.waitKey(1000)
+
+
+if __name__ == "__main__":
+    run()