added farm bot

fixed magic needles

farm/Field_Representation.py

@@ -0,0 +1,472 @@
+import random
+import cv2 as cv
+import numpy as np
+import pydirectinput
+from window_capture import WindowCapture
+from vision import Vision
+from config_file import UserConfigs
+from utils import mse
+
+GREEN = 1
+YELLOW = 2
+BLUE = 3
+RED = 4
+PINK = 5
+RAINBOW = 6
+BIGBOMB = 7
+BOMB = 8
+ARROW_DOWN = 9
+ARROW_RIGHT = 10
+ROCK_1 = 11
+ROCK_2 = 12
+ROCK_3 = 13
+
+class Field:
+    data_value_grid = []
+    data_coordinates = []
+    next_level = cv.imread("next_level.jpg", cv.IMREAD_COLOR)
+    next_level_x = cv.imread("next_level_x.jpg", cv.IMREAD_COLOR)
+    explosives = []
+    explosives.append(RAINBOW)
+    explosives.append(ARROW_RIGHT)
+    explosives.append(ARROW_DOWN)
+    explosives.append(BIGBOMB)
+    explosives.append(BOMB)
+
+    def __init__(self):
+        self.data_value_grid = np.zeros((8, 14), dtype=int)
+        self.data_coordinates = np.zeros((8, 14), dtype=object)
+
+        # 230 to 2110 = 1883 / 14 = 134.5
+        # 60 to 1130 = 1076 / 8 =  134.5
+        dim = 134.5
+        for e in range(0, 8, 1):
+            for i in range(0, 14, 1):
+                self.data_coordinates[e][i] = [i * dim, e * dim, dim, dim]
+
+        # initialize the user-class
+        self.config = UserConfigs()
+
+        # initialize the StunWindowCapture class
+        self.capture_window = WindowCapture(None, None, self.config)
+        # initialize the StunVision class
+        self.vision_stun = Vision()
+
+        self.needles = {GREEN: cv.imread("green.jpg", cv.IMREAD_COLOR),
+                        YELLOW: cv.imread("yellow.jpg", cv.IMREAD_COLOR),
+                        BLUE: cv.imread("blue.jpg", cv.IMREAD_COLOR),
+                        RED: cv.imread("red.jpg", cv.IMREAD_COLOR),
+                        PINK: cv.imread("pink.jpg", cv.IMREAD_COLOR),
+                        RAINBOW: cv.imread("rainbow.jpg", cv.IMREAD_COLOR),
+                        BIGBOMB: cv.imread("bigbomb.jpg", cv.IMREAD_COLOR),
+                        BOMB: cv.imread("bomb.jpg", cv.IMREAD_COLOR),
+                        ARROW_DOWN: cv.imread("arrow_down.jpg", cv.IMREAD_COLOR),
+                        ARROW_RIGHT: cv.imread("arrow_right.jpg", cv.IMREAD_COLOR),
+                        ROCK_1: cv.imread("rock1.jpg", cv.IMREAD_COLOR),
+                        ROCK_2: cv.imread("rock2.jpg", cv.IMREAD_COLOR),
+                        ROCK_3: cv.imread("rock3.jpg", cv.IMREAD_COLOR)
+                        }
+
+    def reset(self):
+        self.episode_step = 0
+        self.last_reward = 0
+        self.last_score = 0
+        self.kill_counter = 0
+        # hit reset button and confirm
+        # self.check_for_button_and_click_it("needles/repeat.jpg")
+        # self.check_for_button_and_click_it("needles/reset.jpg")
+
+        self.dig_point(1800, 160, 600)
+        self.dig_point(1800, 1000, 300)
+        self.observation, screen = self.get_current_board_state()
+        return self.observation
+
+    def shift_playfield(self, action):
+        self.episode_step += 1
+        # move to indicated direction
+        self.action(action)
+        # get new field status
+        new_observation, new_screenshot = self.get_current_board_state()
+        current_score = 0
+        reward = 0
+
+        # wrong movement detection
+        # last board state is same as actual
+        if mse(new_observation, self.observation) == 0.0:
+            # no movement detected -> punish
+            if len(new_observation[new_observation == 0]) >= 1:
+                reward = -100
+            else:
+                self.kill_counter = self.kill_counter + 1
+                reward = -5
+        else:
+            # calculate current board score
+            self.kill_counter = 0
+            for e in range(0, 4, 1):
+                for i in range(0, 4, 1):
+                    current_score = current_score + (2 ** new_observation[e][i] - 1)
+            bonus_for_empty_cells = len(new_observation[new_observation == 0])
+            reward = current_score - self.last_score + bonus_for_empty_cells
+            self.last_score = current_score
+
+        if self.kill_counter >= 5:
+            self.kill_counter = 0
+            done = True
+        else:
+            done = False
+        self.observation = new_observation
+        return new_observation, reward, done
+
+    def action(self, choice):
+        '''
+        Gives us 4 total movement options. (0,1,2,3)
+        '''
+        if choice == 0:
+            # right
+            self.move_to(1200, 598)
+        elif choice == 1:
+            # left
+            self.move_to(1000, 598)
+        elif choice == 2:
+            # up
+            self.move_to(1113, 498)
+        elif choice == 3:
+            # down
+            self.move_to(1113, 698)
+
+    def move_to(self, x, y):
+        point_src = (1113, 598)
+        pydirectinput.moveTo(point_src[0], point_src[1])
+        pydirectinput.mouseDown()
+        w = random.randint(1, 100)
+        cv.waitKey(150 + w)
+        pydirectinput.moveTo(x, y)
+        pydirectinput.mouseUp()
+        cv.waitKey(500 + w)
+
+    def change_value(self, x, y, val):
+        self.data_value_grid[x][y] = val
+
+    def pointInRect(self, point):
+        for e in range(0, 8, 1):
+            for i in range(0, 14, 1):
+                x1, y1, w, h = self.data_coordinates[e][i]
+                x2, y2 = x1 + w, y1 + h
+                x, y = point
+                if (x1 < x and x < x2):
+                    if (y1 < y and y < y2):
+                        return e, i
+        return None, None
+
+    def check_for_next_level(self, screen, needle):
+        offset_left = 230
+        offset_down = 58
+        rectangles = self.vision_stun.find(screen, needle, 0.70, 1)
+        if len(rectangles) == 0:
+            return False
+        point = self.vision_stun.get_click_points(rectangles)[0]
+        self.dig_point(point[0]+ offset_left, point[1] + offset_down, 500)
+        return True
+
+    def get_current_board_state(self):
+        try:
+            # get an updated image of the game
+            #screenshot = self.capture_window.get_screenshot()
+            screenshot = cv.imread("field_farm.jpg")
+            screenshot = screenshot[58:1134, 230:2113]  # 1883,1076
+            # gray = cv.cvtColor(screenshot, cv.COLOR_BGR2GRAY)
+            # thresh = cv.threshold(gray, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1]
+            if self.check_for_next_level(screenshot, self.next_level):
+                cv.waitKey(500)
+                screenshot = self.capture_window.get_screenshot()
+                screenshot = screenshot[58:1134, 230:2113]
+            if self.check_for_next_level(screenshot, self.next_level_x):
+                cv.waitKey(500)
+                screenshot = self.capture_window.get_screenshot()
+                screenshot = screenshot[58:1134, 230:2113]
+
+        except:
+            self.capture_window.release()
+            print("Game window not available - shutting down application")
+            return None
+        # cv.imshow("screenshot", screenshot)
+        # cv.waitKey(150)
+        # continue
+        data_coords = np.zeros((8, 14), dtype=object)
+        # field = Field()
+        for needle_key in self.needles.keys():
+            # gray_needle = cv.cvtColor(self.needles[needle_key], cv.COLOR_BGR2GRAY)
+            # thresh_needle = cv.threshold(gray_needle, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1]
+            rectangles = self.vision_stun.find(screenshot, self.needles[needle_key], 0.70, 56)
+            if len(rectangles) == 0:
+                continue
+            points = self.vision_stun.get_click_points(rectangles)
+
+            for point in points:
+                x, y = self.pointInRect(point)
+                if x is not None and y is not None:
+                    data_coords[x][y] = int(needle_key)
+                    # self.change_value(x, y, int(needle_key))
+                    # print(field.data_value_grid)
+                    # cv.circle(screenshot, points[0], 7, (0, 255, 0), -1)
+                    # output_image = vision_stun.draw_rectangles(screenshot, rectangles)
+                    # cv.imshow("output_image", output_image)
+                    # cv.waitKey(150)
+        return data_coords, screenshot
+
+    def analyse_boardstate(self, state):
+        for e in range(0, 8, 1):
+            for i in range(0, 14, 1):
+                if self.check_explosives(state, e, i):
+                    return
+
+        for e in range(0, 8, 1):
+            for i in range(0, 14, 1):
+                for color in range(1, 6, 1):
+                    if self.check_5_horizontal(state, e, i, color):
+                        return
+                    if self.check_5_vertical(state, e, i, color):
+                        return
+
+        for e in range(0, 8, 1):
+            for i in range(0, 14, 1):
+                for color in range(1, 6, 1):
+                    if self.check_3_horizontal(state, e, i, color):
+                        return
+                    if self.check_3_vertical(state, e, i, color):
+                        return
+
+    def check_explosives(self, state, e, i):
+        for explosive in self.explosives:
+            if self.local_pos_check(state, e, i, 0, 0, explosive):
+                dest_pt = self.get_click_point(self.data_coordinates[e, i])
+                if self.local_pos_checks(state, e, i, 1, 0, self.explosives):
+                    src_pt = self.get_click_point(self.data_coordinates[e + 1, i])
+                    self.move_tile(src_pt, dest_pt)
+                elif self.local_pos_checks(state, e, i, 0, 1, self.explosives):
+                    src_pt = self.get_click_point(self.data_coordinates[e, i + 1])
+                    self.move_tile(src_pt, dest_pt)
+                elif self.local_pos_checks(state, e, i, -1, 0, self.explosives):
+                    src_pt = self.get_click_point(self.data_coordinates[e - 1, i])
+                    self.move_tile(src_pt, dest_pt)
+                elif self.local_pos_checks(state, e, i, 0, -1, self.explosives):
+                    src_pt = self.get_click_point(self.data_coordinates[e, i - 1])
+                    self.move_tile(src_pt, dest_pt)
+                else:
+                    continue
+                return True
+            else:
+                continue
+        return False
+
+    def local_pos_check(self, state, e, i, e_check, i_check, needle):
+        if e + e_check >= 0 and e + e_check <=7 and i + i_check >= 0 and i + i_check <=13:
+            if state[e + e_check, i + i_check] == needle:
+                return True
+            else:
+                return False
+
+    def local_pos_checks(self, state, e, i, e_check, i_check, needles):
+        if e + e_check >= 0 and e + e_check <=7 and i + i_check >= 0 and i + i_check <=13:
+            for needle in needles:
+                if state[e + e_check, i + i_check] == needle:
+                    return True
+                else:
+                    continue
+        return False
+
+    def check_5_horizontal(self, state, e, i, color):
+        try:
+            #
+            if state[e, i] == color and state[e, i + 1] == color and state[e, i + 3] == color and state[
+                e, i + 4] == color:
+                if state[e, i + 2] <= 0 or state[e, i + 2] >= 6:
+                    return False
+                #
+                if e - 1 >= 0 and i + 2 <= 13:
+                    if state[e - 1, i + 2] == color:
+                        return True
+            return False
+        except:
+            return False
+
+    def check_5_vertical(self, state, e, i, color):
+        try:
+            #
+            if state[e, i] == color and state[e + 1, i] == color and state[e + 3, i] == color and state[
+                e + 4, i] == color:
+                if state[e + 2, i] <= 0 or state[e + 2, i] >= 6:
+                    return False
+                # third left upper
+                if e + 2 <= 7 and i - 1 >= 0:
+                    if state[e + 2, i - 1] == color:
+                        print("upper left", color, e, i)
+                        src_pt = self.get_click_point(self.data_coordinates[e + 2, i - 1])
+                        dest_pt = self.get_click_point(self.data_coordinates[e + 2, i])
+                        self.move_tile(src_pt, dest_pt)
+                        return True
+                if e + 2 <= 7 and i + 1 <= 13:
+                    if state[e + 2, i + 1] == color:
+                        print("upper left", color, e, i)
+                        src_pt = self.get_click_point(self.data_coordinates[e + 2, i + 1])
+                        dest_pt = self.get_click_point(self.data_coordinates[e + 2, i])
+                        self.move_tile(src_pt, dest_pt)
+                        return True
+            return False
+        except:
+            return False
+
+    def check_3_horizontal(self, state, e, i, color):
+        try:
+            # second color next to starting point
+            if state[e, i] == color and state[e, i + 1] == color:
+                # third left upper
+                if e - 1 >= 0 and i - 1 >= 0:
+                    if state[e - 1, i - 1] == color and (state[e, i - 1] >= 1 and state[e, i - 1] <= 5):
+                        print("upper left", color, e, i)
+                        src_pt = self.get_click_point(self.data_coordinates[e - 1, i - 1])
+                        dest_pt = self.get_click_point(self.data_coordinates[e, i - 1])
+                        self.move_tile(src_pt, dest_pt)
+                        return True
+                # third left lower
+                if e + 1 <= 7 and i - 1 >= 0:
+                    if state[e + 1, i - 1] == color and (state[e, i - 1] >= 1 and state[e, i - 1] <= 5):
+                        print("lower left", color, e, i)
+                        src_pt = self.get_click_point(self.data_coordinates[e + 1, i - 1])
+                        dest_pt = self.get_click_point(self.data_coordinates[e, i - 1])
+                        self.move_tile(src_pt, dest_pt)
+                        return True
+                # third left with gap
+                if i - 2 >= 0:
+                    if state[e, i - 2] == color and (state[e, i - 1] >= 1 and state[e, i - 1] <= 5):
+                        print("left gap ", color, e, i)
+                        src_pt = self.get_click_point(self.data_coordinates[e, i - 2])
+                        dest_pt = self.get_click_point(self.data_coordinates[e, i - 1])
+                        self.move_tile(src_pt, dest_pt)
+                        return True
+                # third right upper
+                if e - 1 >= 0 and i + 2 <= 13:
+                    if state[e - 1, i + 2] == color and (state[e, i + 2] >= 1 and state[e, i + 2] <= 5):
+                        print("upper right", color, e, i)
+                        src_pt = self.get_click_point(self.data_coordinates[e - 1, i + 2])
+                        dest_pt = self.get_click_point(self.data_coordinates[e, i + 2])
+                        self.move_tile(src_pt, dest_pt)
+                        return True
+                # third right lower
+                if e + 1 <= 7 and i + 2 <= 13:
+                    if state[e + 1, i + 2] == color and (state[e, i + 2] >= 1 and state[e, i + 2] <= 5):
+                        print("upper lower", color, e, i)
+                        src_pt = self.get_click_point(self.data_coordinates[e + 1, i + 2])
+                        dest_pt = self.get_click_point(self.data_coordinates[e, i + 2])
+                        self.move_tile(src_pt, dest_pt)
+                        return True
+
+                # third right with gap
+                if i + 3 <= 13:
+                    if state[e, i + 3] == color and (state[e, i + 2] >= 1 and state[e, i + 2] <= 5):
+                        print("right gap ", color, e, i)
+                        src_pt = self.get_click_point(self.data_coordinates[e, i + 3])
+                        dest_pt = self.get_click_point(self.data_coordinates[e, i + 2])
+                        self.move_tile(src_pt, dest_pt)
+                        return True
+        except:
+            return False
+
+    def check_3_vertical(self, state, e, i, color):
+        try:
+            # second color next to starting point o
+            if state[e, i] == color and state[e + 1, i] == color:
+                # third left upper
+                if e - 1 >= 0 and i - 1 >= 0:
+                    if state[e - 1, i - 1] == color and (state[e - 1, i] >= 1 and state[e - 1, i] <= 5):
+                        print("upper left", color, e, i)
+                        src_pt = self.get_click_point(self.data_coordinates[e - 1, i - 1])
+                        dest_pt = self.get_click_point(self.data_coordinates[e - 1, i])
+                        self.move_tile(src_pt, dest_pt)
+                        return True
+                # third left lower
+                if e + 2 <= 7 and i - 1 >= 0:
+                    if state[e + 2, i - 1] == color and (state[e + 2, i] >= 1 and state[e + 2, i] <= 5):
+                        print("lower left", color, e, i)
+                        src_pt = self.get_click_point(self.data_coordinates[e + 2, i - 1])
+                        dest_pt = self.get_click_point(self.data_coordinates[e + 2, i])
+                        self.move_tile(src_pt, dest_pt)
+                        return True
+                # third right upper
+                if e - 1 >= 0 and i + 1 <= 13:
+                    if state[e - 1, i + 1] == color and (state[e - 1, i] >= 1 and state[e - 1, i] <= 5):
+                        print("upper right", color, e, i)
+                        src_pt = self.get_click_point(self.data_coordinates[e - 1, i + 1])
+                        dest_pt = self.get_click_point(self.data_coordinates[e - 1, i])
+                        self.move_tile(src_pt, dest_pt)
+                        return True
+                # third right lower
+                if e + 2 <= 7 and i + 1 <= 13:
+                    if state[e + 2, i + 1] == color and (state[e + 2, i] >= 1 and state[e + 2, i] <= 5):
+                        print("upper lower", color, e, i)
+                        src_pt = self.get_click_point(self.data_coordinates[e + 2, i + 1])
+                        dest_pt = self.get_click_point(self.data_coordinates[e + 2, i])
+                        self.move_tile(src_pt, dest_pt)
+                        return True
+                # third upper with gap
+                if e - 2 >= 0:
+                    if state[e - 2, i] == color and (state[e - 1, i] >= 1 and state[e - 1, i] <= 5):
+                        print("upper gap ", color, e, i)
+                        src_pt = self.get_click_point(self.data_coordinates[e - 2, i])
+                        dest_pt = self.get_click_point(self.data_coordinates[e - 1, i])
+                        self.move_tile(src_pt, dest_pt)
+                        return True
+                # third lower with gap
+                if e + 3 <= 7:
+                    if state[e + 3, i] == color and (state[e + 2, i] >= 1 and state[e + 2, i] <= 5):
+                        print("lower gap ", color, e, i)
+                        src_pt = self.get_click_point(self.data_coordinates[e + 3, i])
+                        dest_pt = self.get_click_point(self.data_coordinates[e + 2, i])
+                        self.move_tile(src_pt, dest_pt)
+                        return True
+        except:
+            return False
+
+    def move_tile(self, point_source, point_dest):
+        offset_left = 230
+        offset_down = 58
+        pydirectinput.moveTo(point_source[0] + offset_left, point_source[1] + offset_down)
+        # pydirectinput.moveTo(0,0)
+        pydirectinput.mouseDown()
+        w = random.randint(25, 50)
+        cv.waitKey(100 + w)
+        pydirectinput.moveTo(point_dest[0] + offset_left, point_dest[1] + offset_down)
+        pydirectinput.mouseUp()
+        cv.waitKey(400 + w)
+
+    def check_for_button_and_click_it(self, button_url):
+        screenshot = self.capture_window.get_screenshot()
+        # gray = cv.cvtColor(screenshot, cv.COLOR_BGR2GRAY)
+        # thresh = cv.threshold(gray, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1]
+        # gray_needle = cv.cvtColor(cv.imread(button_url, cv.IMREAD_UNCHANGED), cv.COLOR_BGR2GRAY)
+        # thresh_needle = cv.threshold(gray_needle, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1]
+        needle = cv.imread(button_url, cv.IMREAD_UNCHANGED)
+        # rectangles = self.vision_stun.find(thresh, thresh_needle, 0.4, 1)
+        rectangles = self.vision_stun.find(screenshot, needle, 0.7, 1)
+
+        if len(rectangles) == 1:
+            pointis = self.vision_stun.get_click_points(rectangles)
+            for pointi in pointis:
+                self.dig_point(pointi[0], pointi[1], 150)
+
+    def dig_point(self, point1, point2, dig_time):
+        pydirectinput.moveTo(point1, point2)
+        cv.waitKey(dig_time)
+        pydirectinput.mouseDown()
+        w = random.randint(50, 100)
+        cv.waitKey(w)
+        pydirectinput.mouseUp()
+
+    def get_click_point(self, rectangle):
+        # Loop over all the rectangles
+        x, y, w, h = rectangle
+        # Determine the center position
+        center_x = x + int(w / 2)
+        center_y = y + int(h / 2)
+        # Save the points
+        return int(center_x), int(center_y)

farm/main_farm.py

@@ -0,0 +1,39 @@
+from Field_Representation import Field
+from time import time
+import cv2 as cv
+import keyboard
+
+def run():
+    field = Field()
+    loop_time_p = time()
+    pause = True
+
+    while True:
+        if keyboard.is_pressed('p') == True:
+            pause = True
+            print('q pressed')
+        elif keyboard.is_pressed('o') == True:
+            pause = False
+            print('o pressed')
+
+        if pause:
+            # cv.waitKey(500)
+            if (time() - loop_time_p) >= 5:
+                loop_time_p = time()
+                print("pausing")
+            continue
+
+
+        cords, screenshot = field.get_current_board_state()
+        field.analyse_boardstate(cords)
+        print(cords)
+        cv.waitKey(1000)
+
+
+    #cv.imshow("screenshot", screenshot)
+    #cv.waitKey(150)
+    #print(cords)
+
+
+if __name__ == "__main__":
+    run()