import random import cv2 as cv import numpy as np import pydirectinput from window_capture import WindowCapture from vision import Vision from config_file import UserConfigs from utils import mse GREEN = 1 YELLOW = 2 BLUE = 3 RED = 4 PINK = 5 RAINBOW = 6 BIGBOMB = 7 BOMB = 8 ARROW_DOWN = 9 ARROW_RIGHT = 10 ROCK_1 = 11 ROCK_2 = 12 ROCK_3 = 13 class Field: data_value_grid = [] data_coordinates = [] next_level = cv.imread("next_level.jpg", cv.IMREAD_COLOR) next_level_x = cv.imread("next_level_x.jpg", cv.IMREAD_COLOR) explosives = [] explosives.append(RAINBOW) explosives.append(ARROW_RIGHT) explosives.append(ARROW_DOWN) explosives.append(BIGBOMB) explosives.append(BOMB) def __init__(self): self.data_value_grid = np.zeros((8, 14), dtype=int) self.data_coordinates = np.zeros((8, 14), dtype=object) # 230 to 2110 = 1883 / 14 = 134.5 # 60 to 1130 = 1076 / 8 = 134.5 dim = 134.5 for e in range(0, 8, 1): for i in range(0, 14, 1): self.data_coordinates[e][i] = [i * dim, e * dim, dim, dim] # initialize the user-class self.config = UserConfigs() # initialize the StunWindowCapture class self.capture_window = WindowCapture(None, None, self.config) # initialize the StunVision class self.vision_stun = Vision() self.needles = {GREEN: cv.imread("green.jpg", cv.IMREAD_COLOR), YELLOW: cv.imread("yellow.jpg", cv.IMREAD_COLOR), BLUE: cv.imread("blue.jpg", cv.IMREAD_COLOR), RED: cv.imread("red.jpg", cv.IMREAD_COLOR), PINK: cv.imread("pink.jpg", cv.IMREAD_COLOR), RAINBOW: cv.imread("rainbow.jpg", cv.IMREAD_COLOR), BIGBOMB: cv.imread("bigbomb.jpg", cv.IMREAD_COLOR), BOMB: cv.imread("bomb.jpg", cv.IMREAD_COLOR), ARROW_DOWN: cv.imread("arrow_down.jpg", cv.IMREAD_COLOR), ARROW_RIGHT: cv.imread("arrow_right.jpg", cv.IMREAD_COLOR), ROCK_1: cv.imread("rock1.jpg", cv.IMREAD_COLOR), ROCK_2: cv.imread("rock2.jpg", cv.IMREAD_COLOR), ROCK_3: cv.imread("rock3.jpg", cv.IMREAD_COLOR) } def reset(self): self.episode_step = 0 self.last_reward = 0 self.last_score = 0 self.kill_counter = 0 # hit reset button and confirm # self.check_for_button_and_click_it("needles/repeat.jpg") # self.check_for_button_and_click_it("needles/reset.jpg") self.dig_point(1800, 160, 600) self.dig_point(1800, 1000, 300) self.observation, screen = self.get_current_board_state() return self.observation def shift_playfield(self, action): self.episode_step += 1 # move to indicated direction self.action(action) # get new field status new_observation, new_screenshot = self.get_current_board_state() current_score = 0 reward = 0 # wrong movement detection # last board state is same as actual if mse(new_observation, self.observation) == 0.0: # no movement detected -> punish if len(new_observation[new_observation == 0]) >= 1: reward = -100 else: self.kill_counter = self.kill_counter + 1 reward = -5 else: # calculate current board score self.kill_counter = 0 for e in range(0, 4, 1): for i in range(0, 4, 1): current_score = current_score + (2 ** new_observation[e][i] - 1) bonus_for_empty_cells = len(new_observation[new_observation == 0]) reward = current_score - self.last_score + bonus_for_empty_cells self.last_score = current_score if self.kill_counter >= 5: self.kill_counter = 0 done = True else: done = False self.observation = new_observation return new_observation, reward, done def action(self, choice): ''' Gives us 4 total movement options. (0,1,2,3) ''' if choice == 0: # right self.move_to(1200, 598) elif choice == 1: # left self.move_to(1000, 598) elif choice == 2: # up self.move_to(1113, 498) elif choice == 3: # down self.move_to(1113, 698) def move_to(self, x, y): point_src = (1113, 598) pydirectinput.moveTo(point_src[0], point_src[1]) pydirectinput.mouseDown() w = random.randint(1, 100) cv.waitKey(150 + w) pydirectinput.moveTo(x, y) pydirectinput.mouseUp() cv.waitKey(500 + w) def change_value(self, x, y, val): self.data_value_grid[x][y] = val def pointInRect(self, point): for e in range(0, 8, 1): for i in range(0, 14, 1): x1, y1, w, h = self.data_coordinates[e][i] x2, y2 = x1 + w, y1 + h x, y = point if (x1 < x and x < x2): if (y1 < y and y < y2): return e, i return None, None def check_for_next_level(self, screen, needle): offset_left = 230 offset_down = 58 rectangles = self.vision_stun.find(screen, needle, 0.70, 1) if len(rectangles) == 0: return False point = self.vision_stun.get_click_points(rectangles)[0] self.dig_point(point[0]+ offset_left, point[1] + offset_down, 500) return True def get_current_board_state(self): try: # get an updated image of the game #screenshot = self.capture_window.get_screenshot() screenshot = cv.imread("field_farm.jpg") screenshot = screenshot[58:1134, 230:2113] # 1883,1076 # gray = cv.cvtColor(screenshot, cv.COLOR_BGR2GRAY) # thresh = cv.threshold(gray, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1] if self.check_for_next_level(screenshot, self.next_level): cv.waitKey(500) screenshot = self.capture_window.get_screenshot() screenshot = screenshot[58:1134, 230:2113] if self.check_for_next_level(screenshot, self.next_level_x): cv.waitKey(500) screenshot = self.capture_window.get_screenshot() screenshot = screenshot[58:1134, 230:2113] except: self.capture_window.release() print("Game window not available - shutting down application") return None # cv.imshow("screenshot", screenshot) # cv.waitKey(150) # continue data_coords = np.zeros((8, 14), dtype=object) # field = Field() for needle_key in self.needles.keys(): # gray_needle = cv.cvtColor(self.needles[needle_key], cv.COLOR_BGR2GRAY) # thresh_needle = cv.threshold(gray_needle, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1] rectangles = self.vision_stun.find(screenshot, self.needles[needle_key], 0.70, 56) if len(rectangles) == 0: continue points = self.vision_stun.get_click_points(rectangles) for point in points: x, y = self.pointInRect(point) if x is not None and y is not None: data_coords[x][y] = int(needle_key) # self.change_value(x, y, int(needle_key)) # print(field.data_value_grid) # cv.circle(screenshot, points[0], 7, (0, 255, 0), -1) # output_image = vision_stun.draw_rectangles(screenshot, rectangles) # cv.imshow("output_image", output_image) # cv.waitKey(150) return data_coords, screenshot def analyse_boardstate(self, state): for e in range(0, 8, 1): for i in range(0, 14, 1): if self.check_explosives(state, e, i): return for e in range(0, 8, 1): for i in range(0, 14, 1): for color in range(1, 6, 1): if self.check_5_horizontal(state, e, i, color): return if self.check_5_vertical(state, e, i, color): return for e in range(0, 8, 1): for i in range(0, 14, 1): for color in range(1, 6, 1): if self.check_3_horizontal(state, e, i, color): return if self.check_3_vertical(state, e, i, color): return def check_explosives(self, state, e, i): for explosive in self.explosives: if self.local_pos_check(state, e, i, 0, 0, explosive): dest_pt = self.get_click_point(self.data_coordinates[e, i]) if self.local_pos_checks(state, e, i, 1, 0, self.explosives): src_pt = self.get_click_point(self.data_coordinates[e + 1, i]) self.move_tile(src_pt, dest_pt) elif self.local_pos_checks(state, e, i, 0, 1, self.explosives): src_pt = self.get_click_point(self.data_coordinates[e, i + 1]) self.move_tile(src_pt, dest_pt) elif self.local_pos_checks(state, e, i, -1, 0, self.explosives): src_pt = self.get_click_point(self.data_coordinates[e - 1, i]) self.move_tile(src_pt, dest_pt) elif self.local_pos_checks(state, e, i, 0, -1, self.explosives): src_pt = self.get_click_point(self.data_coordinates[e, i - 1]) self.move_tile(src_pt, dest_pt) else: continue return True else: continue return False def local_pos_check(self, state, e, i, e_check, i_check, needle): if e + e_check >= 0 and e + e_check <=7 and i + i_check >= 0 and i + i_check <=13: if state[e + e_check, i + i_check] == needle: return True else: return False def local_pos_checks(self, state, e, i, e_check, i_check, needles): if e + e_check >= 0 and e + e_check <=7 and i + i_check >= 0 and i + i_check <=13: for needle in needles: if state[e + e_check, i + i_check] == needle: return True else: continue return False def check_5_horizontal(self, state, e, i, color): try: # if state[e, i] == color and state[e, i + 1] == color and state[e, i + 3] == color and state[ e, i + 4] == color: if state[e, i + 2] <= 0 or state[e, i + 2] >= 6: return False # if e - 1 >= 0 and i + 2 <= 13: if state[e - 1, i + 2] == color: return True return False except: return False def check_5_vertical(self, state, e, i, color): try: # if state[e, i] == color and state[e + 1, i] == color and state[e + 3, i] == color and state[ e + 4, i] == color: if state[e + 2, i] <= 0 or state[e + 2, i] >= 6: return False # third left upper if e + 2 <= 7 and i - 1 >= 0: if state[e + 2, i - 1] == color: print("upper left", color, e, i) src_pt = self.get_click_point(self.data_coordinates[e + 2, i - 1]) dest_pt = self.get_click_point(self.data_coordinates[e + 2, i]) self.move_tile(src_pt, dest_pt) return True if e + 2 <= 7 and i + 1 <= 13: if state[e + 2, i + 1] == color: print("upper left", color, e, i) src_pt = self.get_click_point(self.data_coordinates[e + 2, i + 1]) dest_pt = self.get_click_point(self.data_coordinates[e + 2, i]) self.move_tile(src_pt, dest_pt) return True return False except: return False def check_3_horizontal(self, state, e, i, color): try: # second color next to starting point if state[e, i] == color and state[e, i + 1] == color: # third left upper if e - 1 >= 0 and i - 1 >= 0: if state[e - 1, i - 1] == color and (state[e, i - 1] >= 1 and state[e, i - 1] <= 5): print("upper left", color, e, i) src_pt = self.get_click_point(self.data_coordinates[e - 1, i - 1]) dest_pt = self.get_click_point(self.data_coordinates[e, i - 1]) self.move_tile(src_pt, dest_pt) return True # third left lower if e + 1 <= 7 and i - 1 >= 0: if state[e + 1, i - 1] == color and (state[e, i - 1] >= 1 and state[e, i - 1] <= 5): print("lower left", color, e, i) src_pt = self.get_click_point(self.data_coordinates[e + 1, i - 1]) dest_pt = self.get_click_point(self.data_coordinates[e, i - 1]) self.move_tile(src_pt, dest_pt) return True # third left with gap if i - 2 >= 0: if state[e, i - 2] == color and (state[e, i - 1] >= 1 and state[e, i - 1] <= 5): print("left gap ", color, e, i) src_pt = self.get_click_point(self.data_coordinates[e, i - 2]) dest_pt = self.get_click_point(self.data_coordinates[e, i - 1]) self.move_tile(src_pt, dest_pt) return True # third right upper if e - 1 >= 0 and i + 2 <= 13: if state[e - 1, i + 2] == color and (state[e, i + 2] >= 1 and state[e, i + 2] <= 5): print("upper right", color, e, i) src_pt = self.get_click_point(self.data_coordinates[e - 1, i + 2]) dest_pt = self.get_click_point(self.data_coordinates[e, i + 2]) self.move_tile(src_pt, dest_pt) return True # third right lower if e + 1 <= 7 and i + 2 <= 13: if state[e + 1, i + 2] == color and (state[e, i + 2] >= 1 and state[e, i + 2] <= 5): print("upper lower", color, e, i) src_pt = self.get_click_point(self.data_coordinates[e + 1, i + 2]) dest_pt = self.get_click_point(self.data_coordinates[e, i + 2]) self.move_tile(src_pt, dest_pt) return True # third right with gap if i + 3 <= 13: if state[e, i + 3] == color and (state[e, i + 2] >= 1 and state[e, i + 2] <= 5): print("right gap ", color, e, i) src_pt = self.get_click_point(self.data_coordinates[e, i + 3]) dest_pt = self.get_click_point(self.data_coordinates[e, i + 2]) self.move_tile(src_pt, dest_pt) return True except: return False def check_3_vertical(self, state, e, i, color): try: # second color next to starting point o if state[e, i] == color and state[e + 1, i] == color: # third left upper if e - 1 >= 0 and i - 1 >= 0: if state[e - 1, i - 1] == color and (state[e - 1, i] >= 1 and state[e - 1, i] <= 5): print("upper left", color, e, i) src_pt = self.get_click_point(self.data_coordinates[e - 1, i - 1]) dest_pt = self.get_click_point(self.data_coordinates[e - 1, i]) self.move_tile(src_pt, dest_pt) return True # third left lower if e + 2 <= 7 and i - 1 >= 0: if state[e + 2, i - 1] == color and (state[e + 2, i] >= 1 and state[e + 2, i] <= 5): print("lower left", color, e, i) src_pt = self.get_click_point(self.data_coordinates[e + 2, i - 1]) dest_pt = self.get_click_point(self.data_coordinates[e + 2, i]) self.move_tile(src_pt, dest_pt) return True # third right upper if e - 1 >= 0 and i + 1 <= 13: if state[e - 1, i + 1] == color and (state[e - 1, i] >= 1 and state[e - 1, i] <= 5): print("upper right", color, e, i) src_pt = self.get_click_point(self.data_coordinates[e - 1, i + 1]) dest_pt = self.get_click_point(self.data_coordinates[e - 1, i]) self.move_tile(src_pt, dest_pt) return True # third right lower if e + 2 <= 7 and i + 1 <= 13: if state[e + 2, i + 1] == color and (state[e + 2, i] >= 1 and state[e + 2, i] <= 5): print("upper lower", color, e, i) src_pt = self.get_click_point(self.data_coordinates[e + 2, i + 1]) dest_pt = self.get_click_point(self.data_coordinates[e + 2, i]) self.move_tile(src_pt, dest_pt) return True # third upper with gap if e - 2 >= 0: if state[e - 2, i] == color and (state[e - 1, i] >= 1 and state[e - 1, i] <= 5): print("upper gap ", color, e, i) src_pt = self.get_click_point(self.data_coordinates[e - 2, i]) dest_pt = self.get_click_point(self.data_coordinates[e - 1, i]) self.move_tile(src_pt, dest_pt) return True # third lower with gap if e + 3 <= 7: if state[e + 3, i] == color and (state[e + 2, i] >= 1 and state[e + 2, i] <= 5): print("lower gap ", color, e, i) src_pt = self.get_click_point(self.data_coordinates[e + 3, i]) dest_pt = self.get_click_point(self.data_coordinates[e + 2, i]) self.move_tile(src_pt, dest_pt) return True except: return False def move_tile(self, point_source, point_dest): offset_left = 230 offset_down = 58 pydirectinput.moveTo(point_source[0] + offset_left, point_source[1] + offset_down) # pydirectinput.moveTo(0,0) pydirectinput.mouseDown() w = random.randint(25, 50) cv.waitKey(100 + w) pydirectinput.moveTo(point_dest[0] + offset_left, point_dest[1] + offset_down) pydirectinput.mouseUp() cv.waitKey(400 + w) def check_for_button_and_click_it(self, button_url): screenshot = self.capture_window.get_screenshot() # gray = cv.cvtColor(screenshot, cv.COLOR_BGR2GRAY) # thresh = cv.threshold(gray, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1] # gray_needle = cv.cvtColor(cv.imread(button_url, cv.IMREAD_UNCHANGED), cv.COLOR_BGR2GRAY) # thresh_needle = cv.threshold(gray_needle, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1] needle = cv.imread(button_url, cv.IMREAD_UNCHANGED) # rectangles = self.vision_stun.find(thresh, thresh_needle, 0.4, 1) rectangles = self.vision_stun.find(screenshot, needle, 0.7, 1) if len(rectangles) == 1: pointis = self.vision_stun.get_click_points(rectangles) for pointi in pointis: self.dig_point(pointi[0], pointi[1], 150) def dig_point(self, point1, point2, dig_time): pydirectinput.moveTo(point1, point2) cv.waitKey(dig_time) pydirectinput.mouseDown() w = random.randint(50, 100) cv.waitKey(w) pydirectinput.mouseUp() def get_click_point(self, rectangle): # Loop over all the rectangles x, y, w, h = rectangle # Determine the center position center_x = x + int(w / 2) center_y = y + int(h / 2) # Save the points return int(center_x), int(center_y)