import random import cv2 as cv import numpy as np import pydirectinput from window_capture import WindowCapture from vision import Vision from config_file import UserConfigs from utils import mse from utils import get_click_point from hsvfilter import HsvFilter GREEN = 1 YELLOW = 2 RED = 3 BLUE = 4 PURPLE = 5 RAINBOW = 6 BIGBOMB = 7 BOMB = 8 ARROW_DOWN = 9 ARROW_RIGHT = 10 ROCK_1 = 11 ROCK_2 = 12 ROCK_3 = 13 BURGER = 14 PAB1 = 15 GOLDBAR = 16 MAGINENT = 21 CHEMTRANT = 22 TENESENT = 23 CIBUTRANT = 24 ARTISENT = 25 STATUS_FOUND_CONTINUATION = "1" STATUS_FOUND_DEADEND = "2" STATUS_FOUND_ENDING = "3" RAINBOW_STRATEGY = 11 BIGBOMB_STRATEGY = 9 ROCKET_STRATEGY = 7 BOMB_STRATEGY = 5 class Field: data_value_grid = [] data_coordinates = [] screenshot = [] next_level = cv.imread("next_level.jpg", cv.IMREAD_COLOR) next_level_x = cv.imread("next_level_x.jpg", cv.IMREAD_COLOR) reset_board = cv.imread("reset_button.jpg", cv.IMREAD_COLOR) reset_confirm = cv.imread("reset_confirm.jpg", cv.IMREAD_COLOR) reset_counter = 0 colors_at_standard = True needles = {} hsh_needles = {} explosives = [RAINBOW, ARROW_RIGHT, ARROW_DOWN, BIGBOMB, BOMB] colors = [GREEN, YELLOW, RED, BLUE, PURPLE, MAGINENT, CHEMTRANT, TENESENT, CIBUTRANT, ARTISENT] def __init__(self): self.data_value_grid = np.zeros((8, 14), dtype=int) self.data_coordinates = np.zeros((8, 14), dtype=object) self.data_score_map = np.zeros((8, 14), dtype=int) self.observation = np.zeros((8, 14), dtype=int) self.current_strategy = RAINBOW_STRATEGY # 230 to 2110 = 1883 / 14 = 134.5 # 60 to 1130 = 1076 / 8 = 134.5 dim = 134.5 for e in range(0, 8, 1): for i in range(0, 14, 1): self.data_coordinates[e][i] = [i * dim, e * dim, dim, dim] # initialize the user-class self.config = UserConfigs() # initialize the StunWindowCapture class self.capture_window = WindowCapture(None, None, self.config) # initialize the StunVision class self.vision_stun = Vision() self.set_color_order((GREEN, YELLOW, RED, BLUE, PURPLE)) def reset(self): self.observation = [] def set_color_order(self, order): self.needles.clear() self.needles = {order[0]: cv.imread("green.jpg", cv.IMREAD_COLOR), order[1]: cv.imread("yellow.jpg", cv.IMREAD_COLOR), order[2]: cv.imread("red.jpg", cv.IMREAD_COLOR), order[3]: cv.imread("blue.jpg", cv.IMREAD_COLOR), order[4]: cv.imread("purple.jpg", cv.IMREAD_COLOR), RAINBOW: cv.imread("rainbow.jpg", cv.IMREAD_COLOR), BIGBOMB: cv.imread("bigbomb.jpg", cv.IMREAD_COLOR), BOMB: cv.imread("bomb.jpg", cv.IMREAD_COLOR), ARROW_DOWN: cv.imread("arrow_down.jpg", cv.IMREAD_COLOR), ARROW_RIGHT: cv.imread("arrow_right.jpg", cv.IMREAD_COLOR), MAGINENT: cv.imread("maginent.jpg", cv.IMREAD_COLOR), CHEMTRANT: cv.imread("chemtrant.jpg", cv.IMREAD_COLOR), TENESENT: cv.imread("tenesent.jpg", cv.IMREAD_COLOR), CIBUTRANT: cv.imread("cibutrant.jpg", cv.IMREAD_COLOR), ARTISENT: cv.imread("artisent.jpg", cv.IMREAD_COLOR) } self.hsh_needles.clear() self.hsh_needles = {order[0]: HsvFilter(0, 0, 0, 179, 255, 255, 0, 0, 0, 0), order[1]: HsvFilter(0, 0, 0, 179, 255, 255, 0, 0, 0, 0), order[2]: HsvFilter(0, 0, 0, 179, 255, 255, 0, 0, 0, 0), order[3]: HsvFilter(60, 40, 0, 115, 255, 255, 0, 0, 0, 0), order[4]: HsvFilter(120, 45, 0, 170, 255, 255, 0, 0, 0, 0), RAINBOW: HsvFilter(0, 0, 0, 179, 255, 255, 0, 0, 0, 0), BIGBOMB: HsvFilter(60, 40, 0, 179, 129, 129, 0, 0, 0, 0), BOMB: HsvFilter(0, 0, 0, 179, 255, 255, 0, 0, 0, 0), ARROW_DOWN: HsvFilter(0, 0, 0, 179, 255, 255, 0, 0, 0, 0), ARROW_RIGHT: HsvFilter(0, 0, 0, 179, 255, 255, 0, 0, 0, 0), MAGINENT: HsvFilter(0, 0, 0, 179, 255, 255, 0, 0, 0, 0), CHEMTRANT: HsvFilter(0, 0, 0, 179, 255, 255, 0, 0, 0, 0), TENESENT: HsvFilter(0, 0, 0, 179, 255, 255, 0, 0, 0, 0), CIBUTRANT: HsvFilter(0, 0, 0, 179, 255, 255, 0, 0, 0, 0), ARTISENT: HsvFilter(0, 0, 0, 179, 255, 255, 0, 0, 0, 0) } def assess_playfield_and_make_move(self): new_observation, new_screenshot = self.get_current_board_state() # wrong movement detection # last board state is same as actual if mse(new_observation, self.observation) == 0.0: # no movement detected -> blow explosives or reset self.reset_counter += 1 if self.reset_counter == 1: pass elif self.reset_counter == 2: if self.detonate_explosive_when_stuck(new_observation): self.reset_counter = 0 return elif self.reset_counter >= 3: screenshot = self.capture_window.get_screenshot() if self.check_for_button_and_execute(screenshot, self.reset_board): cv.waitKey(500) screenshot = self.capture_window.get_screenshot() if self.check_for_button_and_execute(screenshot, self.reset_confirm): cv.waitKey(500) color_list = [PURPLE, BLUE, RED, YELLOW, GREEN] random.shuffle(color_list) self.set_color_order(color_list) self.current_strategy = random.choice([RAINBOW_STRATEGY, BIGBOMB_STRATEGY, ROCKET_STRATEGY, BOMB_STRATEGY]) self.reset_counter = 0 return else: return self.find_patterns_and_valid_moves(new_observation) self.observation = new_observation return new_observation def move_to(self, x, y): point_src = (1113, 598) pydirectinput.moveTo(point_src[0], point_src[1]) pydirectinput.mouseDown() w = random.randint(1, 100) cv.waitKey(150 + w) pydirectinput.moveTo(x, y) pydirectinput.mouseUp() cv.waitKey(500 + w) def change_value(self, x, y, val): self.data_value_grid[x][y] = val def point_in_rect(self, point): for e in range(0, 8, 1): for i in range(0, 14, 1): x1, y1, w, h = self.data_coordinates[e][i] x2, y2 = x1 + w, y1 + h x, y = point if x1 < x and x < x2: if y1 < y and y < y2: return e, i return None, None def check_for_button_and_execute(self, screen, needle, offset_left=0, offset_down=0): rectangles = self.vision_stun.find(screen, needle, 0.70, 1) if len(rectangles) == 0: return False point = self.vision_stun.get_click_points(rectangles)[0] self.dig_point(point[0] + offset_left, point[1] + offset_down, 500) return True def get_current_board_state(self): try: # get an updated image of the game screenshot = self.capture_window.get_screenshot() # screenshot = cv.imread("playfield.jpg") screenshot = screenshot[58:1134, 230:2113] # 1883,1076 self.screenshot = screenshot # gray = cv.cvtColor(screenshot, cv.COLOR_BGR2GRAY) # thresh = cv.threshold(gray, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1] offset_left = 230 offset_down = 58 if self.check_for_button_and_execute(screenshot, self.next_level, offset_left, offset_down): self.set_color_order((GREEN, YELLOW, RED, BLUE, PURPLE)) self.current_strategy = RAINBOW_STRATEGY cv.waitKey(500) screenshot = self.capture_window.get_screenshot() screenshot = screenshot[58:1134, 230:2113] if self.check_for_button_and_execute(screenshot, self.next_level_x, offset_left, offset_down): cv.waitKey(500) screenshot = self.capture_window.get_screenshot() screenshot = screenshot[58:1134, 230:2113] except: # self.capture_window.release() # print("Game window not available - shutting down application") # return None pass # cv.imshow("screenshot", screenshot) # cv.waitKey(150) # continue data_coords = np.zeros((8, 14), dtype=object) # field = Field() for needle_key in self.needles.keys(): # gray_needle = cv.cvtColor(self.needles[needle_key], cv.COLOR_BGR2GRAY) # thresh_needle = cv.threshold(gray_needle, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1] processed_screenshot = self.vision_stun.apply_hsv_filter(screenshot, self.hsh_needles[needle_key]) processed_needle = self.vision_stun.apply_hsv_filter(self.needles[needle_key], self.hsh_needles[needle_key]) rectangles = self.vision_stun.find(processed_screenshot, processed_needle, 0.70, 56) # rectangles = self.vision_stun.find(screenshot, self.needles[needle_key], 0.70, 56) if len(rectangles) == 0: continue points = self.vision_stun.get_click_points(rectangles) for point in points: x, y = self.point_in_rect(point) if x is not None and y is not None: data_coords[x][y] = int(needle_key) # self.change_value(x, y, int(needle_key)) # print(field.data_value_grid) # cv.circle(screenshot, points[0], 7, (0, 255, 0), -1) # output_image = vision_stun.draw_rectangles(screenshot, rectangles) # cv.imshow("output_image", output_image) # cv.waitKey(150) score_map = np.zeros((8, 14), dtype=object) for x in range(0, 8, 1): for y in range(0, 14, 1): score_map[x, y] = self.score_for_attached_same_color_all_directions(data_coords, x, y) cv.putText(screenshot, str(score_map[x, y]), self.get_click_point(self.data_coordinates[x, y]), cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 0), 3, 2) cv.imwrite('screenshot_scored.jpg', screenshot) return data_coords, screenshot def detonate_explosive_when_stuck(self, state): for e in range(0, 8, 1): for i in range(0, 14, 1): for explosive in self.explosives: if self.local_pos_check(state, e, i, 0, 0, explosive): src_pt = self.get_click_point(self.data_coordinates[e, i]) if self.local_pos_checks(state, e, i, 1, 0, self.colors): dest_pt = self.get_click_point(self.data_coordinates[e + 1, i]) self.move_tile(src_pt, dest_pt) return True elif self.local_pos_checks(state, e, i, 0, 1, self.colors): dest_pt = self.get_click_point(self.data_coordinates[e, i + 1]) self.move_tile(src_pt, dest_pt) return True elif self.local_pos_checks(state, e, i, -1, 0, self.colors): dest_pt = self.get_click_point(self.data_coordinates[e - 1, i]) self.move_tile(src_pt, dest_pt) return True elif self.local_pos_checks(state, e, i, 0, -1, self.colors): dest_pt = self.get_click_point(self.data_coordinates[e, i - 1]) self.move_tile(src_pt, dest_pt) return True else: continue return False def check_explosives(self, state, e, i): for explosive in self.explosives: if self.local_pos_check(state, e, i, 0, 0, explosive): dest_pt = self.get_click_point(self.data_coordinates[e, i]) if self.local_pos_checks(state, e, i, 1, 0, self.explosives): src_pt = self.get_click_point(self.data_coordinates[e + 1, i]) self.move_tile(src_pt, dest_pt) elif self.local_pos_checks(state, e, i, 0, 1, self.explosives): src_pt = self.get_click_point(self.data_coordinates[e, i + 1]) self.move_tile(src_pt, dest_pt) elif self.local_pos_checks(state, e, i, -1, 0, self.explosives): src_pt = self.get_click_point(self.data_coordinates[e - 1, i]) self.move_tile(src_pt, dest_pt) elif self.local_pos_checks(state, e, i, 0, -1, self.explosives): src_pt = self.get_click_point(self.data_coordinates[e, i - 1]) self.move_tile(src_pt, dest_pt) else: continue return True else: continue return False def find_patterns_and_valid_moves(self, state): # score_map = np.zeros((8, 14), dtype=object) for x in range(0, 8, 1): for y in range(0, 14, 1): self.data_score_map[x, y] = self.score_for_attached_same_color_all_directions(state, x, y) full_moves = [] reserve_moves = [] for x in range(0, 8, 1): for y in range(0, 14, 1): if self.check_explosives(state, x, y): return if self.data_score_map[x, y] >= 4: path_option = [[x, y]] recursion_reminder = [[x, y]] result = self.find_next_same_color_all_directions_recursion(state, x, y, path_option, recursion_reminder, True) if result == STATUS_FOUND_ENDING: path_option.append([x, y]) if len(path_option) >= 5: full_moves.append((state[x, y], path_option)) if self.data_score_map[x, y] >= 1: path_option = [[x, y]] self.find_next_same_color_all_directions_recursion2(state, x, y, path_option) if len(path_option) >= 3: reserve_moves.append((state[x, y], path_option)) # print(self.data_score_map) # screenshot = cv.imread("screenshot_scored.jpg") full_dic = self.convert_moves_to_dic_by_color(full_moves) res_dic = self.convert_moves_to_dic_by_color(reserve_moves) full_moves.sort(key=len, reverse=True) reserve_moves.sort(key=len, reverse=True) if len(full_dic) >= 1: for key in sorted(full_dic): if len(full_dic[key]) >= self.current_strategy: self.print_move(full_dic[key]) self.execute_move(full_dic[key]) return if len(reserve_moves) >= 1: for key in sorted(res_dic): self.print_move(res_dic[key]) self.execute_move(res_dic[key]) break def convert_moves_to_dic_by_color(self, reserve_moves): dic = {} for color, move in reserve_moves: if color in dic: if len(move) > len(dic[color]): dic[color] = move else: dic[color] = move return dic def print_move(self, move): mp = [] for point in move: cp = self.get_click_point(self.data_coordinates[point[0], point[1]]) mp.append(cp) cv.polylines(self.screenshot, [np.array(mp)], isClosed=False, color=(0, 255, 0), thickness=3) cv.imwrite('screenshot_pathed.jpg', self.screenshot) def execute_move(self, move): offset_left = 230 offset_down = 58 first = True for cords in move: square = self.data_coordinates[cords[0], cords[1]] pydirectinput.moveTo(get_click_point(square)[0] + offset_left, get_click_point(square)[1] + offset_down) cv.waitKey(50) if first: pydirectinput.mouseDown() cv.waitKey(100) first = False pydirectinput.mouseUp() cv.waitKey(50) def get_directions_array(self, current_x, current_y): left_x = current_x left_y = current_y - 1 right_x = current_x right_y = current_y + 1 upper_x = current_x - 1 upper_y = current_y lower_x = current_x + 1 lower_y = current_y lower_left_x = current_x + 1 lower_left_y = current_y - 1 lower_right_x = current_x + 1 lower_right_y = current_y + 1 upper_left_x = current_x - 1 upper_left_y = current_y - 1 upper_right_x = current_x - 1 upper_right_y = current_y + 1 directions = [[left_x, left_y], [lower_left_x, lower_left_y], [lower_x, lower_y], [lower_right_x, lower_right_y], [right_x, right_y], [upper_right_x, upper_right_y], [upper_x, upper_y], [upper_left_x, upper_left_y]] return directions def find_next_same_color_all_directions_recursion(self, state, current_x, current_y, path_store, recursion_reminder, look_for_ending): directions = self.get_directions_array(current_x, current_y) color = state[current_x, current_y] for direction in directions: if self.is_direction_in_bounce_and_same_color(state, direction, color) == 1: if (self.data_score_map[direction[0], direction[1]] >= 2) and (direction not in recursion_reminder): recursion_reminder.append(direction) result = self.find_next_same_color_all_directions_recursion(state, direction[0], direction[1], path_store, recursion_reminder, look_for_ending) if result == STATUS_FOUND_CONTINUATION: path_store.append(direction) return STATUS_FOUND_CONTINUATION elif result == STATUS_FOUND_DEADEND: if look_for_ending: continue else: path_store.append(direction) return STATUS_FOUND_DEADEND elif result == STATUS_FOUND_ENDING: path_store.append(direction) return STATUS_FOUND_ENDING if look_for_ending: for direction in directions: if direction == path_store[0]: # path_store.append(direction) return STATUS_FOUND_ENDING return STATUS_FOUND_DEADEND def find_next_same_color_all_directions_recursion2(self, state, current_x, current_y, path_store): directions = self.get_directions_array(current_x, current_y) color = state[current_x, current_y] for direction in directions: if self.is_direction_in_bounce_and_same_color(state, direction, color) == 1: if (self.data_score_map[direction[0], direction[1]] >= 1) and (direction not in path_store): path_store.append(direction) result = self.find_next_same_color_all_directions_recursion2(state, direction[0], direction[1], path_store) if result == STATUS_FOUND_DEADEND: return STATUS_FOUND_DEADEND return STATUS_FOUND_DEADEND def score_for_attached_same_color_all_directions(self, state, current_x, current_y): directions = self.get_directions_array(current_x, current_y) score = 0 color = state[current_x, current_y] if color not in self.colors: return score for direction in directions: score = score + self.is_direction_in_bounce_and_same_color(state, direction, color) return score def is_direction_in_bounce_and_same_color(self, state, loc, color): x, y = loc if x <= 7 and x >= 0 and y <= 13 and y >= 0: if state[x, y] == color: return 1 return 0 def local_pos_check(self, state, e, i, e_check, i_check, needle): if e + e_check >= 0 and e + e_check <= 7 and i + i_check >= 0 and i + i_check <= 13: if state[e + e_check, i + i_check] == needle: return True else: return False def local_pos_checks(self, state, e, i, e_check, i_check, needles): if e + e_check >= 0 and e + e_check <= 7 and i + i_check >= 0 and i + i_check <= 13: for needle in needles: if state[e + e_check, i + i_check] == needle: return True else: continue return False def move_tile(self, point_source, point_dest): offset_left = 230 offset_down = 58 pydirectinput.moveTo(point_source[0] + offset_left, point_source[1] + offset_down) # pydirectinput.moveTo(0,0) pydirectinput.mouseDown() w = random.randint(25, 50) cv.waitKey(100 + w) pydirectinput.moveTo(point_dest[0] + offset_left, point_dest[1] + offset_down) pydirectinput.mouseUp() cv.waitKey(400 + w) def check_for_button_and_click_it(self, button_url): screenshot = self.capture_window.get_screenshot() # gray = cv.cvtColor(screenshot, cv.COLOR_BGR2GRAY) # thresh = cv.threshold(gray, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1] # gray_needle = cv.cvtColor(cv.imread(button_url, cv.IMREAD_UNCHANGED), cv.COLOR_BGR2GRAY) # thresh_needle = cv.threshold(gray_needle, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1] needle = cv.imread(button_url, cv.IMREAD_UNCHANGED) # rectangles = self.vision_stun.find(thresh, thresh_needle, 0.4, 1) rectangles = self.vision_stun.find(screenshot, needle, 0.7, 1) if len(rectangles) == 1: pointis = self.vision_stun.get_click_points(rectangles) for pointi in pointis: self.dig_point(pointi[0], pointi[1], 150) def dig_point(self, point1, point2, dig_time): pydirectinput.moveTo(point1, point2) cv.waitKey(dig_time) pydirectinput.mouseDown() w = random.randint(50, 100) cv.waitKey(w) pydirectinput.mouseUp() def get_click_point(self, rectangle): # Loop over all the rectangles x, y, w, h = rectangle # Determine the center position center_x = x + int(w / 2) center_y = y + int(h / 2) # Save the points return int(center_x), int(center_y)