diff --git a/combined_bot_main.py b/combined_bot_main.py index 4a1a23c..50ae5a2 100644 --- a/combined_bot_main.py +++ b/combined_bot_main.py @@ -4,6 +4,8 @@ import cv2 as cv import keyboard from combined_user_interface import PrimaryOverlay from equipment import Equipment +from crops import Crops +from farm import Farm def run(): @@ -35,7 +37,12 @@ def run(): if overlay.rb_int.get() == 1: equip = Equipment(overlay) equip.execute_main_loop() - + elif overlay.rb_int.get() == 2: + crops = Crops(overlay) + crops.execute_main_loop() + elif overlay.rb_int.get() == 3: + farm = Farm(overlay) + farm.execute_main_loop() if __name__ == "__main__": run() diff --git a/crops.py b/crops.py new file mode 100644 index 0000000..51921b8 --- /dev/null +++ b/crops.py @@ -0,0 +1,577 @@ +import random +import cv2 as cv +import numpy as np +import pydirectinput +from window_capture import WindowCapture +from vision import Vision +from config_file import UserConfigs +from utils import mse +from utils import get_click_point +from hsvfilter import HsvFilter + +GREEN = 1 +YELLOW = 2 +RED = 3 +BLUE = 4 +PURPLE = 5 +RAINBOW = 6 +BIGBOMB = 7 +BOMB = 8 +ARROW_DOWN = 9 +ARROW_RIGHT = 10 +ROCK_1 = 11 +ROCK_2 = 12 +ROCK_3 = 13 +BURGER = 14 +PAB1 = 15 +GOLDBAR = 16 +MAGINENT = 21 +CHEMTRANT = 22 +TENESENT = 23 +CIBUTRANT = 24 +ARTISENT = 25 + +STATUS_FOUND_CONTINUATION = "1" +STATUS_FOUND_DEADEND = "2" +STATUS_FOUND_ENDING = "3" + +RAINBOW_STRATEGY = 11 +BIGBOMB_STRATEGY = 9 +ROCKET_STRATEGY = 7 +BOMB_STRATEGY = 5 + +class Crops: + data_value_grid = [] + data_coordinates = [] + screenshot = [] + next_level = cv.imread("crop/next_level.jpg", cv.IMREAD_COLOR) + next_level_x = cv.imread("crop/next_level_x.jpg", cv.IMREAD_COLOR) + reset_board = cv.imread("crop/reset_button.jpg", cv.IMREAD_COLOR) + reset_confirm = cv.imread("crop/reset_confirm.jpg", cv.IMREAD_COLOR) + reset_counter = 0 + colors_at_standard = True + needles = {} + hsh_needles = {} + explosives = [RAINBOW, ARROW_RIGHT, ARROW_DOWN, BIGBOMB, BOMB] + colors = [GREEN, YELLOW, RED, BLUE, PURPLE, MAGINENT, CHEMTRANT, TENESENT, CIBUTRANT, ARTISENT] + + def __init__(self, overlay): + self.overlay = overlay + self.data_value_grid = np.zeros((8, 14), dtype=int) + self.data_coordinates = np.zeros((8, 14), dtype=object) + self.data_score_map = np.zeros((8, 14), dtype=int) + self.observation = np.zeros((8, 14), dtype=int) + + self.current_strategy = RAINBOW_STRATEGY + + # 230 to 2110 = 1883 / 14 = 134.5 + # 60 to 1130 = 1076 / 8 = 134.5 + dim = 134.5 + for e in range(0, 8, 1): + for i in range(0, 14, 1): + self.data_coordinates[e][i] = [i * dim, e * dim, dim, dim] + + # initialize the user-class + self.config = UserConfigs() + + # initialize the StunWindowCapture class + self.capture_window = WindowCapture(None, None, self.config) + # initialize the StunVision class + self.vision_stun = Vision() + + self.set_color_order((GREEN, YELLOW, RED, BLUE, PURPLE)) + + def execute_main_loop(self): + while True: + if self.overlay.run_mode == 'stopped': + break + + self.assess_playfield_and_make_move() + + def reset(self): + self.observation = [] + + def set_color_order(self, order): + self.needles.clear() + self.needles = {order[0]: cv.imread("crop/green.jpg", cv.IMREAD_COLOR), + order[1]: cv.imread("crop/yellow.jpg", cv.IMREAD_COLOR), + order[2]: cv.imread("crop/red.jpg", cv.IMREAD_COLOR), + order[3]: cv.imread("crop/blue.jpg", cv.IMREAD_COLOR), + order[4]: cv.imread("crop/purple.jpg", cv.IMREAD_COLOR), + + RAINBOW: cv.imread("crop/rainbow.jpg", cv.IMREAD_COLOR), + BIGBOMB: cv.imread("crop/bigbomb.jpg", cv.IMREAD_COLOR), + BOMB: cv.imread("crop/bomb.jpg", cv.IMREAD_COLOR), + ARROW_DOWN: cv.imread("crop/arrow_down.jpg", cv.IMREAD_COLOR), + ARROW_RIGHT: cv.imread("crop/arrow_right.jpg", cv.IMREAD_COLOR), + + MAGINENT: cv.imread("crop/maginent.jpg", cv.IMREAD_COLOR), + CHEMTRANT: cv.imread("crop/chemtrant.jpg", cv.IMREAD_COLOR), + TENESENT: cv.imread("crop/tenesent.jpg", cv.IMREAD_COLOR), + CIBUTRANT: cv.imread("crop/cibutrant.jpg", cv.IMREAD_COLOR), + ARTISENT: cv.imread("crop/artisent.jpg", cv.IMREAD_COLOR) + } + self.hsh_needles.clear() + self.hsh_needles = {order[0]: HsvFilter(0, 0, 0, 179, 255, 255, 0, 0, 0, 0), + order[1]: HsvFilter(0, 0, 0, 179, 255, 255, 0, 0, 0, 0), + order[2]: HsvFilter(0, 0, 0, 179, 255, 255, 0, 0, 0, 0), + order[3]: HsvFilter(60, 40, 0, 115, 255, 255, 0, 0, 0, 0), + order[4]: HsvFilter(120, 45, 0, 170, 255, 255, 0, 0, 0, 0), + + RAINBOW: HsvFilter(0, 0, 0, 179, 255, 255, 0, 0, 0, 0), + BIGBOMB: HsvFilter(60, 40, 0, 179, 129, 129, 0, 0, 0, 0), + BOMB: HsvFilter(0, 0, 0, 179, 255, 255, 0, 0, 0, 0), + ARROW_DOWN: HsvFilter(0, 0, 0, 179, 255, 255, 0, 0, 0, 0), + ARROW_RIGHT: HsvFilter(0, 0, 0, 179, 255, 255, 0, 0, 0, 0), + + MAGINENT: HsvFilter(0, 0, 0, 179, 255, 255, 0, 0, 0, 0), + CHEMTRANT: HsvFilter(0, 0, 0, 179, 255, 255, 0, 0, 0, 0), + TENESENT: HsvFilter(0, 0, 0, 179, 255, 255, 0, 0, 0, 0), + CIBUTRANT: HsvFilter(0, 0, 0, 179, 255, 255, 0, 0, 0, 0), + ARTISENT: HsvFilter(0, 0, 0, 179, 255, 255, 0, 0, 0, 0) + } + + def assess_playfield_and_make_move(self): + + new_observation, new_screenshot = self.get_current_board_state() + + # wrong movement detection + # last board state is same as actual + if mse(new_observation, self.observation) == 0.0: + # no movement detected -> blow explosives or reset + + self.reset_counter += 1 + if self.reset_counter == 1: + pass + elif self.reset_counter == 2: + if self.detonate_explosive_when_stuck(new_observation): + self.reset_counter = 0 + return + elif self.reset_counter >= 3: + screenshot = self.capture_window.get_screenshot() + if self.check_for_button_and_execute(screenshot, self.reset_board): + cv.waitKey(500) + screenshot = self.capture_window.get_screenshot() + if self.check_for_button_and_execute(screenshot, self.reset_confirm): + cv.waitKey(500) + + color_list = [PURPLE, BLUE, RED, YELLOW, GREEN] + random.shuffle(color_list) + self.set_color_order(color_list) + self.current_strategy = random.choice([RAINBOW_STRATEGY, BIGBOMB_STRATEGY, ROCKET_STRATEGY, BOMB_STRATEGY]) + self.reset_counter = 0 + return + else: + return + + self.find_patterns_and_valid_moves(new_observation) + self.observation = new_observation + return new_observation + + def move_to(self, x, y): + point_src = (1113, 598) + pydirectinput.moveTo(point_src[0], point_src[1]) + pydirectinput.mouseDown() + w = random.randint(1, 100) + cv.waitKey(150 + w) + pydirectinput.moveTo(x, y) + pydirectinput.mouseUp() + cv.waitKey(500 + w) + + def change_value(self, x, y, val): + self.data_value_grid[x][y] = val + + def point_in_rect(self, point): + for e in range(0, 8, 1): + for i in range(0, 14, 1): + x1, y1, w, h = self.data_coordinates[e][i] + x2, y2 = x1 + w, y1 + h + x, y = point + if x1 < x and x < x2: + if y1 < y and y < y2: + return e, i + return None, None + + def check_for_button_and_execute(self, screen, needle, offset_left=0, offset_down=0): + rectangles = self.vision_stun.find(screen, needle, 0.70, 1) + if len(rectangles) == 0: + return False + point = self.vision_stun.get_click_points(rectangles)[0] + self.dig_point(point[0] + offset_left, point[1] + offset_down, 500) + return True + + def get_current_board_state(self): + try: + # get an updated image of the game + screenshot = self.capture_window.get_screenshot() + # screenshot = cv.imread("playfield.jpg") + screenshot = screenshot[58:1134, 230:2113] # 1883,1076 + self.screenshot = screenshot + # gray = cv.cvtColor(screenshot, cv.COLOR_BGR2GRAY) + # thresh = cv.threshold(gray, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1] + offset_left = 230 + offset_down = 58 + if self.check_for_button_and_execute(screenshot, self.next_level, offset_left, offset_down): + self.set_color_order((GREEN, YELLOW, RED, BLUE, PURPLE)) + self.current_strategy = RAINBOW_STRATEGY + cv.waitKey(500) + screenshot = self.capture_window.get_screenshot() + screenshot = screenshot[58:1134, 230:2113] + if self.check_for_button_and_execute(screenshot, self.next_level_x, offset_left, offset_down): + cv.waitKey(500) + screenshot = self.capture_window.get_screenshot() + screenshot = screenshot[58:1134, 230:2113] + + except: + # self.capture_window.release() + # print("Game window not available - shutting down application") + # return None + pass + # cv.imshow("screenshot", screenshot) + # cv.waitKey(150) + # continue + data_coords = np.zeros((8, 14), dtype=object) + # field = Field() + + for needle_key in self.needles.keys(): + # gray_needle = cv.cvtColor(self.needles[needle_key], cv.COLOR_BGR2GRAY) + # thresh_needle = cv.threshold(gray_needle, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1] + processed_screenshot = self.vision_stun.apply_hsv_filter(screenshot, self.hsh_needles[needle_key]) + processed_needle = self.vision_stun.apply_hsv_filter(self.needles[needle_key], self.hsh_needles[needle_key]) + rectangles = self.vision_stun.find(processed_screenshot, processed_needle, 0.70, 56) + # rectangles = self.vision_stun.find(screenshot, self.needles[needle_key], 0.70, 56) + if len(rectangles) == 0: + continue + points = self.vision_stun.get_click_points(rectangles) + + for point in points: + x, y = self.point_in_rect(point) + if x is not None and y is not None: + data_coords[x][y] = int(needle_key) + # self.change_value(x, y, int(needle_key)) + # print(field.data_value_grid) + # cv.circle(screenshot, points[0], 7, (0, 255, 0), -1) + # output_image = vision_stun.draw_rectangles(screenshot, rectangles) + # cv.imshow("output_image", output_image) + # cv.waitKey(150) + + score_map = np.zeros((8, 14), dtype=object) + for x in range(0, 8, 1): + for y in range(0, 14, 1): + score_map[x, y] = self.score_for_attached_same_color_all_directions(data_coords, x, y) + #cv.putText(screenshot, str(score_map[x, y]), + # self.get_click_point(self.data_coordinates[x, y]), cv.FONT_HERSHEY_SIMPLEX, + # 1, (0, 0, 0), 3, 2) + #cv.imwrite('screenshot_scored.jpg', screenshot) + + return data_coords, screenshot + + def detonate_explosive_when_stuck(self, state): + for e in range(0, 8, 1): + for i in range(0, 14, 1): + for explosive in self.explosives: + if self.local_pos_check(state, e, i, 0, 0, explosive): + src_pt = self.get_click_point(self.data_coordinates[e, i]) + if self.local_pos_checks(state, e, i, 1, 0, self.colors): + dest_pt = self.get_click_point(self.data_coordinates[e + 1, i]) + self.move_tile(src_pt, dest_pt) + return True + elif self.local_pos_checks(state, e, i, 0, 1, self.colors): + dest_pt = self.get_click_point(self.data_coordinates[e, i + 1]) + self.move_tile(src_pt, dest_pt) + return True + elif self.local_pos_checks(state, e, i, -1, 0, self.colors): + dest_pt = self.get_click_point(self.data_coordinates[e - 1, i]) + self.move_tile(src_pt, dest_pt) + return True + elif self.local_pos_checks(state, e, i, 0, -1, self.colors): + dest_pt = self.get_click_point(self.data_coordinates[e, i - 1]) + self.move_tile(src_pt, dest_pt) + return True + else: + continue + return False + + def check_explosives(self, state, e, i): + for explosive in self.explosives: + if self.local_pos_check(state, e, i, 0, 0, explosive): + dest_pt = self.get_click_point(self.data_coordinates[e, i]) + if self.local_pos_checks(state, e, i, 1, 0, self.explosives): + src_pt = self.get_click_point(self.data_coordinates[e + 1, i]) + self.move_tile(src_pt, dest_pt) + elif self.local_pos_checks(state, e, i, 0, 1, self.explosives): + src_pt = self.get_click_point(self.data_coordinates[e, i + 1]) + self.move_tile(src_pt, dest_pt) + elif self.local_pos_checks(state, e, i, -1, 0, self.explosives): + src_pt = self.get_click_point(self.data_coordinates[e - 1, i]) + self.move_tile(src_pt, dest_pt) + elif self.local_pos_checks(state, e, i, 0, -1, self.explosives): + src_pt = self.get_click_point(self.data_coordinates[e, i - 1]) + self.move_tile(src_pt, dest_pt) + else: + continue + return True + else: + continue + return False + + def find_patterns_and_valid_moves(self, state): + + # score_map = np.zeros((8, 14), dtype=object) + for x in range(0, 8, 1): + for y in range(0, 14, 1): + self.data_score_map[x, y] = self.score_for_attached_same_color_all_directions(state, x, y) + if self.overlay.run_mode == 'stopped': + break + if self.overlay.run_mode == 'stopped': + break + + full_moves = [] + reserve_moves = [] + for x in range(0, 8, 1): + for y in range(0, 14, 1): + if self.check_explosives(state, x, y): + return + if self.data_score_map[x, y] >= 4: + path_option = [[x, y]] + recursion_reminder = [[x, y]] + result = self.find_next_same_color_all_directions_recursion(state, x, y, path_option, + recursion_reminder, True) + if result == STATUS_FOUND_ENDING: + path_option.append([x, y]) + if len(path_option) >= 5: + full_moves.append((state[x, y], path_option)) + if self.overlay.run_mode == 'stopped': + break + if self.data_score_map[x, y] >= 1: + path_option = [[x, y]] + self.find_next_same_color_all_directions_recursion2(state, x, y, path_option) + if len(path_option) >= 3: + reserve_moves.append((state[x, y], path_option)) + if self.overlay.run_mode == 'stopped': + break + if self.overlay.run_mode == 'stopped': + break + if self.overlay.run_mode == 'stopped': + break + # print(self.data_score_map) + + # screenshot = cv.imread("screenshot_scored.jpg") + full_dic = self.convert_moves_to_dic_by_color(full_moves) + res_dic = self.convert_moves_to_dic_by_color(reserve_moves) + + full_moves.sort(key=len, reverse=True) + reserve_moves.sort(key=len, reverse=True) + + if len(full_dic) >= 1: + for key in sorted(full_dic): + if len(full_dic[key]) >= self.current_strategy: + #self.print_move(full_dic[key]) + self.execute_move(full_dic[key]) + return + if len(reserve_moves) >= 1: + for key in sorted(res_dic): + #self.print_move(res_dic[key]) + self.execute_move(res_dic[key]) + break + + def convert_moves_to_dic_by_color(self, reserve_moves): + dic = {} + for color, move in reserve_moves: + if color in dic: + if len(move) > len(dic[color]): + dic[color] = move + else: + dic[color] = move + return dic + + def print_move(self, move): + mp = [] + for point in move: + cp = self.get_click_point(self.data_coordinates[point[0], point[1]]) + mp.append(cp) + cv.polylines(self.screenshot, + [np.array(mp)], + isClosed=False, + color=(0, 255, 0), + thickness=3) + + cv.imwrite('screenshot_pathed.jpg', self.screenshot) + + def execute_move(self, move): + offset_left = 230 + offset_down = 58 + + first = True + for cords in move: + square = self.data_coordinates[cords[0], cords[1]] + pydirectinput.moveTo(get_click_point(square)[0] + offset_left, get_click_point(square)[1] + offset_down) + cv.waitKey(50) + if first: + pydirectinput.mouseDown() + cv.waitKey(100) + first = False + pydirectinput.mouseUp() + cv.waitKey(50) + + def get_directions_array(self, current_x, current_y): + left_x = current_x + left_y = current_y - 1 + + right_x = current_x + right_y = current_y + 1 + + upper_x = current_x - 1 + upper_y = current_y + + lower_x = current_x + 1 + lower_y = current_y + + lower_left_x = current_x + 1 + lower_left_y = current_y - 1 + + lower_right_x = current_x + 1 + lower_right_y = current_y + 1 + + upper_left_x = current_x - 1 + upper_left_y = current_y - 1 + + upper_right_x = current_x - 1 + upper_right_y = current_y + 1 + + directions = [[left_x, left_y], [lower_left_x, lower_left_y], [lower_x, lower_y], + [lower_right_x, lower_right_y], [right_x, right_y], [upper_right_x, upper_right_y], + [upper_x, upper_y], [upper_left_x, upper_left_y]] + + return directions + + def find_next_same_color_all_directions_recursion(self, state, current_x, current_y, path_store, recursion_reminder, + look_for_ending): + directions = self.get_directions_array(current_x, current_y) + color = state[current_x, current_y] + + for direction in directions: + if self.is_direction_in_bounce_and_same_color(state, direction, color) == 1: + if (self.data_score_map[direction[0], direction[1]] >= 2) and (direction not in recursion_reminder): + recursion_reminder.append(direction) + result = self.find_next_same_color_all_directions_recursion(state, direction[0], direction[1], + path_store, recursion_reminder, + look_for_ending) + if self.overlay.run_mode == 'stopped': + break + if result == STATUS_FOUND_CONTINUATION: + path_store.append(direction) + return STATUS_FOUND_CONTINUATION + elif result == STATUS_FOUND_DEADEND: + if look_for_ending: + continue + else: + path_store.append(direction) + return STATUS_FOUND_DEADEND + elif result == STATUS_FOUND_ENDING: + path_store.append(direction) + return STATUS_FOUND_ENDING + + if look_for_ending: + for direction in directions: + if direction == path_store[0]: + # path_store.append(direction) + return STATUS_FOUND_ENDING + + return STATUS_FOUND_DEADEND + + def find_next_same_color_all_directions_recursion2(self, state, current_x, current_y, path_store): + directions = self.get_directions_array(current_x, current_y) + color = state[current_x, current_y] + + for direction in directions: + if self.is_direction_in_bounce_and_same_color(state, direction, color) == 1: + if (self.data_score_map[direction[0], direction[1]] >= 1) and (direction not in path_store): + path_store.append(direction) + result = self.find_next_same_color_all_directions_recursion2(state, direction[0], direction[1], + path_store) + if self.overlay.run_mode == 'stopped': + break + if result == STATUS_FOUND_DEADEND: + return STATUS_FOUND_DEADEND + + return STATUS_FOUND_DEADEND + + def score_for_attached_same_color_all_directions(self, state, current_x, current_y): + directions = self.get_directions_array(current_x, current_y) + + score = 0 + color = state[current_x, current_y] + if color not in self.colors: + return score + for direction in directions: + score = score + self.is_direction_in_bounce_and_same_color(state, direction, color) + + return score + + def is_direction_in_bounce_and_same_color(self, state, loc, color): + x, y = loc + if x <= 7 and x >= 0 and y <= 13 and y >= 0: + if state[x, y] == color: + return 1 + return 0 + + def local_pos_check(self, state, e, i, e_check, i_check, needle): + if e + e_check >= 0 and e + e_check <= 7 and i + i_check >= 0 and i + i_check <= 13: + if state[e + e_check, i + i_check] == needle: + return True + else: + return False + + def local_pos_checks(self, state, e, i, e_check, i_check, needles): + if e + e_check >= 0 and e + e_check <= 7 and i + i_check >= 0 and i + i_check <= 13: + for needle in needles: + if state[e + e_check, i + i_check] == needle: + return True + else: + continue + return False + + def move_tile(self, point_source, point_dest): + offset_left = 230 + offset_down = 58 + pydirectinput.moveTo(point_source[0] + offset_left, point_source[1] + offset_down) + # pydirectinput.moveTo(0,0) + pydirectinput.mouseDown() + w = random.randint(25, 50) + cv.waitKey(100 + w) + pydirectinput.moveTo(point_dest[0] + offset_left, point_dest[1] + offset_down) + pydirectinput.mouseUp() + cv.waitKey(400 + w) + + def check_for_button_and_click_it(self, button_url): + screenshot = self.capture_window.get_screenshot() + # gray = cv.cvtColor(screenshot, cv.COLOR_BGR2GRAY) + # thresh = cv.threshold(gray, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1] + # gray_needle = cv.cvtColor(cv.imread(button_url, cv.IMREAD_UNCHANGED), cv.COLOR_BGR2GRAY) + # thresh_needle = cv.threshold(gray_needle, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1] + needle = cv.imread(button_url, cv.IMREAD_UNCHANGED) + # rectangles = self.vision_stun.find(thresh, thresh_needle, 0.4, 1) + rectangles = self.vision_stun.find(screenshot, needle, 0.7, 1) + + if len(rectangles) == 1: + pointis = self.vision_stun.get_click_points(rectangles) + for pointi in pointis: + self.dig_point(pointi[0], pointi[1], 150) + + def dig_point(self, point1, point2, dig_time): + pydirectinput.moveTo(point1, point2) + cv.waitKey(dig_time) + pydirectinput.mouseDown() + w = random.randint(50, 100) + cv.waitKey(w) + pydirectinput.mouseUp() + + def get_click_point(self, rectangle): + # Loop over all the rectangles + x, y, w, h = rectangle + # Determine the center position + center_x = x + int(w / 2) + center_y = y + int(h / 2) + # Save the points + return int(center_x), int(center_y) diff --git a/farm.py b/farm.py new file mode 100644 index 0000000..c14beb0 --- /dev/null +++ b/farm.py @@ -0,0 +1,575 @@ +import random +import cv2 as cv +import numpy as np +import pydirectinput +from window_capture import WindowCapture +from vision import Vision +from config_file import UserConfigs +from utils import mse + +GREEN = 1 +YELLOW = 2 +BLUE = 3 +RED = 4 +PINK = 5 +RAINBOW = 6 +BIGBOMB = 7 +BOMB = 8 +ARROW_DOWN = 9 +ARROW_RIGHT = 10 +ROCK_1 = 11 +ROCK_2 = 12 +ROCK_3 = 13 +BURGER = 14 +PAB1 = 15 +GOLDBAR = 16 +MAGINENT = 21 +CHEMTRANT = 22 +TENESENT = 23 +CIBUTRANT = 24 +ARTISENT = 25 + + +class Farm: + data_value_grid = [] + data_coordinates = [] + next_level = cv.imread("farm/next_level.jpg", cv.IMREAD_COLOR) + next_level_x = cv.imread("farm/next_level_x.jpg", cv.IMREAD_COLOR) + reset_board = cv.imread("farm/reset_button.jpg", cv.IMREAD_COLOR) + reset_confirm = cv.imread("farm/reset_confirm.jpg", cv.IMREAD_COLOR) + reset_counter = 0 + + explosives = [] + explosives.append(RAINBOW) + explosives.append(ARROW_RIGHT) + explosives.append(ARROW_DOWN) + explosives.append(BIGBOMB) + explosives.append(BOMB) + colors = [] + colors.append(GREEN) + colors.append(YELLOW) + colors.append(BLUE) + colors.append(RED) + colors.append(PINK) + colors.append(MAGINENT) + colors.append(CHEMTRANT) + colors.append(TENESENT) + colors.append(CIBUTRANT) + colors.append(ARTISENT) + + + def __init__(self, overlay): + self.overlay = overlay + self.data_value_grid = np.zeros((8, 14), dtype=int) + self.data_coordinates = np.zeros((8, 14), dtype=object) + self.observation = np.zeros((8, 14), dtype=int) + + # 230 to 2110 = 1883 / 14 = 134.5 + # 60 to 1130 = 1076 / 8 = 134.5 + dim = 134.5 + for e in range(0, 8, 1): + for i in range(0, 14, 1): + self.data_coordinates[e][i] = [i * dim, e * dim, dim, dim] + + # initialize the user-class + self.config = UserConfigs() + + # initialize the StunWindowCapture class + self.capture_window = WindowCapture(None, None, self.config) + # initialize the StunVision class + self.vision_stun = Vision() + + self.needles = {GREEN: cv.imread("farm/green.jpg", cv.IMREAD_COLOR), + YELLOW: cv.imread("farm/yellow.jpg", cv.IMREAD_COLOR), + BLUE: cv.imread("farm/blue.jpg", cv.IMREAD_COLOR), + RED: cv.imread("farm/red.jpg", cv.IMREAD_COLOR), + PINK: cv.imread("farm/pink.jpg", cv.IMREAD_COLOR), + RAINBOW: cv.imread("farm/rainbow.jpg", cv.IMREAD_COLOR), + BIGBOMB: cv.imread("farm/bigbomb.jpg", cv.IMREAD_COLOR), + BOMB: cv.imread("farm/bomb.jpg", cv.IMREAD_COLOR), + ARROW_DOWN: cv.imread("farm/arrow_down.jpg", cv.IMREAD_COLOR), + ARROW_RIGHT: cv.imread("farm/arrow_right.jpg", cv.IMREAD_COLOR), + ROCK_1: cv.imread("farm/rock1.jpg", cv.IMREAD_COLOR), + ROCK_2: cv.imread("farm/rock2.jpg", cv.IMREAD_COLOR), + ROCK_3: cv.imread("farm/rock3.jpg", cv.IMREAD_COLOR), + BURGER: cv.imread("farm/burger.jpg", cv.IMREAD_COLOR), + GOLDBAR: cv.imread("farm/burger.jpg", cv.IMREAD_COLOR), + PAB1: cv.imread("farm/pab1.jpg", cv.IMREAD_COLOR), + MAGINENT: cv.imread("farm/maginent.jpg", cv.IMREAD_COLOR), + CHEMTRANT: cv.imread("farm/chemtrant.jpg", cv.IMREAD_COLOR), + TENESENT: cv.imread("farm/tenesent.jpg", cv.IMREAD_COLOR), + CIBUTRANT: cv.imread("farm/cibutrant.jpg", cv.IMREAD_COLOR), + ARTISENT: cv.imread("farm/artisent.jpg", cv.IMREAD_COLOR) + } + + def execute_main_loop(self): + while True: + if self.overlay.run_mode == 'stopped': + break + + self.assess_playfield_and_make_move() + + def reset(self): + self.observation = [] + + def assess_playfield_and_make_move(self): + + new_observation, new_screenshot = self.get_current_board_state() + + # wrong movement detection + # last board state is same as actual + if mse(new_observation, self.observation) == 0.0: + # no movement detected -> blow explosives or reset + + self.reset_counter += 1 + if self.reset_counter == 1: + pass + elif self.reset_counter == 2: + if self.detonate_explosive_when_stuck(new_observation): + self.reset_counter = 0 + return + elif self.reset_counter >= 3: + screenshot = self.capture_window.get_screenshot() + if self.check_for_button_and_execute(screenshot, self.reset_board): + cv.waitKey(500) + screenshot = self.capture_window.get_screenshot() + if self.check_for_button_and_execute(screenshot, self.reset_confirm): + cv.waitKey(500) + + self.reset_counter = 0 + return + else: + return + + self.find_patterns_and_valid_moves(new_observation) + self.observation = new_observation + return new_observation + + def check_for_button_and_execute(self, screen, needle, offset_left=0, offset_down=0): + rectangles = self.vision_stun.find(screen, needle, 0.70, 1) + if len(rectangles) == 0: + return False + point = self.vision_stun.get_click_points(rectangles)[0] + self.dig_point(point[0] + offset_left, point[1] + offset_down, 500) + return True + + def move_to(self, x, y): + point_src = (1113, 598) + pydirectinput.moveTo(point_src[0], point_src[1]) + pydirectinput.mouseDown() + w = random.randint(1, 100) + cv.waitKey(150 + w) + pydirectinput.moveTo(x, y) + pydirectinput.mouseUp() + cv.waitKey(500 + w) + + def change_value(self, x, y, val): + self.data_value_grid[x][y] = val + + def pointInRect(self, point): + for e in range(0, 8, 1): + for i in range(0, 14, 1): + x1, y1, w, h = self.data_coordinates[e][i] + x2, y2 = x1 + w, y1 + h + x, y = point + if (x1 < x and x < x2): + if (y1 < y and y < y2): + return e, i + return None, None + + def check_for_next_level(self, screen, needle): + offset_left = 230 + offset_down = 58 + rectangles = self.vision_stun.find(screen, needle, 0.70, 1) + if len(rectangles) == 0: + return False + point = self.vision_stun.get_click_points(rectangles)[0] + self.dig_point(point[0]+ offset_left, point[1] + offset_down, 500) + return True + + def get_current_board_state(self): + try: + # get an updated image of the game + screenshot = self.capture_window.get_screenshot() + #screenshot = cv.imread("field_farm.jpg") + screenshot = screenshot[58:1134, 230:2113] # 1883,1076 + # gray = cv.cvtColor(screenshot, cv.COLOR_BGR2GRAY) + # thresh = cv.threshold(gray, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1] + if self.check_for_next_level(screenshot, self.next_level): + cv.waitKey(500) + screenshot = self.capture_window.get_screenshot() + screenshot = screenshot[58:1134, 230:2113] + if self.check_for_next_level(screenshot, self.next_level_x): + cv.waitKey(500) + screenshot = self.capture_window.get_screenshot() + screenshot = screenshot[58:1134, 230:2113] + + except: + self.capture_window.release() + print("Game window not available - shutting down application") + return None + # cv.imshow("screenshot", screenshot) + # cv.waitKey(150) + # continue + data_coords = np.zeros((8, 14), dtype=object) + # field = Field() + for needle_key in self.needles.keys(): + # gray_needle = cv.cvtColor(self.needles[needle_key], cv.COLOR_BGR2GRAY) + # thresh_needle = cv.threshold(gray_needle, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1] + rectangles = self.vision_stun.find(screenshot, self.needles[needle_key], 0.70, 56) + if len(rectangles) == 0: + continue + points = self.vision_stun.get_click_points(rectangles) + + for point in points: + x, y = self.pointInRect(point) + if x is not None and y is not None: + data_coords[x][y] = int(needle_key) + # self.change_value(x, y, int(needle_key)) + # print(field.data_value_grid) + # cv.circle(screenshot, points[0], 7, (0, 255, 0), -1) + # output_image = vision_stun.draw_rectangles(screenshot, rectangles) + # cv.imshow("output_image", output_image) + # cv.waitKey(150) + return data_coords, screenshot + + def find_patterns_and_valid_moves(self, state): + for e in range(0, 8, 1): + for i in range(0, 14, 1): + if self.check_explosives(state, e, i): + return + if self.overlay.run_mode == 'stopped': + break + if self.overlay.run_mode == 'stopped': + break + + for e in range(0, 8, 1): + for i in range(0, 14, 1): + for color in self.colors: + if self.check_5_horizontal(state, e, i, color): + return + if self.check_5_vertical(state, e, i, color): + return + if self.overlay.run_mode == 'stopped': + break + if self.overlay.run_mode == 'stopped': + break + if self.overlay.run_mode == 'stopped': + break + + for e in range(0, 8, 1): + for i in range(0, 14, 1): + for color in self.colors: + if self.check_3_horizontal(state, e, i, color): + return + if self.check_3_vertical(state, e, i, color): + return + if self.check_3_with_gap(state, e, i, color): + return + if self.overlay.run_mode == 'stopped': + break + if self.overlay.run_mode == 'stopped': + break + if self.overlay.run_mode == 'stopped': + break + + def detonate_explosive_when_stuck(self, state): + for e in range(0, 8, 1): + for i in range(0, 14, 1): + for explosive in self.explosives: + if self.local_pos_check(state, e, i, 0, 0, explosive): + dest_pt = self.get_click_point(self.data_coordinates[e, i]) + if self.local_pos_checks(state, e, i, 1, 0, self.colors): + src_pt = self.get_click_point(self.data_coordinates[e + 1, i]) + self.move_tile(src_pt, dest_pt) + return True + elif self.local_pos_checks(state, e, i, 0, 1, self.colors): + src_pt = self.get_click_point(self.data_coordinates[e, i + 1]) + self.move_tile(src_pt, dest_pt) + return True + elif self.local_pos_checks(state, e, i, -1, 0, self.colors): + src_pt = self.get_click_point(self.data_coordinates[e - 1, i]) + self.move_tile(src_pt, dest_pt) + return True + elif self.local_pos_checks(state, e, i, 0, -1, self.colors): + src_pt = self.get_click_point(self.data_coordinates[e, i - 1]) + self.move_tile(src_pt, dest_pt) + return True + else: + continue + return False + + def check_explosives(self, state, e, i): + for explosive in self.explosives: + if self.local_pos_check(state, e, i, 0, 0, explosive): + dest_pt = self.get_click_point(self.data_coordinates[e, i]) + if self.local_pos_checks(state, e, i, 1, 0, self.explosives): + src_pt = self.get_click_point(self.data_coordinates[e + 1, i]) + self.move_tile(src_pt, dest_pt) + elif self.local_pos_checks(state, e, i, 0, 1, self.explosives): + src_pt = self.get_click_point(self.data_coordinates[e, i + 1]) + self.move_tile(src_pt, dest_pt) + elif self.local_pos_checks(state, e, i, -1, 0, self.explosives): + src_pt = self.get_click_point(self.data_coordinates[e - 1, i]) + self.move_tile(src_pt, dest_pt) + elif self.local_pos_checks(state, e, i, 0, -1, self.explosives): + src_pt = self.get_click_point(self.data_coordinates[e, i - 1]) + self.move_tile(src_pt, dest_pt) + else: + continue + return True + else: + continue + return False + + def local_pos_check(self, state, e, i, e_check, i_check, needle): + if e + e_check >= 0 and e + e_check <=7 and i + i_check >= 0 and i + i_check <=13: + if state[e + e_check, i + i_check] == needle: + return True + else: + return False + + def local_pos_checks(self, state, e, i, e_check, i_check, needles): + if e + e_check >= 0 and e + e_check <=7 and i + i_check >= 0 and i + i_check <=13: + for needle in needles: + if state[e + e_check, i + i_check] == needle: + return True + else: + continue + return False + + def check_5_horizontal(self, state, e, i, color): + try: + # + if state[e, i] == color and state[e, i + 1] == color and state[e, i + 3] == color and state[ + e, i + 4] == color: + if state[e, i + 2] <= 0 or state[e, i + 2] >= 6: + return False + # + if e - 1 >= 0 and i + 2 <= 13: + if state[e - 1, i + 2] == color: + return True + return False + except: + return False + + def check_5_vertical(self, state, e, i, color): + try: + # + if state[e, i] == color and state[e + 1, i] == color and state[e + 3, i] == color and state[ + e + 4, i] == color: + if state[e + 2, i] <= 0 or state[e + 2, i] >= 6: + return False + # third left upper + if e + 2 <= 7 and i - 1 >= 0: + if state[e + 2, i - 1] == color: + print("upper left", color, e, i) + src_pt = self.get_click_point(self.data_coordinates[e + 2, i - 1]) + dest_pt = self.get_click_point(self.data_coordinates[e + 2, i]) + self.move_tile(src_pt, dest_pt) + return True + if e + 2 <= 7 and i + 1 <= 13: + if state[e + 2, i + 1] == color: + print("upper left", color, e, i) + src_pt = self.get_click_point(self.data_coordinates[e + 2, i + 1]) + dest_pt = self.get_click_point(self.data_coordinates[e + 2, i]) + self.move_tile(src_pt, dest_pt) + return True + return False + except: + return False + + def check_3_with_gap(self, state, e, i, color): + try: + # second color next to starting point + if i + 2 <= 13: + if state[e, i] == color and state[e, i + 2] == color: + # third upper + if e - 1 >= 0 and i + 1 <= 13: + #if state[e - 1, i - 1] == color and (state[e, i - 1] >= 1 and state[e, i - 1] <= 5): + if state[e - 1, i + 1] == color and (state[e, i + 1] in self.colors): + src_pt = self.get_click_point(self.data_coordinates[e - 1, i + 1]) + dest_pt = self.get_click_point(self.data_coordinates[e, i + 1]) + self.move_tile(src_pt, dest_pt) + return True + # third left lower + if e + 1 <= 7 and i + 1 <= 13: + if state[e + 1, i + 1] == color and (state[e, i + 1] in self.colors): + src_pt = self.get_click_point(self.data_coordinates[e + 1, i + 1]) + dest_pt = self.get_click_point(self.data_coordinates[e, i + 1]) + self.move_tile(src_pt, dest_pt) + return True + if e + 2 <= 7: + if state[e, i] == color and state[e + 2, i] == color: + # third upper + if e + 1 >= 0 and i + 1 <= 13: + #if state[e - 1, i - 1] == color and (state[e, i - 1] >= 1 and state[e, i - 1] <= 5): + if state[e + 1, i + 1] == color and (state[e + 1, i] in self.colors): + src_pt = self.get_click_point(self.data_coordinates[e + 1, i + 1]) + dest_pt = self.get_click_point(self.data_coordinates[e + 1, i]) + self.move_tile(src_pt, dest_pt) + return True + # third left lower + if e + 1 <= 7 and i - 1 >= 0: + if state[e + 1, i - 1] == color and (state[e + 1, i] in self.colors): + src_pt = self.get_click_point(self.data_coordinates[e + 1, i - 1]) + dest_pt = self.get_click_point(self.data_coordinates[e + 1, i]) + self.move_tile(src_pt, dest_pt) + return True + except: + return False + + def check_3_horizontal(self, state, e, i, color): + try: + # second color next to starting point + if state[e, i] == color and state[e, i + 1] == color: + # third left upper + if e - 1 >= 0 and i - 1 >= 0: + if state[e - 1, i - 1] == color and (state[e, i - 1] in self.colors): + print("upper left", color, e, i) + src_pt = self.get_click_point(self.data_coordinates[e - 1, i - 1]) + dest_pt = self.get_click_point(self.data_coordinates[e, i - 1]) + self.move_tile(src_pt, dest_pt) + return True + # third left lower + if e + 1 <= 7 and i - 1 >= 0: + if state[e + 1, i - 1] == color and (state[e, i - 1] in self.colors): + print("lower left", color, e, i) + src_pt = self.get_click_point(self.data_coordinates[e + 1, i - 1]) + dest_pt = self.get_click_point(self.data_coordinates[e, i - 1]) + self.move_tile(src_pt, dest_pt) + return True + # third left with gap + if i - 2 >= 0: + if state[e, i - 2] == color and (state[e, i - 1] in self.colors): + print("left gap ", color, e, i) + src_pt = self.get_click_point(self.data_coordinates[e, i - 2]) + dest_pt = self.get_click_point(self.data_coordinates[e, i - 1]) + self.move_tile(src_pt, dest_pt) + return True + # third right upper + if e - 1 >= 0 and i + 2 <= 13: + if state[e - 1, i + 2] == color and (state[e, i + 2] in self.colors): + print("upper right", color, e, i) + src_pt = self.get_click_point(self.data_coordinates[e - 1, i + 2]) + dest_pt = self.get_click_point(self.data_coordinates[e, i + 2]) + self.move_tile(src_pt, dest_pt) + return True + # third right lower + if e + 1 <= 7 and i + 2 <= 13: + if state[e + 1, i + 2] == color and (state[e, i + 2] in self.colors): + print("upper lower", color, e, i) + src_pt = self.get_click_point(self.data_coordinates[e + 1, i + 2]) + dest_pt = self.get_click_point(self.data_coordinates[e, i + 2]) + self.move_tile(src_pt, dest_pt) + return True + + # third right with gap + if i + 3 <= 13: + if state[e, i + 3] == color and (state[e, i + 2] in self.colors): + print("right gap ", color, e, i) + src_pt = self.get_click_point(self.data_coordinates[e, i + 3]) + dest_pt = self.get_click_point(self.data_coordinates[e, i + 2]) + self.move_tile(src_pt, dest_pt) + return True + except: + return False + + def check_3_vertical(self, state, e, i, color): + try: + # second color next to starting point o + if state[e, i] == color and state[e + 1, i] == color: + # third left upper + if e - 1 >= 0 and i - 1 >= 0: + if state[e - 1, i - 1] == color and (state[e - 1, i] in self.colors): + print("upper left", color, e, i) + src_pt = self.get_click_point(self.data_coordinates[e - 1, i - 1]) + dest_pt = self.get_click_point(self.data_coordinates[e - 1, i]) + self.move_tile(src_pt, dest_pt) + return True + # third left lower + if e + 2 <= 7 and i - 1 >= 0: + if state[e + 2, i - 1] == color and (state[e + 2, i] in self.colors): + print("lower left", color, e, i) + src_pt = self.get_click_point(self.data_coordinates[e + 2, i - 1]) + dest_pt = self.get_click_point(self.data_coordinates[e + 2, i]) + self.move_tile(src_pt, dest_pt) + return True + # third right upper + if e - 1 >= 0 and i + 1 <= 13: + if state[e - 1, i + 1] == color and (state[e - 1, i] in self.colors): + print("upper right", color, e, i) + src_pt = self.get_click_point(self.data_coordinates[e - 1, i + 1]) + dest_pt = self.get_click_point(self.data_coordinates[e - 1, i]) + self.move_tile(src_pt, dest_pt) + return True + # third right lower + if e + 2 <= 7 and i + 1 <= 13: + if state[e + 2, i + 1] == color and (state[e + 2, i] in self.colors): + print("upper lower", color, e, i) + src_pt = self.get_click_point(self.data_coordinates[e + 2, i + 1]) + dest_pt = self.get_click_point(self.data_coordinates[e + 2, i]) + self.move_tile(src_pt, dest_pt) + return True + # third upper with gap + if e - 2 >= 0: + if state[e - 2, i] == color and (state[e - 1, i] in self.colors): + print("upper gap ", color, e, i) + src_pt = self.get_click_point(self.data_coordinates[e - 2, i]) + dest_pt = self.get_click_point(self.data_coordinates[e - 1, i]) + self.move_tile(src_pt, dest_pt) + return True + # third lower with gap + if e + 3 <= 7: + if state[e + 3, i] == color and (state[e + 2, i] in self.colors): + print("lower gap ", color, e, i) + src_pt = self.get_click_point(self.data_coordinates[e + 3, i]) + dest_pt = self.get_click_point(self.data_coordinates[e + 2, i]) + self.move_tile(src_pt, dest_pt) + return True + except: + return False + + def move_tile(self, point_source, point_dest): + offset_left = 230 + offset_down = 58 + pydirectinput.moveTo(point_source[0] + offset_left, point_source[1] + offset_down) + # pydirectinput.moveTo(0,0) + pydirectinput.mouseDown() + w = random.randint(25, 50) + cv.waitKey(100 + w) + pydirectinput.moveTo(point_dest[0] + offset_left, point_dest[1] + offset_down) + pydirectinput.mouseUp() + cv.waitKey(400 + w) + + def check_for_button_and_click_it(self, button_url): + screenshot = self.capture_window.get_screenshot() + # gray = cv.cvtColor(screenshot, cv.COLOR_BGR2GRAY) + # thresh = cv.threshold(gray, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1] + # gray_needle = cv.cvtColor(cv.imread(button_url, cv.IMREAD_UNCHANGED), cv.COLOR_BGR2GRAY) + # thresh_needle = cv.threshold(gray_needle, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1] + needle = cv.imread(button_url, cv.IMREAD_UNCHANGED) + # rectangles = self.vision_stun.find(thresh, thresh_needle, 0.4, 1) + rectangles = self.vision_stun.find(screenshot, needle, 0.7, 1) + + if len(rectangles) == 1: + pointis = self.vision_stun.get_click_points(rectangles) + for pointi in pointis: + self.dig_point(pointi[0], pointi[1], 150) + + def dig_point(self, point1, point2, dig_time): + pydirectinput.moveTo(point1, point2) + cv.waitKey(dig_time) + pydirectinput.mouseDown() + w = random.randint(50, 100) + cv.waitKey(w) + pydirectinput.mouseUp() + + def get_click_point(self, rectangle): + # Loop over all the rectangles + x, y, w, h = rectangle + # Determine the center position + center_x = x + int(w / 2) + center_y = y + int(h / 2) + # Save the points + return int(center_x), int(center_y)