diff --git a/breed1.jpg b/breed1.jpg new file mode 100644 index 0000000..6fc6e58 Binary files /dev/null and b/breed1.jpg differ diff --git a/breed2.jpg b/breed2.jpg new file mode 100644 index 0000000..ba1f903 Binary files /dev/null and b/breed2.jpg differ diff --git a/equip/amus/amu_1_32-removebg-preview.png b/equip/amus/amu_1_32-removebg-preview.png new file mode 100644 index 0000000..509d927 Binary files /dev/null and b/equip/amus/amu_1_32-removebg-preview.png differ diff --git a/farm/Field_Representation.py b/farm/Field_Representation.py new file mode 100644 index 0000000..791305f --- /dev/null +++ b/farm/Field_Representation.py @@ -0,0 +1,472 @@ +import random +import cv2 as cv +import numpy as np +import pydirectinput +from window_capture import WindowCapture +from vision import Vision +from config_file import UserConfigs +from utils import mse + +GREEN = 1 +YELLOW = 2 +BLUE = 3 +RED = 4 +PINK = 5 +RAINBOW = 6 +BIGBOMB = 7 +BOMB = 8 +ARROW_DOWN = 9 +ARROW_RIGHT = 10 +ROCK_1 = 11 +ROCK_2 = 12 +ROCK_3 = 13 + +class Field: + data_value_grid = [] + data_coordinates = [] + next_level = cv.imread("next_level.jpg", cv.IMREAD_COLOR) + next_level_x = cv.imread("next_level_x.jpg", cv.IMREAD_COLOR) + explosives = [] + explosives.append(RAINBOW) + explosives.append(ARROW_RIGHT) + explosives.append(ARROW_DOWN) + explosives.append(BIGBOMB) + explosives.append(BOMB) + + def __init__(self): + self.data_value_grid = np.zeros((8, 14), dtype=int) + self.data_coordinates = np.zeros((8, 14), dtype=object) + + # 230 to 2110 = 1883 / 14 = 134.5 + # 60 to 1130 = 1076 / 8 = 134.5 + dim = 134.5 + for e in range(0, 8, 1): + for i in range(0, 14, 1): + self.data_coordinates[e][i] = [i * dim, e * dim, dim, dim] + + # initialize the user-class + self.config = UserConfigs() + + # initialize the StunWindowCapture class + self.capture_window = WindowCapture(None, None, self.config) + # initialize the StunVision class + self.vision_stun = Vision() + + self.needles = {GREEN: cv.imread("green.jpg", cv.IMREAD_COLOR), + YELLOW: cv.imread("yellow.jpg", cv.IMREAD_COLOR), + BLUE: cv.imread("blue.jpg", cv.IMREAD_COLOR), + RED: cv.imread("red.jpg", cv.IMREAD_COLOR), + PINK: cv.imread("pink.jpg", cv.IMREAD_COLOR), + RAINBOW: cv.imread("rainbow.jpg", cv.IMREAD_COLOR), + BIGBOMB: cv.imread("bigbomb.jpg", cv.IMREAD_COLOR), + BOMB: cv.imread("bomb.jpg", cv.IMREAD_COLOR), + ARROW_DOWN: cv.imread("arrow_down.jpg", cv.IMREAD_COLOR), + ARROW_RIGHT: cv.imread("arrow_right.jpg", cv.IMREAD_COLOR), + ROCK_1: cv.imread("rock1.jpg", cv.IMREAD_COLOR), + ROCK_2: cv.imread("rock2.jpg", cv.IMREAD_COLOR), + ROCK_3: cv.imread("rock3.jpg", cv.IMREAD_COLOR) + } + + def reset(self): + self.episode_step = 0 + self.last_reward = 0 + self.last_score = 0 + self.kill_counter = 0 + # hit reset button and confirm + # self.check_for_button_and_click_it("needles/repeat.jpg") + # self.check_for_button_and_click_it("needles/reset.jpg") + + self.dig_point(1800, 160, 600) + self.dig_point(1800, 1000, 300) + self.observation, screen = self.get_current_board_state() + return self.observation + + def shift_playfield(self, action): + self.episode_step += 1 + # move to indicated direction + self.action(action) + # get new field status + new_observation, new_screenshot = self.get_current_board_state() + current_score = 0 + reward = 0 + + # wrong movement detection + # last board state is same as actual + if mse(new_observation, self.observation) == 0.0: + # no movement detected -> punish + if len(new_observation[new_observation == 0]) >= 1: + reward = -100 + else: + self.kill_counter = self.kill_counter + 1 + reward = -5 + else: + # calculate current board score + self.kill_counter = 0 + for e in range(0, 4, 1): + for i in range(0, 4, 1): + current_score = current_score + (2 ** new_observation[e][i] - 1) + bonus_for_empty_cells = len(new_observation[new_observation == 0]) + reward = current_score - self.last_score + bonus_for_empty_cells + self.last_score = current_score + + if self.kill_counter >= 5: + self.kill_counter = 0 + done = True + else: + done = False + self.observation = new_observation + return new_observation, reward, done + + def action(self, choice): + ''' + Gives us 4 total movement options. (0,1,2,3) + ''' + if choice == 0: + # right + self.move_to(1200, 598) + elif choice == 1: + # left + self.move_to(1000, 598) + elif choice == 2: + # up + self.move_to(1113, 498) + elif choice == 3: + # down + self.move_to(1113, 698) + + def move_to(self, x, y): + point_src = (1113, 598) + pydirectinput.moveTo(point_src[0], point_src[1]) + pydirectinput.mouseDown() + w = random.randint(1, 100) + cv.waitKey(150 + w) + pydirectinput.moveTo(x, y) + pydirectinput.mouseUp() + cv.waitKey(500 + w) + + def change_value(self, x, y, val): + self.data_value_grid[x][y] = val + + def pointInRect(self, point): + for e in range(0, 8, 1): + for i in range(0, 14, 1): + x1, y1, w, h = self.data_coordinates[e][i] + x2, y2 = x1 + w, y1 + h + x, y = point + if (x1 < x and x < x2): + if (y1 < y and y < y2): + return e, i + return None, None + + def check_for_next_level(self, screen, needle): + offset_left = 230 + offset_down = 58 + rectangles = self.vision_stun.find(screen, needle, 0.70, 1) + if len(rectangles) == 0: + return False + point = self.vision_stun.get_click_points(rectangles)[0] + self.dig_point(point[0]+ offset_left, point[1] + offset_down, 500) + return True + + def get_current_board_state(self): + try: + # get an updated image of the game + #screenshot = self.capture_window.get_screenshot() + screenshot = cv.imread("field_farm.jpg") + screenshot = screenshot[58:1134, 230:2113] # 1883,1076 + # gray = cv.cvtColor(screenshot, cv.COLOR_BGR2GRAY) + # thresh = cv.threshold(gray, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1] + if self.check_for_next_level(screenshot, self.next_level): + cv.waitKey(500) + screenshot = self.capture_window.get_screenshot() + screenshot = screenshot[58:1134, 230:2113] + if self.check_for_next_level(screenshot, self.next_level_x): + cv.waitKey(500) + screenshot = self.capture_window.get_screenshot() + screenshot = screenshot[58:1134, 230:2113] + + except: + self.capture_window.release() + print("Game window not available - shutting down application") + return None + # cv.imshow("screenshot", screenshot) + # cv.waitKey(150) + # continue + data_coords = np.zeros((8, 14), dtype=object) + # field = Field() + for needle_key in self.needles.keys(): + # gray_needle = cv.cvtColor(self.needles[needle_key], cv.COLOR_BGR2GRAY) + # thresh_needle = cv.threshold(gray_needle, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1] + rectangles = self.vision_stun.find(screenshot, self.needles[needle_key], 0.70, 56) + if len(rectangles) == 0: + continue + points = self.vision_stun.get_click_points(rectangles) + + for point in points: + x, y = self.pointInRect(point) + if x is not None and y is not None: + data_coords[x][y] = int(needle_key) + # self.change_value(x, y, int(needle_key)) + # print(field.data_value_grid) + # cv.circle(screenshot, points[0], 7, (0, 255, 0), -1) + # output_image = vision_stun.draw_rectangles(screenshot, rectangles) + # cv.imshow("output_image", output_image) + # cv.waitKey(150) + return data_coords, screenshot + + def analyse_boardstate(self, state): + for e in range(0, 8, 1): + for i in range(0, 14, 1): + if self.check_explosives(state, e, i): + return + + for e in range(0, 8, 1): + for i in range(0, 14, 1): + for color in range(1, 6, 1): + if self.check_5_horizontal(state, e, i, color): + return + if self.check_5_vertical(state, e, i, color): + return + + for e in range(0, 8, 1): + for i in range(0, 14, 1): + for color in range(1, 6, 1): + if self.check_3_horizontal(state, e, i, color): + return + if self.check_3_vertical(state, e, i, color): + return + + def check_explosives(self, state, e, i): + for explosive in self.explosives: + if self.local_pos_check(state, e, i, 0, 0, explosive): + dest_pt = self.get_click_point(self.data_coordinates[e, i]) + if self.local_pos_checks(state, e, i, 1, 0, self.explosives): + src_pt = self.get_click_point(self.data_coordinates[e + 1, i]) + self.move_tile(src_pt, dest_pt) + elif self.local_pos_checks(state, e, i, 0, 1, self.explosives): + src_pt = self.get_click_point(self.data_coordinates[e, i + 1]) + self.move_tile(src_pt, dest_pt) + elif self.local_pos_checks(state, e, i, -1, 0, self.explosives): + src_pt = self.get_click_point(self.data_coordinates[e - 1, i]) + self.move_tile(src_pt, dest_pt) + elif self.local_pos_checks(state, e, i, 0, -1, self.explosives): + src_pt = self.get_click_point(self.data_coordinates[e, i - 1]) + self.move_tile(src_pt, dest_pt) + else: + continue + return True + else: + continue + return False + + def local_pos_check(self, state, e, i, e_check, i_check, needle): + if e + e_check >= 0 and e + e_check <=7 and i + i_check >= 0 and i + i_check <=13: + if state[e + e_check, i + i_check] == needle: + return True + else: + return False + + def local_pos_checks(self, state, e, i, e_check, i_check, needles): + if e + e_check >= 0 and e + e_check <=7 and i + i_check >= 0 and i + i_check <=13: + for needle in needles: + if state[e + e_check, i + i_check] == needle: + return True + else: + continue + return False + + def check_5_horizontal(self, state, e, i, color): + try: + # + if state[e, i] == color and state[e, i + 1] == color and state[e, i + 3] == color and state[ + e, i + 4] == color: + if state[e, i + 2] <= 0 or state[e, i + 2] >= 6: + return False + # + if e - 1 >= 0 and i + 2 <= 13: + if state[e - 1, i + 2] == color: + return True + return False + except: + return False + + def check_5_vertical(self, state, e, i, color): + try: + # + if state[e, i] == color and state[e + 1, i] == color and state[e + 3, i] == color and state[ + e + 4, i] == color: + if state[e + 2, i] <= 0 or state[e + 2, i] >= 6: + return False + # third left upper + if e + 2 <= 7 and i - 1 >= 0: + if state[e + 2, i - 1] == color: + print("upper left", color, e, i) + src_pt = self.get_click_point(self.data_coordinates[e + 2, i - 1]) + dest_pt = self.get_click_point(self.data_coordinates[e + 2, i]) + self.move_tile(src_pt, dest_pt) + return True + if e + 2 <= 7 and i + 1 <= 13: + if state[e + 2, i + 1] == color: + print("upper left", color, e, i) + src_pt = self.get_click_point(self.data_coordinates[e + 2, i + 1]) + dest_pt = self.get_click_point(self.data_coordinates[e + 2, i]) + self.move_tile(src_pt, dest_pt) + return True + return False + except: + return False + + def check_3_horizontal(self, state, e, i, color): + try: + # second color next to starting point + if state[e, i] == color and state[e, i + 1] == color: + # third left upper + if e - 1 >= 0 and i - 1 >= 0: + if state[e - 1, i - 1] == color and (state[e, i - 1] >= 1 and state[e, i - 1] <= 5): + print("upper left", color, e, i) + src_pt = self.get_click_point(self.data_coordinates[e - 1, i - 1]) + dest_pt = self.get_click_point(self.data_coordinates[e, i - 1]) + self.move_tile(src_pt, dest_pt) + return True + # third left lower + if e + 1 <= 7 and i - 1 >= 0: + if state[e + 1, i - 1] == color and (state[e, i - 1] >= 1 and state[e, i - 1] <= 5): + print("lower left", color, e, i) + src_pt = self.get_click_point(self.data_coordinates[e + 1, i - 1]) + dest_pt = self.get_click_point(self.data_coordinates[e, i - 1]) + self.move_tile(src_pt, dest_pt) + return True + # third left with gap + if i - 2 >= 0: + if state[e, i - 2] == color and (state[e, i - 1] >= 1 and state[e, i - 1] <= 5): + print("left gap ", color, e, i) + src_pt = self.get_click_point(self.data_coordinates[e, i - 2]) + dest_pt = self.get_click_point(self.data_coordinates[e, i - 1]) + self.move_tile(src_pt, dest_pt) + return True + # third right upper + if e - 1 >= 0 and i + 2 <= 13: + if state[e - 1, i + 2] == color and (state[e, i + 2] >= 1 and state[e, i + 2] <= 5): + print("upper right", color, e, i) + src_pt = self.get_click_point(self.data_coordinates[e - 1, i + 2]) + dest_pt = self.get_click_point(self.data_coordinates[e, i + 2]) + self.move_tile(src_pt, dest_pt) + return True + # third right lower + if e + 1 <= 7 and i + 2 <= 13: + if state[e + 1, i + 2] == color and (state[e, i + 2] >= 1 and state[e, i + 2] <= 5): + print("upper lower", color, e, i) + src_pt = self.get_click_point(self.data_coordinates[e + 1, i + 2]) + dest_pt = self.get_click_point(self.data_coordinates[e, i + 2]) + self.move_tile(src_pt, dest_pt) + return True + + # third right with gap + if i + 3 <= 13: + if state[e, i + 3] == color and (state[e, i + 2] >= 1 and state[e, i + 2] <= 5): + print("right gap ", color, e, i) + src_pt = self.get_click_point(self.data_coordinates[e, i + 3]) + dest_pt = self.get_click_point(self.data_coordinates[e, i + 2]) + self.move_tile(src_pt, dest_pt) + return True + except: + return False + + def check_3_vertical(self, state, e, i, color): + try: + # second color next to starting point o + if state[e, i] == color and state[e + 1, i] == color: + # third left upper + if e - 1 >= 0 and i - 1 >= 0: + if state[e - 1, i - 1] == color and (state[e - 1, i] >= 1 and state[e - 1, i] <= 5): + print("upper left", color, e, i) + src_pt = self.get_click_point(self.data_coordinates[e - 1, i - 1]) + dest_pt = self.get_click_point(self.data_coordinates[e - 1, i]) + self.move_tile(src_pt, dest_pt) + return True + # third left lower + if e + 2 <= 7 and i - 1 >= 0: + if state[e + 2, i - 1] == color and (state[e + 2, i] >= 1 and state[e + 2, i] <= 5): + print("lower left", color, e, i) + src_pt = self.get_click_point(self.data_coordinates[e + 2, i - 1]) + dest_pt = self.get_click_point(self.data_coordinates[e + 2, i]) + self.move_tile(src_pt, dest_pt) + return True + # third right upper + if e - 1 >= 0 and i + 1 <= 13: + if state[e - 1, i + 1] == color and (state[e - 1, i] >= 1 and state[e - 1, i] <= 5): + print("upper right", color, e, i) + src_pt = self.get_click_point(self.data_coordinates[e - 1, i + 1]) + dest_pt = self.get_click_point(self.data_coordinates[e - 1, i]) + self.move_tile(src_pt, dest_pt) + return True + # third right lower + if e + 2 <= 7 and i + 1 <= 13: + if state[e + 2, i + 1] == color and (state[e + 2, i] >= 1 and state[e + 2, i] <= 5): + print("upper lower", color, e, i) + src_pt = self.get_click_point(self.data_coordinates[e + 2, i + 1]) + dest_pt = self.get_click_point(self.data_coordinates[e + 2, i]) + self.move_tile(src_pt, dest_pt) + return True + # third upper with gap + if e - 2 >= 0: + if state[e - 2, i] == color and (state[e - 1, i] >= 1 and state[e - 1, i] <= 5): + print("upper gap ", color, e, i) + src_pt = self.get_click_point(self.data_coordinates[e - 2, i]) + dest_pt = self.get_click_point(self.data_coordinates[e - 1, i]) + self.move_tile(src_pt, dest_pt) + return True + # third lower with gap + if e + 3 <= 7: + if state[e + 3, i] == color and (state[e + 2, i] >= 1 and state[e + 2, i] <= 5): + print("lower gap ", color, e, i) + src_pt = self.get_click_point(self.data_coordinates[e + 3, i]) + dest_pt = self.get_click_point(self.data_coordinates[e + 2, i]) + self.move_tile(src_pt, dest_pt) + return True + except: + return False + + def move_tile(self, point_source, point_dest): + offset_left = 230 + offset_down = 58 + pydirectinput.moveTo(point_source[0] + offset_left, point_source[1] + offset_down) + # pydirectinput.moveTo(0,0) + pydirectinput.mouseDown() + w = random.randint(25, 50) + cv.waitKey(100 + w) + pydirectinput.moveTo(point_dest[0] + offset_left, point_dest[1] + offset_down) + pydirectinput.mouseUp() + cv.waitKey(400 + w) + + def check_for_button_and_click_it(self, button_url): + screenshot = self.capture_window.get_screenshot() + # gray = cv.cvtColor(screenshot, cv.COLOR_BGR2GRAY) + # thresh = cv.threshold(gray, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1] + # gray_needle = cv.cvtColor(cv.imread(button_url, cv.IMREAD_UNCHANGED), cv.COLOR_BGR2GRAY) + # thresh_needle = cv.threshold(gray_needle, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1] + needle = cv.imread(button_url, cv.IMREAD_UNCHANGED) + # rectangles = self.vision_stun.find(thresh, thresh_needle, 0.4, 1) + rectangles = self.vision_stun.find(screenshot, needle, 0.7, 1) + + if len(rectangles) == 1: + pointis = self.vision_stun.get_click_points(rectangles) + for pointi in pointis: + self.dig_point(pointi[0], pointi[1], 150) + + def dig_point(self, point1, point2, dig_time): + pydirectinput.moveTo(point1, point2) + cv.waitKey(dig_time) + pydirectinput.mouseDown() + w = random.randint(50, 100) + cv.waitKey(w) + pydirectinput.mouseUp() + + def get_click_point(self, rectangle): + # Loop over all the rectangles + x, y, w, h = rectangle + # Determine the center position + center_x = x + int(w / 2) + center_y = y + int(h / 2) + # Save the points + return int(center_x), int(center_y) diff --git a/farm/arrow_down.jpg b/farm/arrow_down.jpg new file mode 100644 index 0000000..e9f60d6 Binary files /dev/null and b/farm/arrow_down.jpg differ diff --git a/farm/arrow_right.jpg b/farm/arrow_right.jpg new file mode 100644 index 0000000..9757744 Binary files /dev/null and b/farm/arrow_right.jpg differ diff --git a/farm/bigbomb.jpg b/farm/bigbomb.jpg new file mode 100644 index 0000000..e336b41 Binary files /dev/null and b/farm/bigbomb.jpg differ diff --git a/farm/blue.jpg b/farm/blue.jpg new file mode 100644 index 0000000..112e828 Binary files /dev/null and b/farm/blue.jpg differ diff --git a/farm/bomb.jpg b/farm/bomb.jpg new file mode 100644 index 0000000..29bb142 Binary files /dev/null and b/farm/bomb.jpg differ diff --git a/farm/box.jpg b/farm/box.jpg new file mode 100644 index 0000000..df77d6b Binary files /dev/null and b/farm/box.jpg differ diff --git a/farm/field_complete.jpg b/farm/field_complete.jpg new file mode 100644 index 0000000..077152a Binary files /dev/null and b/farm/field_complete.jpg differ diff --git a/farm/field_farm.jpg b/farm/field_farm.jpg new file mode 100644 index 0000000..b7f0ba9 Binary files /dev/null and b/farm/field_farm.jpg differ diff --git a/farm/field_farm2.jpg b/farm/field_farm2.jpg new file mode 100644 index 0000000..ac60163 Binary files /dev/null and b/farm/field_farm2.jpg differ diff --git a/farm/green.jpg b/farm/green.jpg new file mode 100644 index 0000000..727d446 Binary files /dev/null and b/farm/green.jpg differ diff --git a/farm/main_farm.py b/farm/main_farm.py new file mode 100644 index 0000000..75087a6 --- /dev/null +++ b/farm/main_farm.py @@ -0,0 +1,39 @@ +from Field_Representation import Field +from time import time +import cv2 as cv +import keyboard + +def run(): + field = Field() + loop_time_p = time() + pause = True + + while True: + if keyboard.is_pressed('p') == True: + pause = True + print('q pressed') + elif keyboard.is_pressed('o') == True: + pause = False + print('o pressed') + + if pause: + # cv.waitKey(500) + if (time() - loop_time_p) >= 5: + loop_time_p = time() + print("pausing") + continue + + + cords, screenshot = field.get_current_board_state() + field.analyse_boardstate(cords) + print(cords) + cv.waitKey(1000) + + + #cv.imshow("screenshot", screenshot) + #cv.waitKey(150) + #print(cords) + + +if __name__ == "__main__": + run() \ No newline at end of file diff --git a/farm/next_level.jpg b/farm/next_level.jpg new file mode 100644 index 0000000..e81710c Binary files /dev/null and b/farm/next_level.jpg differ diff --git a/farm/next_level_x.jpg b/farm/next_level_x.jpg new file mode 100644 index 0000000..3dc0ad9 Binary files /dev/null and b/farm/next_level_x.jpg differ diff --git a/farm/pink.jpg b/farm/pink.jpg new file mode 100644 index 0000000..8e6fa02 Binary files /dev/null and b/farm/pink.jpg differ diff --git a/farm/rainbow.jpg b/farm/rainbow.jpg new file mode 100644 index 0000000..d52e9d9 Binary files /dev/null and b/farm/rainbow.jpg differ diff --git a/farm/red.jpg b/farm/red.jpg new file mode 100644 index 0000000..9cc841a Binary files /dev/null and b/farm/red.jpg differ diff --git a/farm/rock1.jpg b/farm/rock1.jpg new file mode 100644 index 0000000..c292365 Binary files /dev/null and b/farm/rock1.jpg differ diff --git a/farm/rock2.jpg b/farm/rock2.jpg new file mode 100644 index 0000000..9133f11 Binary files /dev/null and b/farm/rock2.jpg differ diff --git a/farm/rock3.jpg b/farm/rock3.jpg new file mode 100644 index 0000000..faaa0e2 Binary files /dev/null and b/farm/rock3.jpg differ diff --git a/farm/yellow.jpg b/farm/yellow.jpg new file mode 100644 index 0000000..788b13f Binary files /dev/null and b/farm/yellow.jpg differ