473 lines
21 KiB
Python
473 lines
21 KiB
Python
import random
|
|
import cv2 as cv
|
|
import numpy as np
|
|
import pydirectinput
|
|
from window_capture import WindowCapture
|
|
from vision import Vision
|
|
from config_file import UserConfigs
|
|
from utils import mse
|
|
|
|
GREEN = 1
|
|
YELLOW = 2
|
|
BLUE = 3
|
|
RED = 4
|
|
PINK = 5
|
|
RAINBOW = 6
|
|
BIGBOMB = 7
|
|
BOMB = 8
|
|
ARROW_DOWN = 9
|
|
ARROW_RIGHT = 10
|
|
ROCK_1 = 11
|
|
ROCK_2 = 12
|
|
ROCK_3 = 13
|
|
|
|
class Field:
|
|
data_value_grid = []
|
|
data_coordinates = []
|
|
next_level = cv.imread("next_level.jpg", cv.IMREAD_COLOR)
|
|
next_level_x = cv.imread("next_level_x.jpg", cv.IMREAD_COLOR)
|
|
explosives = []
|
|
explosives.append(RAINBOW)
|
|
explosives.append(ARROW_RIGHT)
|
|
explosives.append(ARROW_DOWN)
|
|
explosives.append(BIGBOMB)
|
|
explosives.append(BOMB)
|
|
|
|
def __init__(self):
|
|
self.data_value_grid = np.zeros((8, 14), dtype=int)
|
|
self.data_coordinates = np.zeros((8, 14), dtype=object)
|
|
|
|
# 230 to 2110 = 1883 / 14 = 134.5
|
|
# 60 to 1130 = 1076 / 8 = 134.5
|
|
dim = 134.5
|
|
for e in range(0, 8, 1):
|
|
for i in range(0, 14, 1):
|
|
self.data_coordinates[e][i] = [i * dim, e * dim, dim, dim]
|
|
|
|
# initialize the user-class
|
|
self.config = UserConfigs()
|
|
|
|
# initialize the StunWindowCapture class
|
|
self.capture_window = WindowCapture(None, None, self.config)
|
|
# initialize the StunVision class
|
|
self.vision_stun = Vision()
|
|
|
|
self.needles = {GREEN: cv.imread("green.jpg", cv.IMREAD_COLOR),
|
|
YELLOW: cv.imread("yellow.jpg", cv.IMREAD_COLOR),
|
|
BLUE: cv.imread("blue.jpg", cv.IMREAD_COLOR),
|
|
RED: cv.imread("red.jpg", cv.IMREAD_COLOR),
|
|
PINK: cv.imread("pink.jpg", cv.IMREAD_COLOR),
|
|
RAINBOW: cv.imread("rainbow.jpg", cv.IMREAD_COLOR),
|
|
BIGBOMB: cv.imread("bigbomb.jpg", cv.IMREAD_COLOR),
|
|
BOMB: cv.imread("bomb.jpg", cv.IMREAD_COLOR),
|
|
ARROW_DOWN: cv.imread("arrow_down.jpg", cv.IMREAD_COLOR),
|
|
ARROW_RIGHT: cv.imread("arrow_right.jpg", cv.IMREAD_COLOR),
|
|
ROCK_1: cv.imread("rock1.jpg", cv.IMREAD_COLOR),
|
|
ROCK_2: cv.imread("rock2.jpg", cv.IMREAD_COLOR),
|
|
ROCK_3: cv.imread("rock3.jpg", cv.IMREAD_COLOR)
|
|
}
|
|
|
|
def reset(self):
|
|
self.episode_step = 0
|
|
self.last_reward = 0
|
|
self.last_score = 0
|
|
self.kill_counter = 0
|
|
# hit reset button and confirm
|
|
# self.check_for_button_and_click_it("needles/repeat.jpg")
|
|
# self.check_for_button_and_click_it("needles/reset.jpg")
|
|
|
|
self.dig_point(1800, 160, 600)
|
|
self.dig_point(1800, 1000, 300)
|
|
self.observation, screen = self.get_current_board_state()
|
|
return self.observation
|
|
|
|
def shift_playfield(self, action):
|
|
self.episode_step += 1
|
|
# move to indicated direction
|
|
self.action(action)
|
|
# get new field status
|
|
new_observation, new_screenshot = self.get_current_board_state()
|
|
current_score = 0
|
|
reward = 0
|
|
|
|
# wrong movement detection
|
|
# last board state is same as actual
|
|
if mse(new_observation, self.observation) == 0.0:
|
|
# no movement detected -> punish
|
|
if len(new_observation[new_observation == 0]) >= 1:
|
|
reward = -100
|
|
else:
|
|
self.kill_counter = self.kill_counter + 1
|
|
reward = -5
|
|
else:
|
|
# calculate current board score
|
|
self.kill_counter = 0
|
|
for e in range(0, 4, 1):
|
|
for i in range(0, 4, 1):
|
|
current_score = current_score + (2 ** new_observation[e][i] - 1)
|
|
bonus_for_empty_cells = len(new_observation[new_observation == 0])
|
|
reward = current_score - self.last_score + bonus_for_empty_cells
|
|
self.last_score = current_score
|
|
|
|
if self.kill_counter >= 5:
|
|
self.kill_counter = 0
|
|
done = True
|
|
else:
|
|
done = False
|
|
self.observation = new_observation
|
|
return new_observation, reward, done
|
|
|
|
def action(self, choice):
|
|
'''
|
|
Gives us 4 total movement options. (0,1,2,3)
|
|
'''
|
|
if choice == 0:
|
|
# right
|
|
self.move_to(1200, 598)
|
|
elif choice == 1:
|
|
# left
|
|
self.move_to(1000, 598)
|
|
elif choice == 2:
|
|
# up
|
|
self.move_to(1113, 498)
|
|
elif choice == 3:
|
|
# down
|
|
self.move_to(1113, 698)
|
|
|
|
def move_to(self, x, y):
|
|
point_src = (1113, 598)
|
|
pydirectinput.moveTo(point_src[0], point_src[1])
|
|
pydirectinput.mouseDown()
|
|
w = random.randint(1, 100)
|
|
cv.waitKey(150 + w)
|
|
pydirectinput.moveTo(x, y)
|
|
pydirectinput.mouseUp()
|
|
cv.waitKey(500 + w)
|
|
|
|
def change_value(self, x, y, val):
|
|
self.data_value_grid[x][y] = val
|
|
|
|
def pointInRect(self, point):
|
|
for e in range(0, 8, 1):
|
|
for i in range(0, 14, 1):
|
|
x1, y1, w, h = self.data_coordinates[e][i]
|
|
x2, y2 = x1 + w, y1 + h
|
|
x, y = point
|
|
if (x1 < x and x < x2):
|
|
if (y1 < y and y < y2):
|
|
return e, i
|
|
return None, None
|
|
|
|
def check_for_next_level(self, screen, needle):
|
|
offset_left = 230
|
|
offset_down = 58
|
|
rectangles = self.vision_stun.find(screen, needle, 0.70, 1)
|
|
if len(rectangles) == 0:
|
|
return False
|
|
point = self.vision_stun.get_click_points(rectangles)[0]
|
|
self.dig_point(point[0]+ offset_left, point[1] + offset_down, 500)
|
|
return True
|
|
|
|
def get_current_board_state(self):
|
|
try:
|
|
# get an updated image of the game
|
|
#screenshot = self.capture_window.get_screenshot()
|
|
screenshot = cv.imread("field_farm.jpg")
|
|
screenshot = screenshot[58:1134, 230:2113] # 1883,1076
|
|
# gray = cv.cvtColor(screenshot, cv.COLOR_BGR2GRAY)
|
|
# thresh = cv.threshold(gray, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1]
|
|
if self.check_for_next_level(screenshot, self.next_level):
|
|
cv.waitKey(500)
|
|
screenshot = self.capture_window.get_screenshot()
|
|
screenshot = screenshot[58:1134, 230:2113]
|
|
if self.check_for_next_level(screenshot, self.next_level_x):
|
|
cv.waitKey(500)
|
|
screenshot = self.capture_window.get_screenshot()
|
|
screenshot = screenshot[58:1134, 230:2113]
|
|
|
|
except:
|
|
self.capture_window.release()
|
|
print("Game window not available - shutting down application")
|
|
return None
|
|
# cv.imshow("screenshot", screenshot)
|
|
# cv.waitKey(150)
|
|
# continue
|
|
data_coords = np.zeros((8, 14), dtype=object)
|
|
# field = Field()
|
|
for needle_key in self.needles.keys():
|
|
# gray_needle = cv.cvtColor(self.needles[needle_key], cv.COLOR_BGR2GRAY)
|
|
# thresh_needle = cv.threshold(gray_needle, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1]
|
|
rectangles = self.vision_stun.find(screenshot, self.needles[needle_key], 0.70, 56)
|
|
if len(rectangles) == 0:
|
|
continue
|
|
points = self.vision_stun.get_click_points(rectangles)
|
|
|
|
for point in points:
|
|
x, y = self.pointInRect(point)
|
|
if x is not None and y is not None:
|
|
data_coords[x][y] = int(needle_key)
|
|
# self.change_value(x, y, int(needle_key))
|
|
# print(field.data_value_grid)
|
|
# cv.circle(screenshot, points[0], 7, (0, 255, 0), -1)
|
|
# output_image = vision_stun.draw_rectangles(screenshot, rectangles)
|
|
# cv.imshow("output_image", output_image)
|
|
# cv.waitKey(150)
|
|
return data_coords, screenshot
|
|
|
|
def analyse_boardstate(self, state):
|
|
for e in range(0, 8, 1):
|
|
for i in range(0, 14, 1):
|
|
if self.check_explosives(state, e, i):
|
|
return
|
|
|
|
for e in range(0, 8, 1):
|
|
for i in range(0, 14, 1):
|
|
for color in range(1, 6, 1):
|
|
if self.check_5_horizontal(state, e, i, color):
|
|
return
|
|
if self.check_5_vertical(state, e, i, color):
|
|
return
|
|
|
|
for e in range(0, 8, 1):
|
|
for i in range(0, 14, 1):
|
|
for color in range(1, 6, 1):
|
|
if self.check_3_horizontal(state, e, i, color):
|
|
return
|
|
if self.check_3_vertical(state, e, i, color):
|
|
return
|
|
|
|
def check_explosives(self, state, e, i):
|
|
for explosive in self.explosives:
|
|
if self.local_pos_check(state, e, i, 0, 0, explosive):
|
|
dest_pt = self.get_click_point(self.data_coordinates[e, i])
|
|
if self.local_pos_checks(state, e, i, 1, 0, self.explosives):
|
|
src_pt = self.get_click_point(self.data_coordinates[e + 1, i])
|
|
self.move_tile(src_pt, dest_pt)
|
|
elif self.local_pos_checks(state, e, i, 0, 1, self.explosives):
|
|
src_pt = self.get_click_point(self.data_coordinates[e, i + 1])
|
|
self.move_tile(src_pt, dest_pt)
|
|
elif self.local_pos_checks(state, e, i, -1, 0, self.explosives):
|
|
src_pt = self.get_click_point(self.data_coordinates[e - 1, i])
|
|
self.move_tile(src_pt, dest_pt)
|
|
elif self.local_pos_checks(state, e, i, 0, -1, self.explosives):
|
|
src_pt = self.get_click_point(self.data_coordinates[e, i - 1])
|
|
self.move_tile(src_pt, dest_pt)
|
|
else:
|
|
continue
|
|
return True
|
|
else:
|
|
continue
|
|
return False
|
|
|
|
def local_pos_check(self, state, e, i, e_check, i_check, needle):
|
|
if e + e_check >= 0 and e + e_check <=7 and i + i_check >= 0 and i + i_check <=13:
|
|
if state[e + e_check, i + i_check] == needle:
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
def local_pos_checks(self, state, e, i, e_check, i_check, needles):
|
|
if e + e_check >= 0 and e + e_check <=7 and i + i_check >= 0 and i + i_check <=13:
|
|
for needle in needles:
|
|
if state[e + e_check, i + i_check] == needle:
|
|
return True
|
|
else:
|
|
continue
|
|
return False
|
|
|
|
def check_5_horizontal(self, state, e, i, color):
|
|
try:
|
|
#
|
|
if state[e, i] == color and state[e, i + 1] == color and state[e, i + 3] == color and state[
|
|
e, i + 4] == color:
|
|
if state[e, i + 2] <= 0 or state[e, i + 2] >= 6:
|
|
return False
|
|
#
|
|
if e - 1 >= 0 and i + 2 <= 13:
|
|
if state[e - 1, i + 2] == color:
|
|
return True
|
|
return False
|
|
except:
|
|
return False
|
|
|
|
def check_5_vertical(self, state, e, i, color):
|
|
try:
|
|
#
|
|
if state[e, i] == color and state[e + 1, i] == color and state[e + 3, i] == color and state[
|
|
e + 4, i] == color:
|
|
if state[e + 2, i] <= 0 or state[e + 2, i] >= 6:
|
|
return False
|
|
# third left upper
|
|
if e + 2 <= 7 and i - 1 >= 0:
|
|
if state[e + 2, i - 1] == color:
|
|
print("upper left", color, e, i)
|
|
src_pt = self.get_click_point(self.data_coordinates[e + 2, i - 1])
|
|
dest_pt = self.get_click_point(self.data_coordinates[e + 2, i])
|
|
self.move_tile(src_pt, dest_pt)
|
|
return True
|
|
if e + 2 <= 7 and i + 1 <= 13:
|
|
if state[e + 2, i + 1] == color:
|
|
print("upper left", color, e, i)
|
|
src_pt = self.get_click_point(self.data_coordinates[e + 2, i + 1])
|
|
dest_pt = self.get_click_point(self.data_coordinates[e + 2, i])
|
|
self.move_tile(src_pt, dest_pt)
|
|
return True
|
|
return False
|
|
except:
|
|
return False
|
|
|
|
def check_3_horizontal(self, state, e, i, color):
|
|
try:
|
|
# second color next to starting point
|
|
if state[e, i] == color and state[e, i + 1] == color:
|
|
# third left upper
|
|
if e - 1 >= 0 and i - 1 >= 0:
|
|
if state[e - 1, i - 1] == color and (state[e, i - 1] >= 1 and state[e, i - 1] <= 5):
|
|
print("upper left", color, e, i)
|
|
src_pt = self.get_click_point(self.data_coordinates[e - 1, i - 1])
|
|
dest_pt = self.get_click_point(self.data_coordinates[e, i - 1])
|
|
self.move_tile(src_pt, dest_pt)
|
|
return True
|
|
# third left lower
|
|
if e + 1 <= 7 and i - 1 >= 0:
|
|
if state[e + 1, i - 1] == color and (state[e, i - 1] >= 1 and state[e, i - 1] <= 5):
|
|
print("lower left", color, e, i)
|
|
src_pt = self.get_click_point(self.data_coordinates[e + 1, i - 1])
|
|
dest_pt = self.get_click_point(self.data_coordinates[e, i - 1])
|
|
self.move_tile(src_pt, dest_pt)
|
|
return True
|
|
# third left with gap
|
|
if i - 2 >= 0:
|
|
if state[e, i - 2] == color and (state[e, i - 1] >= 1 and state[e, i - 1] <= 5):
|
|
print("left gap ", color, e, i)
|
|
src_pt = self.get_click_point(self.data_coordinates[e, i - 2])
|
|
dest_pt = self.get_click_point(self.data_coordinates[e, i - 1])
|
|
self.move_tile(src_pt, dest_pt)
|
|
return True
|
|
# third right upper
|
|
if e - 1 >= 0 and i + 2 <= 13:
|
|
if state[e - 1, i + 2] == color and (state[e, i + 2] >= 1 and state[e, i + 2] <= 5):
|
|
print("upper right", color, e, i)
|
|
src_pt = self.get_click_point(self.data_coordinates[e - 1, i + 2])
|
|
dest_pt = self.get_click_point(self.data_coordinates[e, i + 2])
|
|
self.move_tile(src_pt, dest_pt)
|
|
return True
|
|
# third right lower
|
|
if e + 1 <= 7 and i + 2 <= 13:
|
|
if state[e + 1, i + 2] == color and (state[e, i + 2] >= 1 and state[e, i + 2] <= 5):
|
|
print("upper lower", color, e, i)
|
|
src_pt = self.get_click_point(self.data_coordinates[e + 1, i + 2])
|
|
dest_pt = self.get_click_point(self.data_coordinates[e, i + 2])
|
|
self.move_tile(src_pt, dest_pt)
|
|
return True
|
|
|
|
# third right with gap
|
|
if i + 3 <= 13:
|
|
if state[e, i + 3] == color and (state[e, i + 2] >= 1 and state[e, i + 2] <= 5):
|
|
print("right gap ", color, e, i)
|
|
src_pt = self.get_click_point(self.data_coordinates[e, i + 3])
|
|
dest_pt = self.get_click_point(self.data_coordinates[e, i + 2])
|
|
self.move_tile(src_pt, dest_pt)
|
|
return True
|
|
except:
|
|
return False
|
|
|
|
def check_3_vertical(self, state, e, i, color):
|
|
try:
|
|
# second color next to starting point o
|
|
if state[e, i] == color and state[e + 1, i] == color:
|
|
# third left upper
|
|
if e - 1 >= 0 and i - 1 >= 0:
|
|
if state[e - 1, i - 1] == color and (state[e - 1, i] >= 1 and state[e - 1, i] <= 5):
|
|
print("upper left", color, e, i)
|
|
src_pt = self.get_click_point(self.data_coordinates[e - 1, i - 1])
|
|
dest_pt = self.get_click_point(self.data_coordinates[e - 1, i])
|
|
self.move_tile(src_pt, dest_pt)
|
|
return True
|
|
# third left lower
|
|
if e + 2 <= 7 and i - 1 >= 0:
|
|
if state[e + 2, i - 1] == color and (state[e + 2, i] >= 1 and state[e + 2, i] <= 5):
|
|
print("lower left", color, e, i)
|
|
src_pt = self.get_click_point(self.data_coordinates[e + 2, i - 1])
|
|
dest_pt = self.get_click_point(self.data_coordinates[e + 2, i])
|
|
self.move_tile(src_pt, dest_pt)
|
|
return True
|
|
# third right upper
|
|
if e - 1 >= 0 and i + 1 <= 13:
|
|
if state[e - 1, i + 1] == color and (state[e - 1, i] >= 1 and state[e - 1, i] <= 5):
|
|
print("upper right", color, e, i)
|
|
src_pt = self.get_click_point(self.data_coordinates[e - 1, i + 1])
|
|
dest_pt = self.get_click_point(self.data_coordinates[e - 1, i])
|
|
self.move_tile(src_pt, dest_pt)
|
|
return True
|
|
# third right lower
|
|
if e + 2 <= 7 and i + 1 <= 13:
|
|
if state[e + 2, i + 1] == color and (state[e + 2, i] >= 1 and state[e + 2, i] <= 5):
|
|
print("upper lower", color, e, i)
|
|
src_pt = self.get_click_point(self.data_coordinates[e + 2, i + 1])
|
|
dest_pt = self.get_click_point(self.data_coordinates[e + 2, i])
|
|
self.move_tile(src_pt, dest_pt)
|
|
return True
|
|
# third upper with gap
|
|
if e - 2 >= 0:
|
|
if state[e - 2, i] == color and (state[e - 1, i] >= 1 and state[e - 1, i] <= 5):
|
|
print("upper gap ", color, e, i)
|
|
src_pt = self.get_click_point(self.data_coordinates[e - 2, i])
|
|
dest_pt = self.get_click_point(self.data_coordinates[e - 1, i])
|
|
self.move_tile(src_pt, dest_pt)
|
|
return True
|
|
# third lower with gap
|
|
if e + 3 <= 7:
|
|
if state[e + 3, i] == color and (state[e + 2, i] >= 1 and state[e + 2, i] <= 5):
|
|
print("lower gap ", color, e, i)
|
|
src_pt = self.get_click_point(self.data_coordinates[e + 3, i])
|
|
dest_pt = self.get_click_point(self.data_coordinates[e + 2, i])
|
|
self.move_tile(src_pt, dest_pt)
|
|
return True
|
|
except:
|
|
return False
|
|
|
|
def move_tile(self, point_source, point_dest):
|
|
offset_left = 230
|
|
offset_down = 58
|
|
pydirectinput.moveTo(point_source[0] + offset_left, point_source[1] + offset_down)
|
|
# pydirectinput.moveTo(0,0)
|
|
pydirectinput.mouseDown()
|
|
w = random.randint(25, 50)
|
|
cv.waitKey(100 + w)
|
|
pydirectinput.moveTo(point_dest[0] + offset_left, point_dest[1] + offset_down)
|
|
pydirectinput.mouseUp()
|
|
cv.waitKey(400 + w)
|
|
|
|
def check_for_button_and_click_it(self, button_url):
|
|
screenshot = self.capture_window.get_screenshot()
|
|
# gray = cv.cvtColor(screenshot, cv.COLOR_BGR2GRAY)
|
|
# thresh = cv.threshold(gray, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1]
|
|
# gray_needle = cv.cvtColor(cv.imread(button_url, cv.IMREAD_UNCHANGED), cv.COLOR_BGR2GRAY)
|
|
# thresh_needle = cv.threshold(gray_needle, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU)[1]
|
|
needle = cv.imread(button_url, cv.IMREAD_UNCHANGED)
|
|
# rectangles = self.vision_stun.find(thresh, thresh_needle, 0.4, 1)
|
|
rectangles = self.vision_stun.find(screenshot, needle, 0.7, 1)
|
|
|
|
if len(rectangles) == 1:
|
|
pointis = self.vision_stun.get_click_points(rectangles)
|
|
for pointi in pointis:
|
|
self.dig_point(pointi[0], pointi[1], 150)
|
|
|
|
def dig_point(self, point1, point2, dig_time):
|
|
pydirectinput.moveTo(point1, point2)
|
|
cv.waitKey(dig_time)
|
|
pydirectinput.mouseDown()
|
|
w = random.randint(50, 100)
|
|
cv.waitKey(w)
|
|
pydirectinput.mouseUp()
|
|
|
|
def get_click_point(self, rectangle):
|
|
# Loop over all the rectangles
|
|
x, y, w, h = rectangle
|
|
# Determine the center position
|
|
center_x = x + int(w / 2)
|
|
center_y = y + int(h / 2)
|
|
# Save the points
|
|
return int(center_x), int(center_y)
|