From 4ffd873578c1164d60dfd5aab58829226f2f44f2 Mon Sep 17 00:00:00 2001 From: Thaloria Date: Mon, 16 May 2022 14:48:23 +0200 Subject: [PATCH] re-implement equipment with masks --- equipment_main.py | 84 ++++++++++++++++++++++++++--------------------- vision.py | 61 +++++++++++++++++++++++++++------- 2 files changed, 96 insertions(+), 49 deletions(-) diff --git a/equipment_main.py b/equipment_main.py index 95fe580..c2426c9 100644 --- a/equipment_main.py +++ b/equipment_main.py @@ -18,9 +18,10 @@ EMITTER_RING = "ring" HSV_DEFAULT = HsvFilter(0, 0, 0, 179, 255, 255, 0, 0, 0, 0) + def run(): EMITTER_TO_USE = EMITTER_RING - SPAWN_COUNT = 1 + SPAWN_COUNT = 15 # initialize the user-class config = UserConfigs() @@ -28,13 +29,11 @@ def run(): try: capture_window = WindowCapture( None, "equip", config) - video_mode = False except: # StunWindowCapture.list_window_names() - # print("Game not running, switching to video mode") + print("Game not running, exiting") # capture_window = cv.VideoCapture("snip_slam.mp4") - video_mode = True - + return # initialize the StunVision class vision_stun = Vision() @@ -72,9 +71,9 @@ def run(): include_staffs(needles, hsvs, tresholds, masks) include_bags(needles, hsvs, tresholds, masks) elif EMITTER_TO_USE == EMITTER_RING: - #include_books(needles, hsvs, tresholds, masks) + include_books(needles, hsvs, tresholds, masks) include_rings(needles, hsvs, tresholds, masks) - #include_bags(needles, hsvs, tresholds, masks) + include_bags(needles, hsvs, tresholds, masks) elif EMITTER_TO_USE == EMITTER_WAND: include_books(needles, hsvs, tresholds, masks) include_wands(needles, hsvs, tresholds, masks) @@ -93,21 +92,19 @@ def run(): print("pausing") continue - if video_mode: + try: + # get an updated image of the game + screenshot = capture_window.get_screenshot() + # screenshot = cv.imread("buffbar.jpg") + except: + capture_window.release() + print("Game window not available - shutting down application") break - else: - try: - # get an updated image of the game - screenshot = capture_window.get_screenshot() - # screenshot = cv.imread("buffbar.jpg") - except: - capture_window.release() - print("Game window not available - shutting down application") - break # cv.imshow("screenshot", screenshot) # cv.waitKey(150) # continue + spawn_0_location = [] spawn_1 = find_emitter(EMITTER_TO_USE, vision_stun, screenshot, 1) if len(spawn_1) == 1: spawn_button_active = True @@ -132,14 +129,14 @@ def run(): while True: # do object detection screenshot = capture_window.get_screenshot() - #processed_screenshot = vision_stun.apply_hsv_filter(screenshot, hsvs[rer]) - #processed_needle = vision_stun.apply_hsv_filter(needles[rer], hsvs[rer]) + # processed_screenshot = vision_stun.apply_hsv_filter(screenshot, hsvs[rer]) + # processed_needle = vision_stun.apply_hsv_filter(needles[rer], hsvs[rer]) - rectangles = vision_stun.find(screenshot, needles[rer], tresholds[rer], 5, True, masks[rer]) + rectangles = vision_stun.find_by_mask_and_validate(screenshot, needles[rer], masks[rer], 5) # draw the detection results onto the original image - output_image = vision_stun.draw_rectangles(screenshot, rectangles) - cv.imshow("output_image", output_image) - cv.waitKey(150) + #output_image = vision_stun.draw_rectangles(screenshot, rectangles) + #cv.imshow("output_image", output_image) + #cv.waitKey(150) if len(rectangles) is not 5: break @@ -192,6 +189,7 @@ def include_chests(needles, hsv, tresh, mask): tresh.append(0.91) mask.append(cv.imread("equip/chests/chest_3_32-mask.png", cv.IMREAD_COLOR)) + def include_books(needles, hsv, tresh, mask): needles.append(cv.imread("equip/books/book_1_32.jpg", cv.IMREAD_UNCHANGED)) hsv.append(HSV_DEFAULT) @@ -234,7 +232,6 @@ def include_books(needles, hsv, tresh, mask): mask.append(cv.imread("equip/books/book_8_32-mask.png", cv.IMREAD_COLOR)) - def include_keys(needles, hsv, tresh, mask): needles.append(cv.imread("equip/keys/key_1_32.jpg", cv.IMREAD_UNCHANGED)) hsv.append(HSV_DEFAULT) @@ -320,6 +317,7 @@ def include_bags(needles, hsv, tresh, mask): tresh.append(0.91) mask.append(cv.imread("equip/bags/bag_7_32-mask.png", cv.IMREAD_COLOR)) + def include_coins(needles, hsv, tresh, mask): needles.append(cv.imread("equip/coins/coin_1_32.jpg", cv.IMREAD_UNCHANGED)) hsv.append(HSV_DEFAULT) @@ -346,6 +344,7 @@ def include_coins(needles, hsv, tresh, mask): tresh.append(0.91) mask.append(cv.imread("equip/coins/coin_5_32-mask.png", cv.IMREAD_COLOR)) + def include_runes(needles, hsv, tresh, mask): needles.append(cv.imread("equip/runes/rune_1_32.jpg", cv.IMREAD_UNCHANGED)) hsv.append(HSV_DEFAULT) @@ -394,6 +393,7 @@ def include_mushs(needles, hsv, tresh, mask): tresh.append(0.93) mask.append(cv.imread("equip/mushrooms/mush_7_32-mask.png", cv.IMREAD_COLOR)) + def include_amus(needles, hsv, tresh, mask): needles.append(cv.imread("equip/amus/amu_1_32.jpg", cv.IMREAD_UNCHANGED)) hsv.append(HSV_DEFAULT) @@ -457,6 +457,7 @@ def include_swords(needles, hsv, tresh, mask): tresh.append(0.93) mask.append(cv.imread("equip/swords/sword_5_32-mask.png", cv.IMREAD_COLOR)) + def include_staffs(needles, hsv, tresh, mask): needles.append(cv.imread("equip/staffs/staff_1_32.jpg", cv.IMREAD_UNCHANGED)) hsv.append(HSV_DEFAULT) @@ -478,10 +479,11 @@ def include_staffs(needles, hsv, tresh, mask): tresh.append(0.93) mask.append(cv.imread("equip/staffs/staff_4_32-mask.png", cv.IMREAD_COLOR)) + def include_rings(needles, hsv, tresh, mask): needles.append(cv.imread("equip/rings/ring_1_32.jpg", cv.IMREAD_UNCHANGED)) hsv.append(HSV_DEFAULT) - tresh.append(0.92) + tresh.append(0.925) mask.append(cv.imread("equip/rings/ring_1_32-mask.png", cv.IMREAD_COLOR)) needles.append(cv.imread("equip/rings/ring_2_32.jpg", cv.IMREAD_UNCHANGED)) @@ -504,6 +506,7 @@ def include_rings(needles, hsv, tresh, mask): tresh.append(0.93) mask.append(cv.imread("equip/rings/ring_5_32-mask.png", cv.IMREAD_COLOR)) + def include_wands(needles, hsv, tresh, mask): needles.append(cv.imread("equip/wands/wand_1_32.jpg", cv.IMREAD_UNCHANGED)) hsv.append(HSV_DEFAULT) @@ -525,6 +528,7 @@ def include_wands(needles, hsv, tresh, mask): tresh.append(0.93) mask.append(cv.imread("equip/wands/wand_4_32-mask.png", cv.IMREAD_COLOR)) + def move_tile(conf, point_source, point_dest): offset_left = conf.returnEquipmentWindowPos()[2] offset_down = conf.returnEquipmentWindowPos()[3] @@ -602,21 +606,25 @@ def find_emitter(emitter_to_use, vis, screen, layer): mask = cv.imread("equip/emitters/wand_e2_32-mask.png", cv.IMREAD_COLOR) return vis.find(screen, needle, 0.96, 1, True, mask) + def check_and_move_tile(capture_win, visio, conf, rect, needl, hsv, tresh, mask, point_source, point_dest): - screenshot_pos = capture_win.get_screenshot_by_area(rect) - #processed_screenshot = visio.apply_hsv_filter(screenshot_pos, hsv) - #processed_needle = visio.apply_hsv_filter(needl, hsv) - rectangles2 = visio.find(screenshot_pos, needl, tresh, 1, True, mask) + screenshot_pos = capture_win.get_screenshot() + w = rect[0] + h = rect[1] + x = rect[2] + y = rect[3] + screenshot_pos = screenshot_pos[y:y+h, x:x+w] # (w, h, x+w, y+h) + # processed_screenshot = visio.apply_hsv_filter(screenshot_pos, hsv) + # processed_needle = visio.apply_hsv_filter(needl, hsv) + result2 = cv.matchTemplate(screenshot_pos, needl, cv.TM_CCOEFF_NORMED) + _minVal2, _maxVal2, minLoc2, maxLoc2 = cv.minMaxLoc(result2, None) + # rectangles2 = visio.find(screenshot_pos, needl, tresh, 1, True, mask) # output_by_area = vision_stun.draw_rectangles(screenshot_pos2, rectangles) - # cv.imshow("output_image_by_area", output_by_area) - # cv.waitKey(150) - if len(rectangles2) == 1: - # pos 2 filled - return - else: - # pos 2 vacant - pass - move_tile(conf, point_source, point_dest) + #cv.imshow("output_image_by_area", screenshot_pos) + #cv.waitKey(150) + print("thresh in movecheck:" + str(_maxVal2)) + if _maxVal2 <= 0.9: + move_tile(conf, point_source, point_dest) def click_point(conf, x, y): diff --git a/vision.py b/vision.py index 97c956f..217f8b8 100644 --- a/vision.py +++ b/vision.py @@ -27,32 +27,71 @@ class Vision: # TM_CCOEFF, TM_CCOEFF_NORMED, TM_CCORR, TM_CCORR_NORMED, TM_SQDIFF, TM_SQDIFF_NORMED self.method = method - def find(self, haystack_img, needle_img, threshold=0.5, max_results=10, normalize=False, mask=None): + + + def find_by_mask_and_validate(self, haystack_img, needle_img, needle_mask, max_results=5): # run the OpenCV algorithm needle_w = needle_img.shape[1] needle_h = needle_img.shape[0] - if normalize: - result = cv.matchTemplate(haystack_img, needle_img, cv.TM_CCORR_NORMED, None, mask) - _minVal, _maxVal, minLoc, maxLoc = cv.minMaxLoc(result, None) - cv.normalize(result, result, 0, 1, cv.NORM_MINMAX, -1) - else: - result = cv.matchTemplate(haystack_img, needle_img, self.method) + result = cv.matchTemplate(haystack_img, needle_img, cv.TM_CCORR_NORMED, None, needle_mask) + cv.normalize(result, result, 0, 1, cv.NORM_MINMAX, -1) + + find_num = 20 + idx_1d = np.argpartition(result.flatten(), -find_num)[-find_num:] + idx_2d = np.unravel_index(idx_1d, result.shape) + + rectangles = [] + for i in range(0, len(idx_2d[0]), 1): + y = int(idx_2d[0][i]) + x = int(idx_2d[1][i]) + rect = [x, y, needle_w, needle_h] + # Add every box to the list twice in order to retain single (non-overlapping) boxes + rectangles.append(rect) + rectangles.append(rect) + + rectangles, weights = cv.groupRectangles(rectangles, groupThreshold=1, eps=0.5) + + keep_rects = [] + for rect in rectangles: + w = rect[0] + h = rect[1] + x = rect[2] + w + y = rect[3] + h + screenshot_pos = haystack_img[h:y, w:x] # (w, h, x+w, y+h) + result2 = cv.matchTemplate(screenshot_pos, needle_img, cv.TM_CCOEFF_NORMED) + _minVal2, _maxVal2, minLoc2, maxLoc2 = cv.minMaxLoc(result2, None) + #screenshot_pos_img = self.draw_rectangles(screenshot_pos, rectangles) + #cv.imshow("screenshot_pos", screenshot_pos) + #cv.waitKey(150) + if _maxVal2 >= 0.9: + keep_rects.append(rect) + + if len(keep_rects) > max_results: + keep_rects = keep_rects[:max_results] + + return keep_rects + + def find(self, haystack_img, needle_img, threshold=0.5, max_results=10): + # run the OpenCV algorithm + needle_w = needle_img.shape[1] + needle_h = needle_img.shape[0] + + result = cv.matchTemplate(haystack_img, needle_img, self.method) # Get the all the positions from the match result that exceed our threshold locations = np.where(result >= threshold) locations = list(zip(*locations[::-1])) # print(locations) - _minVal, _maxVal, minLoc, maxLoc = cv.minMaxLoc(result, None) + + + #_minVal, _maxVal, minLoc, maxLoc = cv.minMaxLoc(result, None) # if we found no results, return now. this reshape of the empty array allows us to # concatenate together results without causing an error if not locations: return np.array([], dtype=np.int32).reshape(0, 4) - if len(locations) > 5000: - return np.array([], dtype=np.int32).reshape(0, 4) - # You'll notice a lot of overlapping rectangles get drawn. We can eliminate those redundant # locations by using groupRectangles(). # First we need to create the list of [x, y, w, h] rectangles