re-implement equipment with masks

create_mask_from_png.py

@@ -0,0 +1,23 @@
+import cv2 as cv
+import os
+import numpy as np
+from window_capture import WindowCapture
+from config_file import UserConfigs
+
+#uc = UserConfigs()
+#wc = WindowCapture(None, None, uc)
+#screen = wc.get_screenshot(False)
+# load the original input image and display it to our screen
+
+#filename = "equip/chests/chest_23_32"
+path = "equip/wands/"
+os.chdir(path)
+
+for entry in os.listdir():
+    if entry.endswith('.png'):
+
+        nelk = entry[:len(entry)-4]
+        if nelk.find('-mask') == -1:
+            out = nelk + "-mask.png"
+            img = cv.imread(entry, cv.IMREAD_UNCHANGED)
+            cv.imwrite(out, img[:, :, 3])

cv_test_2.py

@@ -0,0 +1,172 @@
+from __future__ import print_function
+import sys
+import cv2 as cv
+import numpy as np
+from hsvfilter import HsvFilter
+
+use_mask = False
+img = None
+templ = None
+mask = None
+image_window = "Source Image"
+result_window = "Result window"
+match_method = 3
+max_Trackbar = 5
+
+
+def main():
+
+
+    global img
+    global templ
+    img = cv.imread("equip/rings/test_screen.jpg", cv.IMREAD_COLOR)
+    templ = cv.imread("equip/rings/ring_1_32.jpg", cv.IMREAD_COLOR)
+
+    hsv = HsvFilter(13, 40, 85, 135, 255, 255, 0, 0, 55, 53)
+    #img = apply_hsv_filter(img, hsv)
+    #templ = apply_hsv_filter(templ, hsv)
+
+
+    global use_mask
+    use_mask = True
+    global mask
+    mask = cv.imread("equip/rings/ring_1_32-mask.png", cv.IMREAD_COLOR)
+
+
+    cv.namedWindow(image_window, cv.WINDOW_AUTOSIZE)
+    cv.namedWindow(result_window, cv.WINDOW_AUTOSIZE)
+
+    trackbar_label = 'Method: \n 0: SQDIFF \n 1: SQDIFF NORMED \n 2: TM CCORR \n 3: TM CCORR NORMED \n 4: TM COEFF \n 5: TM COEFF NORMED'
+    cv.createTrackbar(trackbar_label, image_window, match_method, max_Trackbar, MatchingMethod)
+
+    MatchingMethod(match_method)
+
+    cv.waitKey(0)
+    return 0
+
+def draw_rectangles(haystack_img, rectangles):
+    # these colors are actually BGR
+    line_color = (0, 255, 0)
+    line_type = cv.LINE_4
+    pic = None
+    for (x, y, w, h) in rectangles:
+        # determine the box positions
+        top_left = (x, y)
+        bottom_right = (x + w, y + h)
+        # draw the box
+        cv.rectangle(haystack_img, top_left, bottom_right, line_color, lineType=line_type)
+
+        #pic = haystack_img[y:y + h, x:x + w]
+
+    return haystack_img
+
+def shift_channel(c, amount):
+    if amount > 0:
+        lim = 255 - amount
+        c[c >= lim] = 255
+        c[c < lim] += amount
+    elif amount < 0:
+        amount = -amount
+        lim = amount
+        c[c <= lim] = 0
+        c[c > lim] -= amount
+    return c
+
+def apply_hsv_filter(original_image, hsv_filter):
+    # convert image to HSV
+    hsv = cv.cvtColor(original_image, cv.COLOR_BGR2HSV)
+
+
+    # add/subtract saturation and value
+    h, s, v = cv.split(hsv)
+    s = shift_channel(s, hsv_filter.sAdd)
+    s = shift_channel(s, -hsv_filter.sSub)
+    v = shift_channel(v, hsv_filter.vAdd)
+    v = shift_channel(v, -hsv_filter.vSub)
+    hsv = cv.merge([h, s, v])
+
+    # Set minimum and maximum HSV values to display
+    lower = np.array([hsv_filter.hMin, hsv_filter.sMin, hsv_filter.vMin])
+    upper = np.array([hsv_filter.hMax, hsv_filter.sMax, hsv_filter.vMax])
+    # Apply the thresholds
+    mask = cv.inRange(hsv, lower, upper)
+    result = cv.bitwise_and(hsv, hsv, mask=mask)
+
+    # convert back to BGR for imshow() to display it properly
+    img = cv.cvtColor(result, cv.COLOR_HSV2BGR)
+
+    return img
+
+def MatchingMethod(param):
+    global match_method
+    match_method = param
+
+    img_display = img.copy()
+
+    method_accepts_mask = (cv.TM_SQDIFF == match_method or match_method == cv.TM_CCORR_NORMED)
+    if (use_mask and method_accepts_mask):
+        result = cv.matchTemplate(img, templ, match_method, None, mask)
+    else:
+        result = cv.matchTemplate(img, templ, match_method)
+    #_minVal, _maxVal, minLoc, maxLoc = cv.minMaxLoc(result, None)
+    cv.normalize(result, result, 0, 1, cv.NORM_MINMAX, -1)
+    #_minVal, _maxVal, minLoc, maxLoc = cv.minMaxLoc(result, None)
+
+    locations = np.where(result >= 0.91)
+    locations = list(zip(*locations[::-1]))
+
+    needle_w = templ.shape[1]
+    needle_h = templ.shape[0]
+
+    find_num = 20
+    idx_1d = np.argpartition(result.flatten(), -find_num)[-find_num:]
+    #new_res = result.flatten()[idx_1d]
+    #new_res.append()
+    idx_2d = np.unravel_index(idx_1d, result.shape)
+
+    rectangles = []
+    for i in range(0, len(idx_2d[0]), 1):
+        y = int(idx_2d[0][i])
+        x = int(idx_2d[1][i])
+        rect = [x, y, needle_w, needle_h]
+        # Add every box to the list twice in order to retain single (non-overlapping) boxes
+        rectangles.append(rect)
+        rectangles.append(rect)
+
+
+    for loc in locations:
+        rect = [int(loc[0]), int(loc[1]), needle_w, needle_h]
+        # Add every box to the list twice in order to retain single (non-overlapping) boxes
+        #rectangles.append(rect)
+        #rectangles.append(rect)
+
+
+    rectangles, weights = cv.groupRectangles(rectangles, groupThreshold=1, eps=0.5)
+    keep_rects = []
+    for rect in rectangles:
+        w = rect[0]
+        h = rect[1]
+        x = rect[2] + w
+        y = rect[3] + h
+        screenshot_pos = img_display[h:y, w:x]  # (w, h, x+w, y+h)
+        result2 = cv.matchTemplate(screenshot_pos, templ, 5)
+        _minVal2, _maxVal2, minLoc2, maxLoc2 = cv.minMaxLoc(result2, None)
+        if _maxVal2 >= 0.5:
+            keep_rects.append(rect)
+            print("matching error")
+
+
+
+
+
+
+    res1 = draw_rectangles(img_display, keep_rects)
+    res2 = draw_rectangles(img, keep_rects)
+
+    cv.imshow(image_window, res1)
+    cv.imshow(result_window, res2)
+
+
+
+if __name__ == "__main__":
+    main()

equipment_main.py

@@ -18,9 +18,10 @@ EMITTER_RING = "ring"
 
 HSV_DEFAULT = HsvFilter(0, 0, 0, 179, 255, 255, 0, 0, 0, 0)
 
+
 def run():
     EMITTER_TO_USE = EMITTER_RING
-    SPAWN_COUNT = 1
+    SPAWN_COUNT = 15
     # initialize the user-class
     config = UserConfigs()
 
@@ -28,13 +29,11 @@ def run():
     try:
         capture_window = WindowCapture(
             None, "equip", config)
-        video_mode = False
     except:
         # StunWindowCapture.list_window_names()
-        # print("Game not running, switching to video mode")
+        print("Game not running, exiting")
         # capture_window = cv.VideoCapture("snip_slam.mp4")
-        video_mode = True
-
+        return
 
     # initialize the StunVision class
     vision_stun = Vision()
@@ -72,9 +71,9 @@ def run():
         include_staffs(needles, hsvs, tresholds, masks)
         include_bags(needles, hsvs, tresholds, masks)
     elif EMITTER_TO_USE == EMITTER_RING:
-        #include_books(needles, hsvs, tresholds, masks)
+        include_books(needles, hsvs, tresholds, masks)
         include_rings(needles, hsvs, tresholds, masks)
-        #include_bags(needles, hsvs, tresholds, masks)
+        include_bags(needles, hsvs, tresholds, masks)
     elif EMITTER_TO_USE == EMITTER_WAND:
         include_books(needles, hsvs, tresholds, masks)
         include_wands(needles, hsvs, tresholds, masks)
@@ -93,21 +92,19 @@ def run():
             print("pausing")
             continue
 
-        if video_mode:
+        try:
+            # get an updated image of the game
+            screenshot = capture_window.get_screenshot()
+            # screenshot = cv.imread("buffbar.jpg")
+        except:
+            capture_window.release()
+            print("Game window not available - shutting down application")
             break
-        else:
-            try:
-                # get an updated image of the game
-                screenshot = capture_window.get_screenshot()
-                # screenshot = cv.imread("buffbar.jpg")
-            except:
-                capture_window.release()
-                print("Game window not available - shutting down application")
-                break
         # cv.imshow("screenshot", screenshot)
         # cv.waitKey(150)
         # continue
 
+        spawn_0_location = []
         spawn_1 = find_emitter(EMITTER_TO_USE, vision_stun, screenshot, 1)
         if len(spawn_1) == 1:
             spawn_button_active = True
@@ -132,14 +129,14 @@ def run():
             while True:
                 # do object detection
                 screenshot = capture_window.get_screenshot()
-                #processed_screenshot = vision_stun.apply_hsv_filter(screenshot, hsvs[rer])
-                #processed_needle = vision_stun.apply_hsv_filter(needles[rer], hsvs[rer])
+                # processed_screenshot = vision_stun.apply_hsv_filter(screenshot, hsvs[rer])
+                # processed_needle = vision_stun.apply_hsv_filter(needles[rer], hsvs[rer])
 
-                rectangles = vision_stun.find(screenshot, needles[rer], tresholds[rer], 5, True, masks[rer])
+                rectangles = vision_stun.find_by_mask_and_validate(screenshot, needles[rer], masks[rer], 5)
                 # draw the detection results onto the original image
-                output_image = vision_stun.draw_rectangles(screenshot, rectangles)
-                cv.imshow("output_image", output_image)
-                cv.waitKey(150)
+                #output_image = vision_stun.draw_rectangles(screenshot, rectangles)
+                #cv.imshow("output_image", output_image)
+                #cv.waitKey(150)
 
                 if len(rectangles) is not 5:
                     break
@@ -192,6 +189,7 @@ def include_chests(needles, hsv, tresh, mask):
     tresh.append(0.91)
     mask.append(cv.imread("equip/chests/chest_3_32-mask.png", cv.IMREAD_COLOR))
 
+
 def include_books(needles, hsv, tresh, mask):
     needles.append(cv.imread("equip/books/book_1_32.jpg", cv.IMREAD_UNCHANGED))
     hsv.append(HSV_DEFAULT)
@@ -234,7 +232,6 @@ def include_books(needles, hsv, tresh, mask):
     mask.append(cv.imread("equip/books/book_8_32-mask.png", cv.IMREAD_COLOR))
 
 
-
 def include_keys(needles, hsv, tresh, mask):
     needles.append(cv.imread("equip/keys/key_1_32.jpg", cv.IMREAD_UNCHANGED))
     hsv.append(HSV_DEFAULT)
@@ -320,6 +317,7 @@ def include_bags(needles, hsv, tresh, mask):
     tresh.append(0.91)
     mask.append(cv.imread("equip/bags/bag_7_32-mask.png", cv.IMREAD_COLOR))
 
+
 def include_coins(needles, hsv, tresh, mask):
     needles.append(cv.imread("equip/coins/coin_1_32.jpg", cv.IMREAD_UNCHANGED))
     hsv.append(HSV_DEFAULT)
@@ -346,6 +344,7 @@ def include_coins(needles, hsv, tresh, mask):
     tresh.append(0.91)
     mask.append(cv.imread("equip/coins/coin_5_32-mask.png", cv.IMREAD_COLOR))
 
+
 def include_runes(needles, hsv, tresh, mask):
     needles.append(cv.imread("equip/runes/rune_1_32.jpg", cv.IMREAD_UNCHANGED))
     hsv.append(HSV_DEFAULT)
@@ -394,6 +393,7 @@ def include_mushs(needles, hsv, tresh, mask):
     tresh.append(0.93)
     mask.append(cv.imread("equip/mushrooms/mush_7_32-mask.png", cv.IMREAD_COLOR))
 
+
 def include_amus(needles, hsv, tresh, mask):
     needles.append(cv.imread("equip/amus/amu_1_32.jpg", cv.IMREAD_UNCHANGED))
     hsv.append(HSV_DEFAULT)
@@ -457,6 +457,7 @@ def include_swords(needles, hsv, tresh, mask):
     tresh.append(0.93)
     mask.append(cv.imread("equip/swords/sword_5_32-mask.png", cv.IMREAD_COLOR))
 
+
 def include_staffs(needles, hsv, tresh, mask):
     needles.append(cv.imread("equip/staffs/staff_1_32.jpg", cv.IMREAD_UNCHANGED))
     hsv.append(HSV_DEFAULT)
@@ -478,10 +479,11 @@ def include_staffs(needles, hsv, tresh, mask):
     tresh.append(0.93)
     mask.append(cv.imread("equip/staffs/staff_4_32-mask.png", cv.IMREAD_COLOR))
 
+
 def include_rings(needles, hsv, tresh, mask):
     needles.append(cv.imread("equip/rings/ring_1_32.jpg", cv.IMREAD_UNCHANGED))
     hsv.append(HSV_DEFAULT)
-    tresh.append(0.92)
+    tresh.append(0.925)
     mask.append(cv.imread("equip/rings/ring_1_32-mask.png", cv.IMREAD_COLOR))
 
     needles.append(cv.imread("equip/rings/ring_2_32.jpg", cv.IMREAD_UNCHANGED))
@@ -504,6 +506,7 @@ def include_rings(needles, hsv, tresh, mask):
     tresh.append(0.93)
     mask.append(cv.imread("equip/rings/ring_5_32-mask.png", cv.IMREAD_COLOR))
 
+
 def include_wands(needles, hsv, tresh, mask):
     needles.append(cv.imread("equip/wands/wand_1_32.jpg", cv.IMREAD_UNCHANGED))
     hsv.append(HSV_DEFAULT)
@@ -525,6 +528,7 @@ def include_wands(needles, hsv, tresh, mask):
     tresh.append(0.93)
     mask.append(cv.imread("equip/wands/wand_4_32-mask.png", cv.IMREAD_COLOR))
 
+
 def move_tile(conf, point_source, point_dest):
     offset_left = conf.returnEquipmentWindowPos()[2]
     offset_down = conf.returnEquipmentWindowPos()[3]
@@ -602,21 +606,25 @@ def find_emitter(emitter_to_use, vis, screen, layer):
             mask = cv.imread("equip/emitters/wand_e2_32-mask.png", cv.IMREAD_COLOR)
             return vis.find(screen, needle, 0.96, 1, True, mask)
 
+
 def check_and_move_tile(capture_win, visio, conf, rect, needl, hsv, tresh, mask, point_source, point_dest):
-    screenshot_pos = capture_win.get_screenshot_by_area(rect)
-    #processed_screenshot = visio.apply_hsv_filter(screenshot_pos, hsv)
-    #processed_needle = visio.apply_hsv_filter(needl, hsv)
-    rectangles2 = visio.find(screenshot_pos, needl, tresh, 1, True, mask)
+    screenshot_pos = capture_win.get_screenshot()
+    w = rect[0]
+    h = rect[1]
+    x = rect[2]
+    y = rect[3]
+    screenshot_pos = screenshot_pos[y:y+h, x:x+w]  # (w, h, x+w, y+h)
+    # processed_screenshot = visio.apply_hsv_filter(screenshot_pos, hsv)
+    # processed_needle = visio.apply_hsv_filter(needl, hsv)
+    result2 = cv.matchTemplate(screenshot_pos, needl, cv.TM_CCOEFF_NORMED)
+    _minVal2, _maxVal2, minLoc2, maxLoc2 = cv.minMaxLoc(result2, None)
+    # rectangles2 = visio.find(screenshot_pos, needl, tresh, 1, True, mask)
     # output_by_area = vision_stun.draw_rectangles(screenshot_pos2, rectangles)
-    # cv.imshow("output_image_by_area", output_by_area)
-    # cv.waitKey(150)
-    if len(rectangles2) == 1:
-        # pos 2 filled
-        return
-    else:
-        # pos 2 vacant
-        pass
-    move_tile(conf, point_source, point_dest)
+    #cv.imshow("output_image_by_area", screenshot_pos)
+    #cv.waitKey(150)
+    print("thresh in movecheck:" + str(_maxVal2))
+    if _maxVal2 <= 0.9:
+        move_tile(conf, point_source, point_dest)
 
 
 def click_point(conf, x, y):

vision.py

@@ -27,32 +27,71 @@ class Vision:
         # TM_CCOEFF, TM_CCOEFF_NORMED, TM_CCORR, TM_CCORR_NORMED, TM_SQDIFF, TM_SQDIFF_NORMED
         self.method = method
 
-    def find(self, haystack_img, needle_img, threshold=0.5, max_results=10, normalize=False, mask=None):
+
+
+    def find_by_mask_and_validate(self, haystack_img, needle_img, needle_mask, max_results=5):
         # run the OpenCV algorithm
         needle_w = needle_img.shape[1]
         needle_h = needle_img.shape[0]
 
-        if normalize:
-            result = cv.matchTemplate(haystack_img, needle_img, cv.TM_CCORR_NORMED, None, mask)
-            _minVal, _maxVal, minLoc, maxLoc = cv.minMaxLoc(result, None)
-            cv.normalize(result, result, 0, 1, cv.NORM_MINMAX, -1)
-        else:
-            result = cv.matchTemplate(haystack_img, needle_img, self.method)
+        result = cv.matchTemplate(haystack_img, needle_img, cv.TM_CCORR_NORMED, None, needle_mask)
+        cv.normalize(result, result, 0, 1, cv.NORM_MINMAX, -1)
+
+        find_num = 20
+        idx_1d = np.argpartition(result.flatten(), -find_num)[-find_num:]
+        idx_2d = np.unravel_index(idx_1d, result.shape)
+
+        rectangles = []
+        for i in range(0, len(idx_2d[0]), 1):
+            y = int(idx_2d[0][i])
+            x = int(idx_2d[1][i])
+            rect = [x, y, needle_w, needle_h]
+            # Add every box to the list twice in order to retain single (non-overlapping) boxes
+            rectangles.append(rect)
+            rectangles.append(rect)
+
+        rectangles, weights = cv.groupRectangles(rectangles, groupThreshold=1, eps=0.5)
+
+        keep_rects = []
+        for rect in rectangles:
+            w = rect[0]
+            h = rect[1]
+            x = rect[2] + w
+            y = rect[3] + h
+            screenshot_pos = haystack_img[h:y, w:x]  # (w, h, x+w, y+h)
+            result2 = cv.matchTemplate(screenshot_pos, needle_img, cv.TM_CCOEFF_NORMED)
+            _minVal2, _maxVal2, minLoc2, maxLoc2 = cv.minMaxLoc(result2, None)
+            #screenshot_pos_img = self.draw_rectangles(screenshot_pos, rectangles)
+            #cv.imshow("screenshot_pos", screenshot_pos)
+            #cv.waitKey(150)
+            if _maxVal2 >= 0.9:
+                keep_rects.append(rect)
+
+        if len(keep_rects) > max_results:
+            keep_rects = keep_rects[:max_results]
+
+        return keep_rects
+
+    def find(self, haystack_img, needle_img, threshold=0.5, max_results=10):
+        # run the OpenCV algorithm
+        needle_w = needle_img.shape[1]
+        needle_h = needle_img.shape[0]
+
+        result = cv.matchTemplate(haystack_img, needle_img, self.method)
 
         # Get the all the positions from the match result that exceed our threshold
         locations = np.where(result >= threshold)
         locations = list(zip(*locations[::-1]))
         # print(locations)
-        _minVal, _maxVal, minLoc, maxLoc = cv.minMaxLoc(result, None)
+
+
+        #_minVal, _maxVal, minLoc, maxLoc = cv.minMaxLoc(result, None)
 
         # if we found no results, return now. this reshape of the empty array allows us to 
         # concatenate together results without causing an error
         if not locations:
             return np.array([], dtype=np.int32).reshape(0, 4)
 
-        if len(locations) > 5000:
-            return np.array([], dtype=np.int32).reshape(0, 4)
-
         # You'll notice a lot of overlapping rectangles get drawn. We can eliminate those redundant
         # locations by using groupRectangles().
         # First we need to create the list of [x, y, w, h] rectangles