diff --git a/runs/detect/加购_88/detect - 快捷方式.lnk b/runs/detect/加购_88/detect - 快捷方式.lnk new file mode 100644 index 0000000..eef2b9d Binary files /dev/null and b/runs/detect/加购_88/detect - 快捷方式.lnk differ diff --git a/track_reid.py b/track_reid.py index f55a638..9fdfc32 100644 --- a/track_reid.py +++ b/track_reid.py @@ -177,7 +177,7 @@ def run( save_dir = Path(project) / Path(source).stem if save_dir.exists(): print(Path(source).stem) - # return + return save_dir = increment_path(Path(project) / name, exist_ok=exist_ok) # increment run (save_dir / 'labels' if save_txt else save_dir).mkdir(parents=True, exist_ok=True) # make dir @@ -517,9 +517,10 @@ def main_loop(opt): optdict = vars(opt) # p = r"D:\datasets\ym\永辉测试数据_比对" - # p = r"D:\datasets\ym\广告板遮挡测试\8" - p = r"D:\datasets\ym\videos\标记视频" + p = r"D:\datasets\ym\广告板遮挡测试\8" + # p = r"D:\datasets\ym\videos\标记视频" # p = r"D:\datasets\ym\实验室测试" + # p = r"D:\datasets\ym\永辉双摄视频\新建文件夹" k = 0 if os.path.isdir(p): @@ -531,7 +532,7 @@ def main_loop(opt): # r"D:\datasets\ym\广告板遮挡测试\8\2500441577966_20240508-175946_front_addGood_70f75407b7ae_155_17788571404.mp4" # ] - files = [r"D:\datasets\ym\videos\标记视频\test_20240402-173935_6920152400975_back_174037372.mp4"] + # files = [r"D:\datasets\ym\videos\标记视频\test_20240402-173935_6920152400975_back_174037372.mp4"] for file in files: optdict["source"] = file run(**optdict) diff --git a/tracking/dotrack/__pycache__/dotracks.cpython-39.pyc b/tracking/dotrack/__pycache__/dotracks.cpython-39.pyc index e48e938..10c7a0a 100644 Binary files a/tracking/dotrack/__pycache__/dotracks.cpython-39.pyc and b/tracking/dotrack/__pycache__/dotracks.cpython-39.pyc differ diff --git a/tracking/dotrack/__pycache__/dotracks_back.cpython-39.pyc b/tracking/dotrack/__pycache__/dotracks_back.cpython-39.pyc index fe155c6..ff055b2 100644 Binary files a/tracking/dotrack/__pycache__/dotracks_back.cpython-39.pyc and b/tracking/dotrack/__pycache__/dotracks_back.cpython-39.pyc differ diff --git a/tracking/dotrack/__pycache__/dotracks_front.cpython-39.pyc b/tracking/dotrack/__pycache__/dotracks_front.cpython-39.pyc index 36c4ddd..917e917 100644 Binary files a/tracking/dotrack/__pycache__/dotracks_front.cpython-39.pyc and b/tracking/dotrack/__pycache__/dotracks_front.cpython-39.pyc differ diff --git a/tracking/dotrack/__pycache__/track_back.cpython-39.pyc b/tracking/dotrack/__pycache__/track_back.cpython-39.pyc index 2fa00a9..8d8b22b 100644 Binary files a/tracking/dotrack/__pycache__/track_back.cpython-39.pyc and b/tracking/dotrack/__pycache__/track_back.cpython-39.pyc differ diff --git a/tracking/dotrack/__pycache__/track_front.cpython-39.pyc b/tracking/dotrack/__pycache__/track_front.cpython-39.pyc index b990dd9..2f53f33 100644 Binary files a/tracking/dotrack/__pycache__/track_front.cpython-39.pyc and b/tracking/dotrack/__pycache__/track_front.cpython-39.pyc differ diff --git a/tracking/dotrack/dotracks.py b/tracking/dotrack/dotracks.py index a8d9802..843ad2f 100644 --- a/tracking/dotrack/dotracks.py +++ b/tracking/dotrack/dotracks.py @@ -8,7 +8,7 @@ import numpy as np import cv2 from pathlib import Path from scipy.spatial.distance import cdist -from utils.mergetrack import track_equal_track +from utils.mergetrack import track_equal_track, readDict curpath = Path(__file__).resolve().parents[0] curpath = Path(curpath) @@ -79,16 +79,18 @@ class ShoppingCart: class Track: '''抽象基类，不能实例化对象''' - def __init__(self, boxes, imgshape=(1024, 1280)): + def __init__(self, boxes, features, imgshape=(1024, 1280)): ''' boxes: [x1, y1, x2, y2, track_id, score, cls, frame_index, box_index] 0 1 2 3 4 5 6 7 8 ''' # 不满足以下条件时会如何？ - assert len(set(boxes[:, 4].astype(int))) == 1, "For a Track, track_id more than 1" - assert len(set(boxes[:, 6].astype(int))) == 1, "For a Track, class number more than 1" + # assert len(set(boxes[:, 4].astype(int))) == 1, "For a Track, track_id more than 1" + # assert len(set(boxes[:, 6].astype(int))) == 1, "For a Track, class number more than 1" self.boxes = boxes + self.features = features + self.tid = int(boxes[0, 4]) self.cls = int(boxes[0, 6]) self.frnum = boxes.shape[0] @@ -100,6 +102,9 @@ class Track: self.start_fid = int(np.min(boxes[:, 7])) self.end_fid = int(np.max(boxes[:, 7])) + self.Hands = [] + self.HandsIou = [] + '''5个关键点（中心点、左上点、右上点、左下点、右下点 ）坐标''' @@ -212,7 +217,7 @@ class Track: self.trajmin = trajmin self.trajmax = trajmax - self.feature = [trajlen_min, trajlen_max, + self.TrajFeat = [trajlen_min, trajlen_max, trajdist_min, trajdist_max, trajlen_rate, trajdist_rate] @@ -268,6 +273,53 @@ class Track: return ranges, rangex + def PositionState(self, camerType="back"): + ''' + camerType: back, 后置摄像头 + front, 前置摄像头 + ''' + if camerType=="front": + incart = cv2.imread("./shopcart/cart_tempt/incart.png", cv2.IMREAD_GRAYSCALE) + outcart = cv2.imread("./shopcart/cart_tempt/outcart.png", cv2.IMREAD_GRAYSCALE) + else: + incart = cv2.imread("./shopcart/cart_tempt/incart_ftmp.png", cv2.IMREAD_GRAYSCALE) + outcart = cv2.imread("./shopcart/cart_tempt/outcart_ftmp.png", cv2.IMREAD_GRAYSCALE) + + xc, yc = self.cornpoints[:,0].clip(0,self.imgshape[0]-1).astype(np.int64), self.cornpoints[:,1].clip(0,self.imgshape[1]-1).astype(np.int64) + x1, y1 = self.cornpoints[:,6].clip(0,self.imgshape[0]-1).astype(np.int64), self.cornpoints[:,7].clip(0,self.imgshape[1]-1).astype(np.int64) + x2, y2 = self.cornpoints[:,8].clip(0,self.imgshape[0]-1).astype(np.int64), self.cornpoints[:,9].clip(0,self.imgshape[1]-1).astype(np.int64) + + # print(self.tid) + Cent_inCartnum = np.count_nonzero(incart[(yc, xc)]) + LB_inCartnum = np.count_nonzero(incart[(y1, x1)]) + RB_inCartnum = np.count_nonzero(incart[(y2, x2)]) + + Cent_outCartnum = np.count_nonzero(outcart[(yc, xc)]) + LB_outCartnum = np.count_nonzero(outcart[(y1, x1)]) + RB_outCartnum = np.count_nonzero(outcart[(y2, x2)]) + + '''Track完全在车内：左下角点、右下角点与 outcart 的交集为 0''' + self.isWholeInCart = False + if LB_outCartnum + RB_outCartnum == 0: + self.isWholeInCart = True + + '''Track完全在车外：左下角点、中心点与 incart 的交集为 0 + 右下角点、中心点与 incart 的交集为 0 + ''' + self.isWholeOutCart = False + if Cent_inCartnum + LB_inCartnum == 0 or Cent_inCartnum + RB_inCartnum == 0: + self.isWholeOutCart = True + + + self.Cent_isIncart = False + self.LB_isIncart = False + self.RB_isIncart = False + if Cent_inCartnum: self.Cent_isIncart = True + if LB_inCartnum: self.LB_isIncart = True + if RB_inCartnum: self.RB_isIncart = True + + self.posState = self.Cent_isIncart+self.LB_isIncart+self.RB_isIncart + def extract_hand_features(self): assert self.cls == 0, "The class of traj must be HAND!" @@ -296,10 +348,12 @@ class doTracks: def __init__(self, bboxes, TracksDict): '''fundamental property''' self.bboxes = bboxes - self.TracksDict = TracksDict + # self.TracksDict = TracksDict self.frameID = np.unique(bboxes[:, 7].astype(int)) self.trackID = np.unique(bboxes[:, 4].astype(int)) - self.lboxes = self.array2list() + + self.lboxes = self.array2list() + self.lfeats = self.getfeats(TracksDict) '''对 self.tracks 中的元素进行分类，将 track 归入相应列表中''' self.Hands = [] @@ -332,6 +386,19 @@ class doTracks: return lboxes + def getfeats(self, TracksDict): + lboxes = self.lboxes + lfeats = [] + for boxes in lboxes: + afeat = readDict(boxes, TracksDict) + lfeats.append(afeat) + + return lfeats + + + + + ''' def classify(self): @@ -362,12 +429,7 @@ class doTracks: return tracks ''' - - - - - - + def similarity(self): nt = len(self.tracks) @@ -428,7 +490,7 @@ class doTracks: blist = [b for b in alist] alist = [] for btrack in blist: - if track_equal_track(atrack, btrack, self.TracksDict): + if track_equal_track(atrack, btrack): cur_list.append(btrack) else: alist.append(btrack) diff --git a/tracking/dotrack/dotracks_back.py b/tracking/dotrack/dotracks_back.py index eab3da0..71f369e 100644 --- a/tracking/dotrack/dotracks_back.py +++ b/tracking/dotrack/dotracks_back.py @@ -16,7 +16,7 @@ class doBackTracks(doTracks): super().__init__(bboxes, TracksDict) - self.tracks = [backTrack(b) for b in self.lboxes] + self.tracks = [backTrack(b, f) for b, f in zip(self.lboxes, self.lfeats)] # self.similar_dict = self.similarity() @@ -67,6 +67,9 @@ class doBackTracks(doTracks): self.Kids = [x for x in zip(kid_tracks, kid_states)] tracks = self.sub_tracks(tracks, kid_tracks) + + out_trcak = [t for t in tracks if t.isWholeOutCart] + tracks = self.sub_tracks(tracks, out_trcak) static_tracks = [t for t in tracks if t.frnum>1 and t.is_static()] self.Static.extend(static_tracks) @@ -75,20 +78,84 @@ class doBackTracks(doTracks): tracks = self.sub_tracks(tracks, static_tracks) - '''购物框边界外具有运动状态的干扰目标''' - out_trcak = [t for t in tracks if t.is_OutTrack()] - - tracks = self.sub_tracks(tracks, out_trcak) + # '''购物框边界外具有运动状态的干扰目标''' + # out_trcak = [t for t in tracks if t.is_OutTrack()] + # tracks = self.sub_tracks(tracks, out_trcak) '''轨迹循环归并''' # merged_tracks = self.merge_tracks(tracks) merged_tracks = self.merge_tracks_loop(tracks) tracks = [t for t in merged_tracks if t.frnum > 1] - + + self.merged_tracks = merged_tracks + + static_tracks = [t for t in tracks if t.frnum>1 and t.is_static()] + self.Static.extend(static_tracks) + + tracks = self.sub_tracks(tracks, static_tracks) + + for gtrack in tracks: + # print(f"Goods ID:{gtrack.tid}") + for htrack in hand_tracks: + hand_ious = self.associate_with_hand(htrack, gtrack) + if len(hand_ious): + gtrack.Hands.append(htrack) + gtrack.HandsIou.append(hand_ious) + self.Residual = tracks - + def associate_with_hand(self, htrack, gtrack): + ''' + 迁移至基类： + 手部 Track、商品 Track 建立关联的依据： + a. 运动帧的帧索引有交集 + b. 帧索引交集部分iou均大于0 + ''' + + assert htrack.cls==0 and gtrack.cls!=0 and gtrack.cls!=9, 'Track cls is Error!' + + hand_ious = [] + + hboxes = np.empty(shape=(0, 9), dtype = np.float) + gboxes = np.empty(shape=(0, 9), dtype = np.float) + + # start, end 为索引值，需要 start:(end+1) + for start, end in htrack.moving_index: + hboxes = np.concatenate((hboxes, htrack.boxes[start:end+1, :]), axis=0) + for start, end in gtrack.moving_index: + gboxes = np.concatenate((gboxes, gtrack.boxes[start:end+1, :]), axis=0) + + hfids, gfids = hboxes[:, 7], gboxes[:, 7] + fids = set(hfids).intersection(set(gfids)) + + + if len(fids)==0: + return hand_ious + + + # print(f"Goods ID: {gtrack.tid}, Hand ID: {htrack.tid}") + + for f in fids: + h = np.where(hfids==f)[0][0] + g = np.where(gfids==f)[0][0] + + x11, y11, x12, y12 = hboxes[h, 0:4] + x21, y21, x22, y22 = gboxes[g, 0:4] + + x1, y1 = max((x11, x21)), max((y11, y21)) + x2, y2 = min((x12, x22)), min((y12, y22)) + + union = (x2 - x1).clip(0) * (y2 - y1).clip(0) + area1 = (x12 - x11) * (y12 - y11) + area2 = (x22 - x21) * (y22 - y21) + + iou = union / (area1 + area2 - union + 1e-6) + + if iou>0: + hand_ious.append((f, iou)) + + return hand_ious def merge_tracks(self, Residual): """ @@ -100,19 +167,27 @@ class doBackTracks(doTracks): for tracklist in mergedTracks: if len(tracklist) > 1: boxes = np.empty((0, 9), dtype=np.float32) + feats = np.empty((0, 256), dtype=np.float32) for i, track in enumerate(tracklist): if i==0: ntid, ncls=track.boxes[0, 4], track.boxes[0, 6] iboxes = track.boxes.copy() + ifeats = track.features.copy() + + # iboxes[:, 4], iboxes[:, 6] = ntid, ncls - iboxes[:, 4], iboxes[:, 6] = ntid, ncls boxes = np.concatenate((boxes, iboxes), axis=0) + feats = np.concatenate((feats, ifeats), axis=0) oldtracks.append(track) fid_indices = np.argsort(boxes[:, 7]) + boxes_fid = boxes[fid_indices] - - newtracks.append(backTrack(boxes_fid)) + feats_fid = feats[fid_indices] + + + + newtracks.append(backTrack(boxes_fid, feats_fid)) elif len(tracklist) == 1: oldtracks.append(tracklist[0]) newtracks.append(tracklist[0]) @@ -137,19 +212,6 @@ class doBackTracks(doTracks): return kidstate - - - def is_associate_with_hand(self): - """ - 分析商品和手之间的关联性 - """ - pass - - - - - - def isuptrack(self, track): Flag = False diff --git a/tracking/dotrack/dotracks_front.py b/tracking/dotrack/dotracks_front.py index be12a07..ccfc97c 100644 --- a/tracking/dotrack/dotracks_front.py +++ b/tracking/dotrack/dotracks_front.py @@ -13,7 +13,9 @@ class doFrontTracks(doTracks): def __init__(self, bboxes, TracksDict): super().__init__(bboxes, TracksDict) - self.tracks = [frontTrack(b) for b in self.lboxes] + # self.tracks = [frontTrack(b) for b in self.lboxes] + self.tracks = [frontTrack(b, f) for b, f in zip(self.lboxes, self.lfeats)] + def classify(self): '''功能：对 tracks 中元素分类 ''' @@ -32,7 +34,9 @@ class doFrontTracks(doTracks): '''提取小孩的 tracks''' kid_tracks = [t for t in tracks if t.cls==9] tracks = self.sub_tracks(tracks, kid_tracks) - + + out_trcak = [t for t in tracks if t.isWholeOutCart] + tracks = self.sub_tracks(tracks, out_trcak) '''静态 tracks''' static_tracks = [t for t in tracks if t.frnum>1 and t.is_static()] @@ -49,8 +53,11 @@ class doFrontTracks(doTracks): for gtrack in tracks: # print(f"Goods ID:{gtrack.tid}") for htrack in hand_tracks: - if self.is_associate_with_hand(htrack, gtrack): - gtrack.hands.append(htrack) + hand_ious = self.associate_with_hand(htrack, gtrack) + if len(hand_ious): + gtrack.Hands.append(htrack) + gtrack.HandsIou.append(hand_ious) + freemoved_tracks = [t for t in tracks if t.is_free_move()] @@ -59,14 +66,17 @@ class doFrontTracks(doTracks): self.Residual = tracks - def is_associate_with_hand(self, htrack, gtrack): - '''手部 Track、商品 Track 建立关联的依据： + def associate_with_hand(self, htrack, gtrack): + ''' + 迁移至基类： + 手部 Track、商品 Track 建立关联的依据： a. 运动帧的帧索引有交集 b. 帧索引交集部分iou均大于0 ''' assert htrack.cls==0 and gtrack.cls!=0 and gtrack.cls!=9, 'Track cls is Error!' + hand_ious = [] hboxes = np.empty(shape=(0, 9), dtype = np.float) gboxes = np.empty(shape=(0, 9), dtype = np.float) @@ -80,7 +90,7 @@ class doFrontTracks(doTracks): fids = set(hfids).intersection(set(gfids)) if len(fids)==0: - return False + return hand_ious # print(f"Goods ID: {gtrack.tid}, Hand ID: {htrack.tid}") @@ -102,9 +112,9 @@ class doFrontTracks(doTracks): iou = union / (area1 + area2 - union + 1e-6) if iou>0: - ious.append(iou) + hand_ious.append((f, iou)) - return len(ious) + return hand_ious @@ -137,17 +147,24 @@ class doFrontTracks(doTracks): for tracklist in mergedTracks: if len(tracklist) > 1: boxes = np.empty((0, 9), dtype=np.float32) + feats = np.empty((0, 256), dtype=np.float32) for i, track in enumerate(tracklist): if i==0: ntid, ncls=track.boxes[0, 4], track.boxes[0, 6] iboxes = track.boxes.copy() - iboxes[:, 4], iboxes[:, 6] = ntid, ncls + ifeats = track.features.copy() + + # iboxes[:, 4], iboxes[:, 6] = ntid, ncls + boxes = np.concatenate((boxes, iboxes), axis=0) + feats = np.concatenate((feats, ifeats), axis=0) + oldtracks.append(track) fid_indices = np.argsort(boxes[:, 7]) boxes_fid = boxes[fid_indices] + feats_fid = feats[fid_indices] - newtracks.append(frontTrack(boxes_fid)) + newtracks.append(frontTrack(boxes_fid, feats_fid)) elif len(tracklist) == 1: oldtracks.append(tracklist[0]) newtracks.append(tracklist[0]) diff --git a/tracking/dotrack/track_back.py b/tracking/dotrack/track_back.py index 5edf008..391e4c8 100644 --- a/tracking/dotrack/track_back.py +++ b/tracking/dotrack/track_back.py @@ -14,29 +14,32 @@ from .dotracks import MoveState, Track class backTrack(Track): # boxes: [x1, y1, x2, y2, track_id, score, cls, frame_index, box_index] # 0, 1, 2, 3, 4, 5, 6, 7, 8 - def __init__(self, boxes, imgshape=(1024, 1280)): + def __init__(self, boxes, features, imgshape=(1024, 1280)): - super().__init__(boxes, imgshape) + super().__init__(boxes, features, imgshape) - '''该函数依赖项: self.cornpoints''' - self.isCornpoint = self.isimgborder() + '''该函数依赖项: self.cornpoints + MarginState: list, seven elements, 表示轨迹中boxes出现在图像的 + [左上，右上，左中，右中，左下，右下底部] + ''' + self.isCornpoint, self.MarginState = self.isimgborder() '''该函数依赖项: self.isCornpoint，不能在父类中初始化''' self.trajfeature() '''静止点帧索引''' - self.static_index = self.compute_static_fids() + # self.static_index = self.compute_static_fids() '''运动点帧索引(运动帧两端的静止帧索引)''' - self.moving_index = self.compute_dynamic_fids() + # self.moving_index = self.compute_dynamic_fids() - # self.static_index, self.moving_index = self.compute_static_dynamic_fids() + self.static_index, self.moving_index = self.compute_static_dynamic_fids() '''该函数依赖项: self.cornpoints，定义 4 个商品位置变量： self.Cent_isIncart, self.LB_isIncart, self.RB_isIncart self.posState = self.Cent_isIncart+self.LB_isIncart+self.RB_isIncart''' - self.PositionState() + self.PositionState(camerType="back") '''self.feature_ious = (incart_iou, outcart_iou, cartboarder_iou, maxbox_iou, minbox_iou) self.incartrates = incartrates''' @@ -51,47 +54,79 @@ class backTrack(Track): x1, y1 = self.cornpoints[:,2], self.cornpoints[:,3], x2, y2 = self.cornpoints[:,8], self.cornpoints[:,9] - cont1 = sum(abs(x1) BoundThresh - cont2 = sum(abs(y1) BoundThresh - cont3 = sum(abs(x2-self.imgshape[0]) BoundThresh - cont4 = sum(abs(y2-self.imgshape[1]) BoundThresh + condt1 = sum(abs(x1) BoundThresh + condt2 = sum(abs(y1) BoundThresh + condt3 = sum(abs(x2-self.imgshape[0]) BoundThresh + condt4 = sum(abs(y2-self.imgshape[1]) BoundThresh - cont = cont1 or cont2 or cont3 or cont4 + condt = condt1 or condt2 or condt3 or condt4 isCornpoint = False - if cont: + if condt: isCornpoint = True - - return isCornpoint + + condtA = condt1 and condt2 + condtB = condt3 and condt2 + condtC = condt1 and not condt2 and not condt4 + condtD = condt3 and not condt2 and not condt4 + condtE = condt1 and condt4 + condtF = condt3 and condt4 + condtG = condt4 and not condt1 and not condt3 + MarginState = [condtA, condtB, condtC, condtD, condtE, condtF, condtG] + + return isCornpoint, MarginState - def PositionState(self, camerType="back"): - ''' - camerType: back, 后置摄像头 - front, 前置摄像头 - ''' - if camerType=="front": - incart = cv2.imread("./shopcart/cart_tempt/incart.png", cv2.IMREAD_GRAYSCALE) - else: - incart = cv2.imread("./shopcart/cart_tempt/incart_ftmp.png", cv2.IMREAD_GRAYSCALE) +# ============================================================================= +# def PositionState(self, camerType="back"): +# ''' +# 已迁移至基类 +# camerType: back, 后置摄像头 +# front, 前置摄像头 +# ''' +# if camerType=="front": +# incart = cv2.imread("./shopcart/cart_tempt/incart.png", cv2.IMREAD_GRAYSCALE) +# outcart = cv2.imread("./shopcart/cart_tempt/outcart.png", cv2.IMREAD_GRAYSCALE) +# else: +# incart = cv2.imread("./shopcart/cart_tempt/incart_ftmp.png", cv2.IMREAD_GRAYSCALE) +# outcart = cv2.imread("./shopcart/cart_tempt/outcart_ftmp.png", cv2.IMREAD_GRAYSCALE) +# +# xc, yc = self.cornpoints[:,0].clip(0,self.imgshape[0]-1).astype(np.int64), self.cornpoints[:,1].clip(0,self.imgshape[1]-1).astype(np.int64) +# x1, y1 = self.cornpoints[:,6].clip(0,self.imgshape[0]-1).astype(np.int64), self.cornpoints[:,7].clip(0,self.imgshape[1]-1).astype(np.int64) +# x2, y2 = self.cornpoints[:,8].clip(0,self.imgshape[0]-1).astype(np.int64), self.cornpoints[:,9].clip(0,self.imgshape[1]-1).astype(np.int64) +# +# # print(self.tid) +# Cent_inCartnum = np.count_nonzero(incart[(yc, xc)]) +# LB_inCartnum = np.count_nonzero(incart[(y1, x1)]) +# RB_inCartnum = np.count_nonzero(incart[(y2, x2)]) +# +# Cent_outCartnum = np.count_nonzero(outcart[(yc, xc)]) +# LB_outCartnum = np.count_nonzero(outcart[(y1, x1)]) +# RB_outCartnum = np.count_nonzero(outcart[(y2, x2)]) +# +# '''Track完全在车内：左下角点、右下角点与 outcart 的交集为 0''' +# self.isWholeInCart = False +# if LB_outCartnum + RB_outCartnum == 0: +# self.isWholeInCart = True +# +# '''Track完全在车外：左下角点、中心点与 incart 的交集为 0 +# 右下角点、中心点与 incart 的交集为 0 +# ''' +# self.isWholeOutCart = False +# if Cent_inCartnum + LB_inCartnum == 0 or Cent_inCartnum + RB_inCartnum == 0: +# self.isWholeOutCart = True +# +# +# self.Cent_isIncart = False +# self.LB_isIncart = False +# self.RB_isIncart = False +# if Cent_inCartnum: self.Cent_isIncart = True +# if LB_inCartnum: self.LB_isIncart = True +# if RB_inCartnum: self.RB_isIncart = True +# +# self.posState = self.Cent_isIncart+self.LB_isIncart+self.RB_isIncart +# ============================================================================= + - xc, yc = self.cornpoints[:,0].clip(0,self.imgshape[0]-1).astype(np.int64), self.cornpoints[:,1].clip(0,self.imgshape[1]-1).astype(np.int64) - x1, y1 = self.cornpoints[:,6].clip(0,self.imgshape[0]-1).astype(np.int64), self.cornpoints[:,7].clip(0,self.imgshape[1]-1).astype(np.int64) - x2, y2 = self.cornpoints[:,8].clip(0,self.imgshape[0]-1).astype(np.int64), self.cornpoints[:,9].clip(0,self.imgshape[1]-1).astype(np.int64) - - # print(self.tid) - Cent_inCartnum = np.count_nonzero(incart[(yc, xc)]) - LB_inCartnum = np.count_nonzero(incart[(y1, x1)]) - RB_inCartnum = np.count_nonzero(incart[(y2, x2)]) - - self.Cent_isIncart = False - self.LB_isIncart = False - self.RB_isIncart = False - if Cent_inCartnum: self.Cent_isIncart = True - if LB_inCartnum: self.LB_isIncart = True - if RB_inCartnum: self.RB_isIncart = True - - self.posState = self.Cent_isIncart+self.LB_isIncart+self.RB_isIncart - def PCA(self): @@ -251,9 +286,25 @@ class backTrack(Track): def compute_static_dynamic_fids(self): - idx2 = self.trajlens.index(min(self.trajlens)) - trajmin = self.trajectory[idx2] + if self.MarginState[0] or self.MarginState[2]: + idx1 = 4 + elif self.MarginState[1] or self.MarginState[3]: + idx1 = 3 + elif self.MarginState[4]: + idx1 = 2 + elif self.MarginState[5]: + idx1 = 1 + elif self.MarginState[6]: + if self.trajlens[1] < self.trajlens[2]: + idx1 = 1 + else: + idx1 = 2 + else: + idx1 = self.trajlens.index(min(self.trajlens)) + # idx1 = self.trajlens.index(min(self.trajlens)) + trajmin = self.trajectory[idx1] + static, dynamic = self.pt_state_fids(trajmin) static = np.array(static) @@ -279,47 +330,60 @@ class backTrack(Track): # # return static_dynamic_fids # ============================================================================= - - - - - - def is_static(self): - + + def is_static(self): '''静态情况 1: 目标关键点最小相对运动轨迹 < 0.2, 指标值偏大 - feature = [trajlen_min, trajlen_max, + TrajFeat = [trajlen_min, trajlen_max, trajdist_min, trajdist_max, trajlen_rate, trajdist_rate] ''' - # print(f"TrackID: {self.tid}") + boxes = self.boxes - - condt1 = self.feature[5] < 0.2 or self.feature[3] < 120 + '''静态情况 1: ''' + condt1 = self.TrajFeat[5] < 0.2 or self.TrajFeat[3] < 120 '''静态情况 2: 目标初始状态为静止，适当放宽关键点最小相对运动轨迹 < 0.5''' condt2 = self.static_index.size > 0 \ and self.static_index[0, 0] <= 2 \ - and self.feature[5] < 0.5 + and self.static_index[0, 1] >= 5 \ + and self.TrajFeat[5] < 0.5 \ + and self.TrajFeat[1] < 240 \ + and self.isWholeInCart + # and self.posState >= 2 + # and self.TrajFeat[0] < 240 \ + '''静态情况 3: 目标初始状态和最终状态均为静止''' condt3 = self.static_index.shape[0] >= 2 \ and self.static_index[0, 0] <= 2 \ + and self.static_index[0, 1] >= 5 \ and self.static_index[-1, 1] >= self.frnum-3 \ + and self.TrajFeat[1] < 240 \ + and self.isWholeInCart + # and self.posState >= 2 + # and self.TrajFeat[0] < 240 \ - condt = condt1 or condt2 or condt3 + condt4 = self.static_index.shape[0] >= 2 \ + and self.static_index[0, 0] <= 2 \ + and self.static_index[0, 1] >= 6 \ + and self.static_index[-1, 0] <= self.frnum-5 \ + and self.static_index[-1, 1] >= self.frnum-2 + + + condt = condt1 or condt2 or condt3 or condt4 return condt # ============================================================================= -# track1 = [t for t in tracks if t.feature[5] < 0.2 -# or t.feature[3] < 120 +# track1 = [t for t in tracks if t.TrajFeat[5] < 0.2 +# or t.TrajFeat[3] < 120 # ] # # track2 = [t for t in tracks if t.static_index.size > 0 # and t.static_index[0, 0] <= 2 -# and t.feature[5] < 0.5] +# and t.TrajFeat[5] < 0.5] # # track3 = [t for t in tracks if t.static_index.shape[0] >= 2 # and t.static_index[0, 0] <= 2 diff --git a/tracking/dotrack/track_front.py b/tracking/dotrack/track_front.py index 42e5d0b..83e178c 100644 --- a/tracking/dotrack/track_front.py +++ b/tracking/dotrack/track_front.py @@ -12,12 +12,10 @@ from .dotracks import MoveState, Track class frontTrack(Track): # boxes: [x1, y1, x2, y2, track_id, score, cls, frame_index, box_index] # 0, 1, 2, 3, 4, 5, 6, 7, 8 - def __init__(self, boxes, imgshape=(1024, 1280)): - - super().__init__(boxes, imgshape) - self.hands = [] - + def __init__(self, boxes, features, imgshape=(1024, 1280)): + super().__init__(boxes, features, imgshape) + '''5个关键点（中心点、左上点、右上点、左下点、右下点 ）轨迹特征''' # self.compute_cornpts_feats() @@ -40,6 +38,7 @@ class frontTrack(Track): '''该函数依赖项: self.isCornpoint，不能在父类中初始化''' self.trajfeature() + self.PositionState(camerType="front") '''手部状态分析''' self.HAND_STATIC_THRESH = 100 @@ -245,7 +244,7 @@ class frontTrack(Track): '''情况2：中心点向上 ''' ## 商品中心点向上移动，但没有关联的Hand轨迹，也不是左右边界点 - condt_b = condt0 and len(self.hands)==0 and y0[-1] < y0[0] and (not self.is_edge_cornpoint()) + condt_b = condt0 and len(self.Hands)==0 and y0[-1] < y0[0] and (not self.is_edge_cornpoint()) '''情况3: 商品在购物车内，但运动方向无序''' diff --git a/tracking/test_tracking.py b/tracking/test_tracking.py index 8fd5c4a..8e35b60 100644 --- a/tracking/test_tracking.py +++ b/tracking/test_tracking.py @@ -21,7 +21,7 @@ from pathlib import Path # ================= using for import ultralytics import sys -sys.path.append(r"D:\yolov5track") +sys.path.append(r"D:\DetectTracking") from utils.gen import Profile from dotrack.dotracks_back import doBackTracks @@ -68,7 +68,7 @@ def detect_start_end(bboxes, features_dict, filename): return start, end -def save_subimgs(vts, file): +def save_subimgs(vts, file, TracksDict): imgdir = Path(f'./result/imgs/{file}') if not imgdir.exists(): imgdir.mkdir(parents=True, exist_ok=True) @@ -78,19 +78,21 @@ def save_subimgs(vts, file): for ii in range(len(boxes)): tid, fid, bid = int(boxes[ii, 4]), int(boxes[ii, 7]), int(boxes[ii, 8]) - img = vts.TracksDict[f"frame_{fid}"]["imgs"][bid] + img = TracksDict[f"frame_{fid}"]["imgs"][bid] # feat = TracksDict[f"frame_{fid}"]["feats"][bid] # box = TracksDict[f"frame_{fid}"]["boxes"][bid] cv2.imwrite(str(imgdir) + f"/{tid}_{fid}_{bid}.png", img) def have_tracked(): - trackdict = r'./data/trackdicts' + trackdict = r'./data/trackdicts_1' alltracks = [] k = 0 gt = Profile() for filename in os.listdir(trackdict): - filename = 'test_20240402-173935_6920152400975_back_174037372.pkl' + # filename = 'test_20240402-173935_6920152400975_back_174037372.pkl' + filename = '加购_91.pkl' + file, ext = os.path.splitext(filename) filepath = os.path.join(trackdict, filename) TracksDict = np.load(filepath, allow_pickle=True) @@ -101,7 +103,7 @@ def have_tracked(): vts = doFrontTracks(bboxes, TracksDict) vts.classify() - save_subimgs(vts, file) + save_subimgs(vts, file, TracksDict) plt = plot_frameID_y2(vts) @@ -114,7 +116,7 @@ def have_tracked(): vts.classify() alltracks.append(vts) - save_subimgs(vts, file) + save_subimgs(vts, file, TracksDict) edgeline = cv2.imread("./shopcart/cart_tempt/edgeline.png") draw_all_trajectories(vts, edgeline, save_dir, filename) print(file+f" need time: {gt.dt:.2f}s") diff --git a/tracking/trackers/__pycache__/bot_sort.cpython-39.pyc b/tracking/trackers/__pycache__/bot_sort.cpython-39.pyc index 0be2497..a6bfb5c 100644 Binary files a/tracking/trackers/__pycache__/bot_sort.cpython-39.pyc and b/tracking/trackers/__pycache__/bot_sort.cpython-39.pyc differ diff --git a/tracking/trackers/bot_sort.py b/tracking/trackers/bot_sort.py index 33847b2..7a69947 100644 --- a/tracking/trackers/bot_sort.py +++ b/tracking/trackers/bot_sort.py @@ -161,7 +161,7 @@ class BOTSORT(BYTETracker): '''1. reid 相似度阈值，低于该值的两 boxes 图像不可能是同一对象，需要确定一个合理的可信阈值 2. iou 的约束为若约束，故 iou_dists 应设置为较大的值 ''' - emb_dists_mask = (emb_dists > 0.85) + emb_dists_mask = (emb_dists > 0.65) iou_dists[emb_dists_mask] = 1 emb_dists[iou_dists_mask] = 1 diff --git a/tracking/utils/__pycache__/drawtracks.cpython-39.pyc b/tracking/utils/__pycache__/drawtracks.cpython-39.pyc index 915f2cb..c801a9f 100644 Binary files a/tracking/utils/__pycache__/drawtracks.cpython-39.pyc and b/tracking/utils/__pycache__/drawtracks.cpython-39.pyc differ diff --git a/tracking/utils/__pycache__/mergetrack.cpython-39.pyc b/tracking/utils/__pycache__/mergetrack.cpython-39.pyc index 586caff..68c9445 100644 Binary files a/tracking/utils/__pycache__/mergetrack.cpython-39.pyc and b/tracking/utils/__pycache__/mergetrack.cpython-39.pyc differ diff --git a/tracking/utils/drawtracks.py b/tracking/utils/drawtracks.py index 436e2c5..f5110a9 100644 --- a/tracking/utils/drawtracks.py +++ b/tracking/utils/drawtracks.py @@ -87,20 +87,29 @@ def draw_all_trajectories(vts, edgeline, save_dir, filename): # edgeline2 = edgeline1.copy() # edgeline = np.concatenate((edgeline1, edgeline2), exis = 1) -# ============================================================================= -# '''1. tracks 5点轨迹''' -# trackpth = save_dir.parent /Path("trajectory")/ Path(f"{file}") -# if not trackpth.exists(): -# trackpth.mkdir(parents=True, exist_ok=True) -# for track in vts.tracks: -# # if track.cls != 0: -# img = edgeline.copy() -# img = draw5points(track, img) -# -# pth = trackpth.joinpath(f"{track.tid}.png") -# cv2.imwrite(str(pth), img) -# -# ============================================================================= + '''1. tracks 5点轨迹''' + trackpth = save_dir.parent /Path("trajectory")/ Path(f"{file}") + if not trackpth.exists(): + trackpth.mkdir(parents=True, exist_ok=True) + for track in vts.tracks: + # if track.cls != 0: + img = edgeline.copy() + img = draw5points(track, img) + + pth = trackpth.joinpath(f"{track.tid}.png") + cv2.imwrite(str(pth), img) + + for track in vts.merged_tracks: + # if track.cls != 0: + img = edgeline.copy() + img = draw5points(track, img) + + pth = trackpth.joinpath(f"{track.tid}_.png") + cv2.imwrite(str(pth), img) + + + + '''2. all tracks 中心轨迹''' img1, img2 = edgeline.copy(), edgeline.copy() @@ -139,7 +148,7 @@ def draw_all_trajectories(vts, edgeline, save_dir, filename): def drawFeatures(allvts, save_dir): # [trajlen_min, trajdist_max, trajlen_rate, trajist_rate]] - feats = [track.feature for vts in allvts for track in vts.tracks] + feats = [track.TrajFeat for vts in allvts for track in vts.tracks] feats = np.array(feats) fig, ax = plt.subplots() ax.scatter(feats[:,3], feats[:, 1], s=10) @@ -269,7 +278,7 @@ def draw5points(track, img): trajstd = 0 - trajlen_min, trajlen_max, trajdist_min, trajdist_max, trajlen_rate, trajdist_rate = track.feature + trajlen_min, trajlen_max, trajdist_min, trajdist_max, trajlen_rate, trajdist_rate = track.TrajFeat for i in range(boxes.shape[0]): cv2.circle(img, (int(cornpoints[i, 0]), int(cornpoints[i, 1])), 6, (255, 255, 255), 2) diff --git a/tracking/utils/mergetrack.py b/tracking/utils/mergetrack.py index a224d1b..d3968d0 100644 --- a/tracking/utils/mergetrack.py +++ b/tracking/utils/mergetrack.py @@ -31,12 +31,19 @@ def readDict(boxes, TracksDict): -def track_equal_track(atrack, btrack, TracksDict): +def track_equal_track(atrack, btrack): # boxes: [x, y, w, h, track_id, score, cls, frame_index, box_index] # 0 1 2 3 4 5 6 7 8 aboxes = atrack.boxes bboxes = btrack.boxes + afeat = atrack.features + bfeat = btrack.features + + # afeat = readDict(aboxes, TracksDict) + # bfeat = readDict(bboxes, TracksDict) + + ''' 1. 判断轨迹在时序上是否有交集 ''' afids = aboxes[:, 7].astype(np.int_) bfids = bboxes[:, 7].astype(np.int_) @@ -50,8 +57,6 @@ def track_equal_track(atrack, btrack, TracksDict): return False ''' 2. 轨迹特征相似度判断''' - afeat = readDict(aboxes, TracksDict) - bfeat = readDict(bboxes, TracksDict) feat = np.concatenate((afeat, bfeat), axis=0) emb_simil = 1-np.maximum(0.0, cdist(feat, feat, 'cosine')) @@ -82,9 +87,15 @@ def track_equal_track(atrack, btrack, TracksDict): idx_pair.append((a_idx, b_idx)) ious = [] + embs = [] for a, b in idx_pair: abox, bbox = aboxes[a, :], bboxes[b, :] + af, bf = afeat[a, :], bfeat[b, :] + + emb_ab = 1-cdist(af[None, :], bf[None, :], 'cosine') + + xa1, ya1 = abox[0] - abox[2]/2, abox[1] - abox[3]/2 xa2, ya2 = abox[0] + abox[2]/2, abox[1] + abox[3]/2 @@ -101,12 +112,15 @@ def track_equal_track(atrack, btrack, TracksDict): union = box1_area + box2_area - inter + 1e-6 ious.append(inter/union) + embs.append(emb_ab[0, 0]) - cont = False if len(interfid) else True - # cont2 = emb_[0, 0]>0.75 - # cont3 = all(iou>0.5 for iou in ious) + cont = False if len(interfid) else True # fid 无交集 + cont1 = all(emb > 0.5 for emb in embs) + cont2 = all(iou > 0.5 for iou in ious) # cont = cont and cont2 and cont3 + + cont = cont and cont1 and cont2 return cont