470 lines
18 KiB
Python
470 lines
18 KiB
Python
# Ultralytics YOLO 🚀, AGPL-3.0 license
|
||
|
||
import numpy as np
|
||
|
||
from .basetrack import BaseTrack, TrackState
|
||
from .utils import matching
|
||
from .utils.kalman_filter import KalmanFilterXYAH
|
||
|
||
|
||
def dists_update(dists, strack_pool, detections):
|
||
'''written by WQG'''
|
||
|
||
if len(strack_pool) and len(detections):
|
||
# alabel = np.array([int(stack.cls) if int(stack.cls)==0 or int(stack.cls)==9 else -1 for stack in strack_pool])
|
||
# blabel = np.array([int(stack.cls) if int(stack.cls)==0 or int(stack.cls)==9 else -1 for stack in detections])
|
||
|
||
alabel = np.array([int(stack.cls) for stack in strack_pool])
|
||
blabel = np.array([int(stack.cls) for stack in detections])
|
||
amlabel = np.expand_dims(alabel, axis=1).repeat(len(detections),axis=1)
|
||
bmlabel = np.expand_dims(blabel, axis=0).repeat(len(strack_pool),axis=0)
|
||
dist_label = 1 - (bmlabel == amlabel)
|
||
dists = np.where(dists > dist_label, dists, dist_label)
|
||
return dists
|
||
|
||
|
||
class STrack(BaseTrack):
|
||
shared_kalman = KalmanFilterXYAH()
|
||
|
||
def __init__(self, tlwh, score, cls):
|
||
"""wait activate."""
|
||
self._tlwh = np.asarray(self.tlbr_to_tlwh(tlwh[:-1]), dtype=np.float32)
|
||
self.kalman_filter = None
|
||
self.mean, self.covariance = None, None
|
||
self.is_activated = False
|
||
|
||
self.first_find = False ###
|
||
|
||
self.score = score
|
||
self.tracklet_len = 0
|
||
self.cls = cls
|
||
self.idx = tlwh[-1]
|
||
|
||
def predict(self):
|
||
"""Predicts mean and covariance using Kalman filter."""
|
||
mean_state = self.mean.copy()
|
||
if self.state != TrackState.Tracked:
|
||
mean_state[7] = 0
|
||
self.mean, self.covariance = self.kalman_filter.predict(mean_state, self.covariance)
|
||
|
||
@staticmethod
|
||
def multi_predict(stracks):
|
||
"""Perform multi-object predictive tracking using Kalman filter for given stracks."""
|
||
if len(stracks) <= 0:
|
||
return
|
||
multi_mean = np.asarray([st.mean.copy() for st in stracks])
|
||
multi_covariance = np.asarray([st.covariance for st in stracks])
|
||
for i, st in enumerate(stracks):
|
||
if st.state != TrackState.Tracked:
|
||
multi_mean[i][7] = 0
|
||
multi_mean, multi_covariance = STrack.shared_kalman.multi_predict(multi_mean, multi_covariance)
|
||
for i, (mean, cov) in enumerate(zip(multi_mean, multi_covariance)):
|
||
stracks[i].mean = mean
|
||
stracks[i].covariance = cov
|
||
|
||
@staticmethod
|
||
def multi_gmc(stracks, H=np.eye(2, 3)):
|
||
"""Update state tracks positions and covariances using a homography matrix."""
|
||
if len(stracks) > 0:
|
||
multi_mean = np.asarray([st.mean.copy() for st in stracks])
|
||
multi_covariance = np.asarray([st.covariance for st in stracks])
|
||
|
||
R = H[:2, :2]
|
||
R8x8 = np.kron(np.eye(4, dtype=float), R)
|
||
t = H[:2, 2]
|
||
|
||
for i, (mean, cov) in enumerate(zip(multi_mean, multi_covariance)):
|
||
mean = R8x8.dot(mean)
|
||
mean[:2] += t
|
||
cov = R8x8.dot(cov).dot(R8x8.transpose())
|
||
|
||
stracks[i].mean = mean
|
||
stracks[i].covariance = cov
|
||
|
||
def activate(self, kalman_filter, frame_id):
|
||
"""Start a new tracklet."""
|
||
self.kalman_filter = kalman_filter
|
||
self.track_id = self.next_id()
|
||
self.mean, self.covariance = self.kalman_filter.initiate(self.convert_coords(self._tlwh))
|
||
|
||
self.tracklet_len = 0
|
||
self.state = TrackState.Tracked
|
||
if frame_id == 1:
|
||
self.is_activated = True
|
||
else:
|
||
self.first_find = True ### Add by WQG
|
||
self.frame_id = frame_id
|
||
self.start_frame = frame_id
|
||
|
||
def re_activate(self, new_track, frame_id, new_id=False):
|
||
"""Reactivates a previously lost track with a new detection."""
|
||
self.mean, self.covariance = self.kalman_filter.update(self.mean, self.covariance,
|
||
self.convert_coords(new_track.tlwh))
|
||
self.tracklet_len = 0
|
||
self.state = TrackState.Tracked
|
||
self.is_activated = True
|
||
self.frame_id = frame_id
|
||
if new_id:
|
||
self.track_id = self.next_id()
|
||
self.score = new_track.score
|
||
self.cls = new_track.cls
|
||
self.idx = new_track.idx
|
||
|
||
self._tlwh = new_track._tlwh
|
||
|
||
def update(self, new_track, frame_id):
|
||
"""
|
||
Update a matched track
|
||
:type new_track: STrack
|
||
:type frame_id: int
|
||
:return:
|
||
"""
|
||
self.frame_id = frame_id
|
||
self.tracklet_len += 1
|
||
|
||
new_tlwh = new_track.tlwh
|
||
self.mean, self.covariance = self.kalman_filter.update(self.mean, self.covariance,
|
||
self.convert_coords(new_tlwh))
|
||
self.state = TrackState.Tracked
|
||
self.is_activated = True
|
||
|
||
self.score = new_track.score
|
||
self.cls = new_track.cls
|
||
self.idx = new_track.idx
|
||
|
||
self._tlwh = new_track._tlwh
|
||
|
||
|
||
def convert_coords(self, tlwh):
|
||
"""Convert a bounding box's top-left-width-height format to its x-y-angle-height equivalent."""
|
||
return self.tlwh_to_xyah(tlwh)
|
||
|
||
@property
|
||
def tlwh(self):
|
||
"""Get current position in bounding box format `(top left x, top left y,
|
||
width, height)`.
|
||
"""
|
||
if self.mean is None:
|
||
return self._tlwh.copy()
|
||
ret = self.mean[:4].copy()
|
||
ret[2] *= ret[3]
|
||
ret[:2] -= ret[2:] / 2
|
||
return ret
|
||
|
||
@property
|
||
def tlbr(self):
|
||
"""Convert bounding box to format `(min x, min y, max x, max y)`, i.e.,
|
||
`(top left, bottom right)`.
|
||
"""
|
||
ret = self.tlwh.copy()
|
||
ret[2:] += ret[:2]
|
||
return ret
|
||
|
||
@staticmethod
|
||
def tlwh_to_xyah(tlwh):
|
||
"""Convert bounding box to format `(center x, center y, aspect ratio,
|
||
height)`, where the aspect ratio is `width / height`.
|
||
"""
|
||
ret = np.asarray(tlwh).copy()
|
||
ret[:2] += ret[2:] / 2
|
||
ret[2] /= ret[3]
|
||
return ret
|
||
|
||
@staticmethod
|
||
def tlbr_to_tlwh(tlbr):
|
||
"""Converts top-left bottom-right format to top-left width height format."""
|
||
ret = np.asarray(tlbr).copy()
|
||
ret[2:] -= ret[:2]
|
||
return ret
|
||
|
||
@staticmethod
|
||
def tlwh_to_tlbr(tlwh):
|
||
"""Converts tlwh bounding box format to tlbr format."""
|
||
ret = np.asarray(tlwh).copy()
|
||
ret[2:] += ret[:2]
|
||
return ret
|
||
|
||
def __repr__(self):
|
||
"""Return a string representation of the BYTETracker object with start and end frames and track ID."""
|
||
return f'OT_{self.track_id}_({self.start_frame}-{self.end_frame})'
|
||
|
||
|
||
class BYTETracker:
|
||
|
||
def __init__(self, args, frame_rate=30):
|
||
"""Initialize a YOLOv8 object to track objects with given arguments and frame rate."""
|
||
self.tracked_stracks = [] # type: list[STrack]
|
||
self.lost_stracks = [] # type: list[STrack]
|
||
self.removed_stracks = [] # type: list[STrack]
|
||
|
||
self.frame_id = 0
|
||
self.args = args
|
||
self.max_time_lost = int(frame_rate / 30.0 * args.track_buffer)
|
||
self.kalman_filter = self.get_kalmanfilter()
|
||
self.reset_id()
|
||
|
||
# Add by WQG
|
||
self.args.new_track_thresh = 0.5
|
||
|
||
|
||
def update(self, results, img=None):
|
||
"""Updates object tracker with new detections and returns tracked object bounding boxes."""
|
||
self.frame_id += 1
|
||
activated_stracks = []
|
||
refind_stracks = []
|
||
lost_stracks = []
|
||
removed_stracks = []
|
||
|
||
first_finded = []
|
||
|
||
scores = results.conf
|
||
cls = results.cls
|
||
|
||
# =============================================================================
|
||
# # get xyxy and add index
|
||
# bboxes = results.xyxy
|
||
# bboxes = np.concatenate([bboxes, np.arange(len(bboxes)).reshape(-1, 1)], axis=-1)
|
||
# =============================================================================
|
||
bboxes = results.xyxyb
|
||
|
||
|
||
remain_inds = scores > self.args.track_high_thresh
|
||
inds_low = scores > self.args.track_low_thresh
|
||
inds_high = scores < self.args.track_high_thresh
|
||
|
||
inds_second = np.logical_and(inds_low, inds_high)
|
||
dets_second = bboxes[inds_second]
|
||
dets = bboxes[remain_inds]
|
||
scores_keep = scores[remain_inds]
|
||
scores_second = scores[inds_second]
|
||
cls_keep = cls[remain_inds]
|
||
cls_second = cls[inds_second]
|
||
|
||
detections = self.init_track(dets, scores_keep, cls_keep, img)
|
||
|
||
# Add newly detected tracklets to tracked_stracks
|
||
unconfirmed = []
|
||
tracked_stracks = [] # type: list[STrack]
|
||
for track in self.tracked_stracks:
|
||
if not track.is_activated:
|
||
unconfirmed.append(track)
|
||
else:
|
||
tracked_stracks.append(track)
|
||
|
||
|
||
# Step 2: First association, with high score detection boxes
|
||
strack_pool = self.joint_stracks(tracked_stracks, self.lost_stracks)
|
||
# Predict the current location with KF
|
||
self.multi_predict(strack_pool)
|
||
|
||
# ============================================================= 没必要gmc，WQG
|
||
# if hasattr(self, 'gmc') and img is not None:
|
||
# warp = self.gmc.apply(img, dets)
|
||
# STrack.multi_gmc(strack_pool, warp)
|
||
# STrack.multi_gmc(unconfirmed, warp)
|
||
# =============================================================================
|
||
|
||
dists = self.get_dists_1(strack_pool, detections)
|
||
|
||
'''written by WQG for different class'''
|
||
dists = dists_update(dists, strack_pool, detections)
|
||
|
||
matches, u_track, u_detection = matching.linear_assignment(dists, thresh=self.args.match_thresh)
|
||
for itracked, idet in matches:
|
||
track = strack_pool[itracked]
|
||
det = detections[idet]
|
||
if track.state == TrackState.Tracked:
|
||
track.update(det, self.frame_id)
|
||
activated_stracks.append(track)
|
||
else:
|
||
track.re_activate(det, self.frame_id, new_id=False)
|
||
refind_stracks.append(track)
|
||
|
||
|
||
# Step 3: Second association, with low score detection boxes
|
||
# association the untrack to the low score detections
|
||
detections_second = self.init_track(dets_second, scores_second, cls_second, img)
|
||
r_tracked_stracks = [strack_pool[i] for i in u_track if strack_pool[i].state == TrackState.Tracked]
|
||
|
||
# TODO
|
||
dists = matching.iou_distance(r_tracked_stracks, detections_second)
|
||
'''written by WQG for different class'''
|
||
dists = dists_update(dists, r_tracked_stracks, detections_second)
|
||
|
||
matches, u_track, u_detection_second = matching.linear_assignment(dists, thresh=0.5)
|
||
for itracked, idet in matches:
|
||
track = r_tracked_stracks[itracked]
|
||
det = detections_second[idet]
|
||
if track.state == TrackState.Tracked:
|
||
track.update(det, self.frame_id)
|
||
activated_stracks.append(track)
|
||
else:
|
||
track.re_activate(det, self.frame_id, new_id=False)
|
||
refind_stracks.append(track)
|
||
|
||
for it in u_track:
|
||
track = r_tracked_stracks[it]
|
||
if track.state != TrackState.Lost:
|
||
track.mark_lost()
|
||
lost_stracks.append(track)
|
||
|
||
# Deal with unconfirmed tracks, usually tracks with only one beginning frame
|
||
detections = [detections[i] for i in u_detection]
|
||
dists = self.get_dists_1(unconfirmed, detections)
|
||
'''written by WQG for different class'''
|
||
dists = dists_update(dists, unconfirmed, detections)
|
||
|
||
matches, u_unconfirmed, u_detection = matching.linear_assignment(dists, thresh=0.7)
|
||
for itracked, idet in matches:
|
||
unconfirmed[itracked].update(detections[idet], self.frame_id)
|
||
activated_stracks.append(unconfirmed[itracked])
|
||
for it in u_unconfirmed:
|
||
track = unconfirmed[it]
|
||
if self.frame_id - track.end_frame > 2: # Add by WQG
|
||
track.mark_removed()
|
||
removed_stracks.append(track)
|
||
# Step 4: Init new stracks
|
||
for inew in u_detection:
|
||
track = detections[inew]
|
||
if track.score < self.args.new_track_thresh:
|
||
continue
|
||
track.activate(self.kalman_filter, self.frame_id)
|
||
activated_stracks.append(track)
|
||
|
||
first_finded.append(track)
|
||
|
||
# Step 5: Update state
|
||
for track in self.lost_stracks:
|
||
if self.frame_id - track.end_frame > self.max_time_lost:
|
||
track.mark_removed()
|
||
removed_stracks.append(track)
|
||
|
||
self.tracked_stracks = [t for t in self.tracked_stracks if t.state == TrackState.Tracked]
|
||
self.tracked_stracks = self.joint_stracks(self.tracked_stracks, activated_stracks)
|
||
self.tracked_stracks = self.joint_stracks(self.tracked_stracks, refind_stracks)
|
||
self.lost_stracks = self.sub_stracks(self.lost_stracks, self.tracked_stracks)
|
||
self.lost_stracks.extend(lost_stracks)
|
||
self.lost_stracks = self.sub_stracks(self.lost_stracks, self.removed_stracks)
|
||
self.tracked_stracks, self.lost_stracks = self.remove_duplicate_stracks(self.tracked_stracks, self.lost_stracks)
|
||
self.removed_stracks.extend(removed_stracks)
|
||
if len(self.removed_stracks) > 1000:
|
||
self.removed_stracks = self.removed_stracks[-999:] # clip remove stracks to 1000 maximum
|
||
|
||
'''x.tlbr have update by function:
|
||
@property
|
||
def tlwh(self):
|
||
'''
|
||
|
||
##================ 原算法输出
|
||
# output = np.asarray([x.tlbr.tolist() + [x.track_id, x.score, x.cls, x.frame_id, x.idx]
|
||
# for x in self.tracked_stracks if x.is_activated], dtype=np.float32)
|
||
|
||
## ===== write by WQG
|
||
output1 = [x.tlwh_to_tlbr(x._tlwh).tolist() + [x.track_id, x.score, x.cls, x.frame_id, x.idx]
|
||
for x in self.tracked_stracks if x.is_activated]
|
||
|
||
output2 = [x.tlwh_to_tlbr(x._tlwh).tolist() + [x.track_id, x.score, x.cls, x.frame_id, x.idx]
|
||
for x in first_finded if x.first_find]
|
||
|
||
output = np.asarray(output1+output2, dtype=np.float32)
|
||
|
||
return output
|
||
|
||
|
||
def get_result(self):
|
||
'''written by WQG'''
|
||
# =============================================================================
|
||
# activate_tracks = np.asarray([x.tlbr.tolist() + [x.track_id, x.score, x.cls, x.idx]
|
||
# for x in self.tracked_stracks if x.is_activated], dtype=np.float32)
|
||
#
|
||
# track_features = []
|
||
# =============================================================================
|
||
tracks = []
|
||
feats = []
|
||
for t in self.tracked_stracks:
|
||
if t.is_activated:
|
||
track = t.tlbr.tolist() + [t.track_id, t.score, t.cls, t.idx]
|
||
feat = t.curr_feature
|
||
|
||
tracks.append(track)
|
||
feats.append(feat)
|
||
|
||
tracks = np.asarray(tracks, dtype=np.float32)
|
||
|
||
return (tracks, feats)
|
||
|
||
|
||
def get_kalmanfilter(self):
|
||
"""Returns a Kalman filter object for tracking bounding boxes."""
|
||
return KalmanFilterXYAH()
|
||
|
||
def init_track(self, dets, scores, cls, img=None):
|
||
"""Initialize object tracking with detections and scores using STrack algorithm."""
|
||
return [STrack(xyxy, s, c) for (xyxy, s, c) in zip(dets, scores, cls)] if len(dets) else [] # detections
|
||
|
||
def get_dists(self, tracks, detections):
|
||
"""Calculates the distance between tracks and detections using IOU and fuses scores."""
|
||
dists = matching.iou_distance(tracks, detections)
|
||
# TODO: mot20
|
||
# if not self.args.mot20:
|
||
dists = matching.fuse_score(dists, detections)
|
||
return dists
|
||
def get_dists_1(self, tracks, detections):
|
||
"""Calculates the distance between tracks and detections using IOU and fuses scores."""
|
||
|
||
pass
|
||
|
||
|
||
def multi_predict(self, tracks):
|
||
"""Returns the predicted tracks using the YOLOv8 network."""
|
||
STrack.multi_predict(tracks)
|
||
|
||
def reset_id(self):
|
||
"""Resets the ID counter of STrack."""
|
||
STrack.reset_id()
|
||
|
||
@staticmethod
|
||
def joint_stracks(tlista, tlistb):
|
||
"""Combine two lists of stracks into a single one."""
|
||
exists = {}
|
||
res = []
|
||
for t in tlista:
|
||
exists[t.track_id] = 1
|
||
res.append(t)
|
||
for t in tlistb:
|
||
tid = t.track_id
|
||
if not exists.get(tid, 0):
|
||
exists[tid] = 1
|
||
res.append(t)
|
||
return res
|
||
|
||
@staticmethod
|
||
def sub_stracks(tlista, tlistb):
|
||
"""DEPRECATED CODE in https://github.com/ultralytics/ultralytics/pull/1890/
|
||
stracks = {t.track_id: t for t in tlista}
|
||
for t in tlistb:
|
||
tid = t.track_id
|
||
if stracks.get(tid, 0):
|
||
del stracks[tid]
|
||
return list(stracks.values())
|
||
"""
|
||
track_ids_b = {t.track_id for t in tlistb}
|
||
return [t for t in tlista if t.track_id not in track_ids_b]
|
||
|
||
@staticmethod
|
||
def remove_duplicate_stracks(stracksa, stracksb):
|
||
"""Remove duplicate stracks with non-maximum IOU distance."""
|
||
pdist = matching.iou_distance(stracksa, stracksb)
|
||
pairs = np.where(pdist < 0.15)
|
||
dupa, dupb = [], []
|
||
for p, q in zip(*pairs):
|
||
timep = stracksa[p].frame_id - stracksa[p].start_frame
|
||
timeq = stracksb[q].frame_id - stracksb[q].start_frame
|
||
if timep > timeq:
|
||
dupb.append(q)
|
||
else:
|
||
dupa.append(p)
|
||
resa = [t for i, t in enumerate(stracksa) if i not in dupa]
|
||
resb = [t for i, t in enumerate(stracksb) if i not in dupb]
|
||
return resa, resb
|