419 lines
17 KiB
Python
419 lines
17 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""
|
||
Created on Mon Mar 4 18:28:47 2024
|
||
|
||
@author: ym
|
||
"""
|
||
import cv2
|
||
import numpy as np
|
||
from scipy.spatial.distance import cdist
|
||
from sklearn.decomposition import PCA
|
||
from .dotracks import MoveState, Track
|
||
|
||
|
||
class backTrack(Track):
|
||
# boxes: [x1, y1, x2, y2, track_id, score, cls, frame_index, box_index]
|
||
# 0, 1, 2, 3, 4, 5, 6, 7, 8
|
||
def __init__(self, boxes, features, imgshape=(1024, 1280)):
|
||
|
||
super().__init__(boxes, features, imgshape)
|
||
|
||
'''该函数依赖项: self.cornpoints
|
||
MarginState: list, seven elements, 表示轨迹中boxes出现在图像的
|
||
[左上，右上，左中，右中，左下，右下底部]
|
||
'''
|
||
self.isCornpoint, self.MarginState = self.isimgborder()
|
||
|
||
'''该函数依赖项: self.isCornpoint，不能在父类中初始化'''
|
||
self.trajfeature()
|
||
|
||
|
||
'''静止点帧索引'''
|
||
# self.static_index = self.compute_static_fids()
|
||
|
||
'''运动点帧索引(运动帧两端的静止帧索引)'''
|
||
# self.moving_index = self.compute_dynamic_fids()
|
||
|
||
self.static_index, self.moving_index = self.compute_static_dynamic_fids()
|
||
|
||
'''该函数依赖项: self.cornpoints，定义 4 个商品位置变量：
|
||
self.Cent_isIncart, self.LB_isIncart, self.RB_isIncart
|
||
self.posState = self.Cent_isIncart+self.LB_isIncart+self.RB_isIncart'''
|
||
self.PositionState(camerType="back")
|
||
|
||
'''self.feature_ious = (incart_iou, outcart_iou, cartboarder_iou, maxbox_iou, minbox_iou)
|
||
self.incartrates = incartrates'''
|
||
self.compute_ious_feat()
|
||
|
||
# self.PCA()
|
||
|
||
|
||
|
||
def isimgborder(self, BoundPixel=10, BoundThresh=0.3):
|
||
|
||
x1, y1 = self.cornpoints[:,2], self.cornpoints[:,3],
|
||
x2, y2 = self.cornpoints[:,8], self.cornpoints[:,9]
|
||
|
||
condt1 = sum(abs(x1)<BoundPixel) / self.frnum > BoundThresh
|
||
condt2 = sum(abs(y1)<BoundPixel) / self.frnum > BoundThresh
|
||
condt3 = sum(abs(x2-self.imgshape[0])<BoundPixel) / self.frnum > BoundThresh
|
||
condt4 = sum(abs(y2-self.imgshape[1])<BoundPixel) / self.frnum > BoundThresh
|
||
|
||
condt = condt1 or condt2 or condt3 or condt4
|
||
isCornpoint = False
|
||
if condt:
|
||
isCornpoint = True
|
||
|
||
condtA = condt1 and condt2
|
||
condtB = condt3 and condt2
|
||
condtC = condt1 and not condt2 and not condt4
|
||
condtD = condt3 and not condt2 and not condt4
|
||
condtE = condt1 and condt4
|
||
condtF = condt3 and condt4
|
||
condtG = condt4 and not condt1 and not condt3
|
||
MarginState = [condtA, condtB, condtC, condtD, condtE, condtF, condtG]
|
||
|
||
return isCornpoint, MarginState
|
||
|
||
|
||
# =============================================================================
|
||
# def PositionState(self, camerType="back"):
|
||
# '''
|
||
# 已迁移至基类
|
||
# camerType: back, 后置摄像头
|
||
# front, 前置摄像头
|
||
# '''
|
||
# if camerType=="front":
|
||
# incart = cv2.imread("./shopcart/cart_tempt/incart.png", cv2.IMREAD_GRAYSCALE)
|
||
# outcart = cv2.imread("./shopcart/cart_tempt/outcart.png", cv2.IMREAD_GRAYSCALE)
|
||
# else:
|
||
# incart = cv2.imread("./shopcart/cart_tempt/incart_ftmp.png", cv2.IMREAD_GRAYSCALE)
|
||
# outcart = cv2.imread("./shopcart/cart_tempt/outcart_ftmp.png", cv2.IMREAD_GRAYSCALE)
|
||
#
|
||
# xc, yc = self.cornpoints[:,0].clip(0,self.imgshape[0]-1).astype(np.int64), self.cornpoints[:,1].clip(0,self.imgshape[1]-1).astype(np.int64)
|
||
# x1, y1 = self.cornpoints[:,6].clip(0,self.imgshape[0]-1).astype(np.int64), self.cornpoints[:,7].clip(0,self.imgshape[1]-1).astype(np.int64)
|
||
# x2, y2 = self.cornpoints[:,8].clip(0,self.imgshape[0]-1).astype(np.int64), self.cornpoints[:,9].clip(0,self.imgshape[1]-1).astype(np.int64)
|
||
#
|
||
# # print(self.tid)
|
||
# Cent_inCartnum = np.count_nonzero(incart[(yc, xc)])
|
||
# LB_inCartnum = np.count_nonzero(incart[(y1, x1)])
|
||
# RB_inCartnum = np.count_nonzero(incart[(y2, x2)])
|
||
#
|
||
# Cent_outCartnum = np.count_nonzero(outcart[(yc, xc)])
|
||
# LB_outCartnum = np.count_nonzero(outcart[(y1, x1)])
|
||
# RB_outCartnum = np.count_nonzero(outcart[(y2, x2)])
|
||
#
|
||
# '''Track完全在车内：左下角点、右下角点与 outcart 的交集为 0'''
|
||
# self.isWholeInCart = False
|
||
# if LB_outCartnum + RB_outCartnum == 0:
|
||
# self.isWholeInCart = True
|
||
#
|
||
# '''Track完全在车外：左下角点、中心点与 incart 的交集为 0
|
||
# 右下角点、中心点与 incart 的交集为 0
|
||
# '''
|
||
# self.isWholeOutCart = False
|
||
# if Cent_inCartnum + LB_inCartnum == 0 or Cent_inCartnum + RB_inCartnum == 0:
|
||
# self.isWholeOutCart = True
|
||
#
|
||
#
|
||
# self.Cent_isIncart = False
|
||
# self.LB_isIncart = False
|
||
# self.RB_isIncart = False
|
||
# if Cent_inCartnum: self.Cent_isIncart = True
|
||
# if LB_inCartnum: self.LB_isIncart = True
|
||
# if RB_inCartnum: self.RB_isIncart = True
|
||
#
|
||
# self.posState = self.Cent_isIncart+self.LB_isIncart+self.RB_isIncart
|
||
# =============================================================================
|
||
|
||
|
||
|
||
|
||
def PCA(self):
|
||
self.pca = PCA()
|
||
|
||
X = self.cornpoints[:, 0:2]
|
||
self.pca.fit(X)
|
||
|
||
|
||
|
||
|
||
|
||
def compute_ious_feat(self):
|
||
'''输出：
|
||
self.feature_ious = (incart_iou, outcart_iou, cartboarder_iou, maxbox_iou, minbox_iou)
|
||
self.incartrates = incartrates，
|
||
其中：
|
||
boxes流：track中所有boxes形成的轨迹图，可分为三部分：incart, outcart, cartboarder
|
||
incart_iou, outcart_iou, cartboarder_iou：各部分和 boxes流的 iou。
|
||
incart_iou = 0，track在购物车外，
|
||
outcart_iou = 0，track在购物车内，也可能是通过左下角、右下角置入购物车，
|
||
maxbox_iou, minbox_iou：track中最大、最小 box 和boxes流的iou，二者差值越小，越接近 1，表明track的运动型越小。
|
||
incartrates: 各box和incart的iou时序，由小变大，反应的是置入过程，由大变小，反应的是取出过程
|
||
'''
|
||
incart = cv2.imread("./shopcart/cart_tempt/incart.png", cv2.IMREAD_GRAYSCALE)
|
||
outcart = cv2.imread("./shopcart/cart_tempt/outcart.png", cv2.IMREAD_GRAYSCALE)
|
||
cartboarder = cv2.imread("./shopcart/cart_tempt/cartboarder.png", cv2.IMREAD_GRAYSCALE)
|
||
|
||
incartrates = []
|
||
temp = np.zeros(incart.shape, np.uint8)
|
||
maxarea, minarea = 0, self.imgshape[0]*self.imgshape[1]
|
||
for i in range(self.frnum):
|
||
# x, y, w, h = self.boxes[i, 0:4]
|
||
|
||
x = (self.boxes[i, 2] + self.boxes[i, 0]) / 2
|
||
w = (self.boxes[i, 2] - self.boxes[i, 0]) / 2
|
||
y = (self.boxes[i, 3] + self.boxes[i, 1]) / 2
|
||
h = (self.boxes[i, 3] - self.boxes[i, 1]) / 2
|
||
|
||
|
||
if w*h > maxarea: maxarea = w*h
|
||
if w*h < minarea: minarea = w*h
|
||
cv2.rectangle(temp, (int(x-w/2), int(y-h/2)), (int(x+w/2), int(y+h/2)), 255, cv2.FILLED)
|
||
|
||
temp1 = np.zeros(incart.shape, np.uint8)
|
||
cv2.rectangle(temp1, (int(x-w/2), int(y-h/2)), (int(x+w/2), int(y+h/2)), 255, cv2.FILLED)
|
||
temp2 = cv2.bitwise_and(incart, temp1)
|
||
inrate = cv2.countNonZero(temp1)/(w*h)
|
||
incartrates.append(inrate)
|
||
|
||
isincart = cv2.bitwise_and(incart, temp)
|
||
isoutcart = cv2.bitwise_and(outcart, temp)
|
||
iscartboarder = cv2.bitwise_and(cartboarder, temp)
|
||
|
||
num_temp = cv2.countNonZero(temp)
|
||
num_incart = cv2.countNonZero(isincart)
|
||
num_outcart = cv2.countNonZero(isoutcart)
|
||
num_cartboarder = cv2.countNonZero(iscartboarder)
|
||
|
||
incart_iou = num_incart/num_temp
|
||
outcart_iou = num_outcart/num_temp
|
||
cartboarder_iou = num_cartboarder/num_temp
|
||
maxbox_iou = maxarea/num_temp
|
||
minbox_iou = minarea/num_temp
|
||
|
||
self.feature_ious = (incart_iou, outcart_iou, cartboarder_iou, maxbox_iou, minbox_iou)
|
||
self.incartrates = incartrates
|
||
|
||
|
||
|
||
|
||
def compute_static_fids(self, thresh1 = 12, thresh2 = 3):
|
||
'''
|
||
计算 track 的轨迹中相对处于静止状态的轨迹点的（start_frame_id, end_frame_id）
|
||
thresh1: 相邻两帧目标中心点是否静止的的阈值，以像素为单位，
|
||
thresh2: 连续捕捉到目标处于静止状态的帧数,当 thresh2 = 3时,至少连续 4个点,
|
||
产生3个相邻点差值均小于 thresh1 时，判定为连续静止.
|
||
处理过程中利用了插值技术，因此start、end并非 self.boxes 中对应的帧索引
|
||
'''
|
||
|
||
BoundPixel = 8
|
||
x1, y1 = self.cornpoints[:,2], self.cornpoints[:,3],
|
||
x2, y2 = self.cornpoints[:,8], self.cornpoints[:,9]
|
||
cont1 = sum(abs(x1)<BoundPixel) > 3
|
||
# cont2 = sum(abs(y1)<BoundPixel) > 3
|
||
cont3 = sum(abs(x2-self.imgshape[0])<BoundPixel) > 3
|
||
# cont4 = sum(abs(y2-self.imgshape[1])<BoundPixel) > 3
|
||
cont = not(cont1 or cont3)
|
||
|
||
## ============== 下一步，启用中心点，选择具有最小运动幅度的角点作为参考点
|
||
static_index = []
|
||
if self.frnum>=2 and cont:
|
||
x1 = self.boxes[1:,7]
|
||
x2 = [i for i in range(int(min(x1)), int(max(x1)+1))]
|
||
dist_adjc = np.interp(x2, x1, self.trajmin)
|
||
|
||
|
||
# dist_adjc = self.trajmin
|
||
|
||
static_thresh = (dist_adjc < thresh1)[:, None].astype(np.uint8)
|
||
static_cnts, _ = cv2.findContours(static_thresh, cv2.RETR_LIST, cv2.CHAIN_APPROX_NONE)
|
||
|
||
for cnt in static_cnts:
|
||
_, start, _, num = cv2.boundingRect(cnt)
|
||
end = start + num
|
||
if num < thresh2:
|
||
continue
|
||
static_index.append((start, end))
|
||
|
||
static_index = np.array(static_index)
|
||
if static_index.size:
|
||
indx = np.argsort(static_index[:, 0])
|
||
static_index = static_index[indx]
|
||
|
||
return static_index
|
||
|
||
def compute_dynamic_fids(self, thresh1 = 12, thresh2 = 3):
|
||
'''
|
||
计算 track 的轨迹中运动轨迹点的（start_frame_id, end_frame_id）
|
||
thresh1: 相邻两帧目标中心点是否运动的阈值，以像素为单位，
|
||
thresh2: 连续捕捉到目标连续运动的帧数
|
||
目标：
|
||
1. 计算轨迹方向
|
||
2. 计算和手部运动的关联性
|
||
'''
|
||
moving_index = []
|
||
if self.frnum>=2:
|
||
x1 = self.boxes[1:,7]
|
||
x2 = [i for i in range(int(min(x1)), int(max(x1)+1))]
|
||
dist_adjc = np.interp(x2, x1, self.trajmin)
|
||
|
||
moving_thresh = (dist_adjc >= thresh1)[:, None].astype(np.uint8)
|
||
moving_cnts, _ = cv2.findContours(moving_thresh, cv2.RETR_LIST, cv2.CHAIN_APPROX_NONE)
|
||
|
||
for cnt in moving_cnts:
|
||
_, start, _, num = cv2.boundingRect(cnt)
|
||
if num < thresh2:
|
||
continue
|
||
end = start + num
|
||
moving_index.append((start, end))
|
||
|
||
# =============================================================================
|
||
# '''========= 输出帧id，不太合适 ========='''
|
||
# moving_fids = []
|
||
# for i in range(len(moving_index)):
|
||
# i1, i2 = moving_index[i]
|
||
# fid1, fid2 = boxes[i1, 7], boxes[i2, 7]
|
||
# moving_fids.append([fid1, fid2])
|
||
# moving_fids = np.array(moving_fids)
|
||
# =============================================================================
|
||
moving_index = np.array(moving_index)
|
||
if moving_index.size:
|
||
indx = np.argsort(moving_index[:, 0])
|
||
moving_index = moving_index[indx]
|
||
|
||
return moving_index
|
||
|
||
def compute_static_dynamic_fids(self):
|
||
|
||
if self.MarginState[0] or self.MarginState[2]:
|
||
idx1 = 4
|
||
elif self.MarginState[1] or self.MarginState[3]:
|
||
idx1 = 3
|
||
elif self.MarginState[4]:
|
||
idx1 = 2
|
||
elif self.MarginState[5]:
|
||
idx1 = 1
|
||
elif self.MarginState[6]:
|
||
if self.trajlens[1] < self.trajlens[2]:
|
||
idx1 = 1
|
||
else:
|
||
idx1 = 2
|
||
else:
|
||
idx1 = self.trajlens.index(min(self.trajlens))
|
||
|
||
# idx1 = self.trajlens.index(min(self.trajlens))
|
||
trajmin = self.trajectory[idx1]
|
||
|
||
static, dynamic = self.pt_state_fids(trajmin)
|
||
|
||
static = np.array(static)
|
||
dynamic = np.array(dynamic)
|
||
|
||
if static.size:
|
||
indx = np.argsort(static[:, 0])
|
||
static = static[indx]
|
||
if dynamic.size:
|
||
indx = np.argsort(dynamic[:, 0])
|
||
dynamic = dynamic[indx]
|
||
|
||
return static, dynamic
|
||
|
||
|
||
|
||
|
||
# =============================================================================
|
||
# static_dynamic_fids = []
|
||
# for traj in self.trajectory:
|
||
# static, dynamic = self.pt_state_fids(traj)
|
||
# static_dynamic_fids.append((static, dynamic))
|
||
#
|
||
# return static_dynamic_fids
|
||
# =============================================================================
|
||
|
||
def is_static(self):
|
||
'''静态情况 1: 目标关键点最小相对运动轨迹 < 0.2, 指标值偏大
|
||
TrajFeat = [trajlen_min, trajlen_max,
|
||
trajdist_min, trajdist_max,
|
||
trajlen_rate, trajdist_rate]
|
||
'''
|
||
# print(f"TrackID: {self.tid}")
|
||
|
||
boxes = self.boxes
|
||
|
||
'''静态情况 1: '''
|
||
condt1 = self.TrajFeat[5] < 0.2 or self.TrajFeat[3] < 120
|
||
|
||
'''静态情况 2: 目标初始状态为静止，适当放宽关键点最小相对运动轨迹 < 0.5'''
|
||
condt2 = self.static_index.size > 0 \
|
||
and self.static_index[0, 0] <= 2 \
|
||
and self.static_index[0, 1] >= 5 \
|
||
and self.TrajFeat[5] < 0.5 \
|
||
and self.TrajFeat[1] < 240 \
|
||
and self.isWholeInCart
|
||
# and self.posState >= 2
|
||
# and self.TrajFeat[0] < 240 \
|
||
|
||
|
||
'''静态情况 3: 目标初始状态和最终状态均为静止'''
|
||
condt3 = self.static_index.shape[0] >= 2 \
|
||
and self.static_index[0, 0] <= 2 \
|
||
and self.static_index[0, 1] >= 5 \
|
||
and self.static_index[-1, 1] >= self.frnum-3 \
|
||
and self.TrajFeat[1] < 240 \
|
||
and self.isWholeInCart
|
||
# and self.posState >= 2
|
||
# and self.TrajFeat[0] < 240 \
|
||
|
||
condt4 = self.static_index.shape[0] >= 2 \
|
||
and self.static_index[0, 0] <= 2 \
|
||
and self.static_index[0, 1] >= 6 \
|
||
and self.static_index[-1, 0] <= self.frnum-5 \
|
||
and self.static_index[-1, 1] >= self.frnum-2
|
||
|
||
|
||
condt = condt1 or condt2 or condt3 or condt4
|
||
|
||
return condt
|
||
|
||
# =============================================================================
|
||
# track1 = [t for t in tracks if t.TrajFeat[5] < 0.2
|
||
# or t.TrajFeat[3] < 120
|
||
# ]
|
||
#
|
||
# track2 = [t for t in tracks if t.static_index.size > 0
|
||
# and t.static_index[0, 0] <= 2
|
||
# and t.TrajFeat[5] < 0.5]
|
||
#
|
||
# track3 = [t for t in tracks if t.static_index.shape[0] >= 2
|
||
# and t.static_index[0, 0] <= 2
|
||
# and t.static_index[-1, 1] >= t.frnum-3]
|
||
#
|
||
# track12 = self.join_tracks(track1, track2)
|
||
#
|
||
# '''提取静止状态的 track'''
|
||
# static_tracks = self.join_tracks(track12, track3)
|
||
# self.Static.extend(static_tracks)
|
||
#
|
||
# =============================================================================
|
||
|
||
def is_OutTrack(self):
|
||
if self.posState <= 1:
|
||
isout = True
|
||
else:
|
||
isout = False
|
||
return isout
|
||
|
||
|
||
|
||
|
||
def compute_distance(self):
|
||
pass
|
||
|
||
|
||
def move_start_fid(self):
|
||
pass
|
||
|
||
|
||
def move_end_fid(self):
|
||
pass |