Files
detecttracking/contrast/onsite_contrast_pr.py
王庆刚 9b5b135fa3 20250313
2025-03-13 15:36:29 +08:00

570 lines
20 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

# -*- coding: utf-8 -*-
"""
Created on Wed Sep 11 11:57:30 2024
contrast_pr:
直接利用测试数据中的 data 文件进行 1:1、1:SN、1:n 性能评估
test_compare:
永辉现场试验输出数据的 1:1 性能评估
适用于202410前数据保存版本的需调用 OneToOneCompare.txt
@author: ym
"""
import os
import numpy as np
from pathlib import Path
import matplotlib.pyplot as plt
import sys
sys.path.append(r"D:\DetectTracking")
from tracking.utils.read_data import read_similar
def read_one2one_data(filepath):
simiList = []
with open(filepath, 'r', encoding='utf-8') as f:
lines = f.readlines()
split_flag = False
simi_dict = {}
for i, line in enumerate(lines):
line = line.strip()
if not line:
if len(simi_dict): simiList.append(simi_dict)
simi_dict = {}
continue
label = line.split(':')[0].strip()
value = line.split(':')[1].strip()
if label.find("SeqDir") >= 0:
simi_dict["SeqDir"] = value
if label.isdigit() and len(label) >= 8:
simi_max, simi_min = value.strip(',').split('.')
simi_dict["barcode"] = label
simi_dict["simi_max"] = float(simi_max) / 1000
simi_dict["simi_min"] = float(simi_min) / 1000
if len(simi_dict): simiList.append(simi_dict)
return simiList
def plot_pr_curve(matrix):
simimax, simimean = [], []
need_analysis = []
for simidict in matrix:
simimax.append(simidict["simi_max"])
simimean.append(simidict["simi_min"])
if simidict["simi_max"]>0.6:
need_analysis.append(simidict)
simimax = np.array(simimax)
simimean = np.array(simimean)
TPFN_max = len(simimax)
TPFN_mean = len(simimean)
fig, axs = plt.subplots(2, 1)
axs[0].hist(simimax, bins=60, edgecolor='black')
axs[0].set_xlim([-0.2, 1])
axs[0].set_title(f'Same Barcode, Num: {TPFN_max}')
axs[1].hist(simimean, bins=60, edgecolor='black')
axs[1].set_xlim([-0.2, 1])
axs[1].set_title(f'Cross Barcode, Num: {TPFN_mean}')
# plt.savefig(f'./result/{file}_hist.png') # svg, png, pdf
Recall_Neg = []
Thresh = np.linspace(-0.2, 1, 100)
for th in Thresh:
TN = np.sum(simimax < th)
Recall_Neg.append(TN/TPFN_max)
fig, ax = plt.subplots()
ax.plot(Thresh, Recall_Neg, 'b', label='Recall_Pos: TP/TPFN')
ax.set_xlim([0, 1])
ax.set_ylim([0, 1])
ax.grid(True)
ax.set_title('Positive recall')
ax.set_xlabel(f"Num: {TPFN_max}")
ax.legend()
plt.show()
# plt.savefig(f'./result/{file}_pr.png') # svg, png, pdf
print("Have done!")
pass
def test_compare():
filepaths = [r"\\192.168.1.28\share\测试_202406\0913_扫A放B\0913_1\OneToOneCompare.txt",
r"\\192.168.1.28\share\测试_202406\0913_扫A放B\0913_2\OneToOneCompare.txt",
r"\\192.168.1.28\share\测试_202406\0914_扫A放B\0914_1\OneToOneCompare.txt",
r"\\192.168.1.28\share\测试_202406\0914_扫A放B\0914_2\OneToOneCompare.txt"
]
simiList = []
for fp in filepaths:
slist = read_one2one_data(fp)
simiList.extend(slist)
plot_pr_curve(simiList)
def contrast_pr(evtPaths):
'''
1:1
'''
evtpaths = []
# date_ = ['2025-3-4_1', '2025-3-5_1', '2025-3-5_2']
# for dt in date_:
# paths = Path(evtPaths) / dt
abc = []
for p in Path(evtPaths).iterdir():
condt1 = p.is_dir()
condt2 = len(p.name.split('_'))>=2
condt3 = len(p.name.split('_')[-1])>=8
condt4 = p.name.split('_')[-1].isdigit()
if condt1 and condt2 and condt3 and condt4:
evtpaths.append(p)
elif p.is_dir():
abc.append(p.stem)
# evtpaths = [p for p in paths.iterdir() if p.is_dir() and len(p.name.split('_'))>=2 and len(p.name.split('_')[-1])>8]
# evtpaths = [p for p in paths.iterdir() if p.is_dir()]
alg_times = []
events, similars = [], []
##===================================== 扫A放A, 扫A放B场景()
one2oneAA, one2oneAB = [], []
one2SNAA, one2SNAB = [], []
##===================================== 应用于 11
_tp_events, _fn_events, _fp_events, _tn_events = [], [], [], []
_tp_simi, _fn_simi, _tn_simi, _fp_simi = [], [], [], []
##===================================== 应用于 1SN
tp_events, fn_events, fp_events, tn_events = [], [], [], []
tp_simi, fn_simi, tn_simi, fp_simi = [], [], [], []
##===================================== 应用于1:n
tpevents, fnevents, fpevents, tnevents = [], [], [], []
tpsimi, fnsimi, tnsimi, fpsimi = [], [], [], []
##===================================== barcodes总数、比对错误事件
bcdList = []
one2onePath, one2onePath1 = [], []
one2SNPath, one2SNPath1 = [], []
one2nPath = []
errorFile_one2one, errorFile_one2SN, errorFile_one2n = [], [], []
errorFile = []
for path in evtpaths:
barcode = path.stem.split('_')[-1]
datapath = path.joinpath('process.data')
if not barcode.isdigit() or len(barcode)<8: continue
if not datapath.is_file(): continue
bcdList.append(barcode)
try:
SimiDict = read_similar(datapath)
except Exception as e:
print(f"{path.stem}, Error: {e}")
'''放入为 1:1相似度取最大值取出时为 1:SN, 相似度取均值'''
one2one = SimiDict['one2one']
one2SN = SimiDict['one2SN']
one2n = SimiDict['one2n']
if len(one2one)+len(one2SN)+len(one2n) == 0:
errorFile.append(path.stem)
dtime = SimiDict["algroStartToEnd"]
if dtime >= 0:
alg_times.append((dtime, path.stem))
'''================== 0. 1:1 ==================='''
barcodes, similars = [], []
barcodes_ = []
for dt in one2one:
one2onePath.append((path.stem))
if dt['similar']==0:
one2onePath1.append((path.stem))
continue
barcodes.append(dt['barcode'])
similars.append(dt['similar'])
barcodes_.append(path.stem)
if len(barcodes)==len(similars) and len(barcodes)!=0:
## 扫A放A, 扫A放B场景
simAA = [similars[i] for i in range(len(barcodes)) if barcodes[i]==barcode]
simAB = [similars[i] for i in range(len(barcodes)) if barcodes[i]!=barcode]
one2oneAA.extend(simAA)
one2oneAB.extend(simAB)
## 相似度排序barcode相等且排名第一为TP适用于多的barcode相似度比较
max_idx = similars.index(max(similars))
max_sim = similars[max_idx]
# max_bcd = barcodes[max_idx]
for i in range(len(one2one)):
bcd, simi = barcodes[i], similars[i]
if bcd==barcode and simi==max_sim:
_tp_simi.append(simi)
_tp_events.append(path.stem)
elif bcd==barcode and simi!=max_sim:
_fn_simi.append(simi)
_fn_events.append(path.stem)
elif bcd!=barcode and simi!=max_sim:
_tn_simi.append(simi)
_tn_events.append(path.stem)
elif bcd!=barcode and simi==max_sim and barcode in barcodes:
_fp_simi.append(simi)
_fp_events.append(path.stem)
else:
errorFile_one2one.append(path.stem)
elif len(one2SN)+len(one2n) == 0:
errorFile_one2one.append(path.stem)
'''================== 2. 取出场景下的 1 : Small N ==================='''
barcodes, similars = [], []
for dt in one2SN:
barcodes.append(dt['barcode'])
similars.append(dt['similar'])
if len(barcodes)==len(similars) and len(barcodes)!=0:
## 扫A放A, 扫A放B场景
simAA = [similars[i] for i in range(len(barcodes)) if barcodes[i]==barcode]
simAB = [similars[i] for i in range(len(barcodes)) if barcodes[i]!=barcode]
one2SNAA.extend(simAA)
one2SNAB.extend(simAB)
one2SNPath.append(path.stem)
if len(simAA)==0:
errorFile_one2SN.append(path.stem)
## 相似度排序barcode相等且排名第一为TP适用于多的barcode相似度比较
max_idx = similars.index(max(similars))
max_sim = similars[max_idx]
# max_bcd = barcodes[max_idx]
for i in range(len(one2SN)):
bcd, simi = barcodes[i], similars[i]
if bcd==barcode and simi==max_sim:
tp_simi.append(simi)
tp_events.append(path.stem)
elif bcd==barcode and simi!=max_sim:
fn_simi.append(simi)
fn_events.append(path.stem)
elif bcd!=barcode and simi!=max_sim:
tn_simi.append(simi)
tn_events.append(path.stem)
elif bcd!=barcode and simi==max_sim and barcode in barcodes:
fp_simi.append(simi)
fp_events.append(path.stem)
else:
errorFile_one2SN.append(path.stem)
'''===================== 3. 取出场景下的 1:n ========================'''
events, evt_barcodes, evt_similars, evt_types = [], [], [], []
for dt in one2n:
events.append(dt["event"])
evt_barcodes.append(dt["barcode"])
evt_similars.append(dt["similar"])
evt_types.append(dt["type"])
if len(events)==len(evt_barcodes)==len(evt_similars)==len(evt_types) and len(events)>0:
if not barcode in evt_barcodes:
errorFile_one2n.append(path.stem)
continue
if len(barcodes_):
print("do")
one2nPath.append(path.stem)
maxsim = evt_similars[evt_similars.index(max(evt_similars))]
for i in range(len(one2n)):
bcd, simi = evt_barcodes[i], evt_similars[i]
if bcd==barcode and simi==maxsim:
tpsimi.append(simi)
tpevents.append(path.stem)
elif bcd==barcode and simi!=maxsim:
fnsimi.append(simi)
fnevents.append(path.stem)
elif bcd!=barcode and simi!=maxsim:
tnsimi.append(simi)
tnevents.append(path.stem)
elif bcd!=barcode and simi==maxsim and barcode in evt_barcodes:
fpsimi.append(simi)
fpevents.append(path.stem)
else:
errorFile_one2n.append(path.stem)
'''命名规则:
1:1 (max) 1:1 (max) 1:n 1:N
_TP TP_ TP TPX
_PPrecise PPrecise_ PPrecise PPreciseX
tpsimi tp_simi
'''
''' 1:1 数据存储, 相似度计算方式:最大值、均值'''
_PPrecise, _PRecall = [], []
_NPrecise, _NRecall = [], []
PPrecise_, PRecall_ = [], []
NPrecise_, NRecall_ = [], []
''' 1:SN 数据存储,需根据相似度排序'''
PPreciseX, PRecallX = [], []
NPreciseX, NRecallX = [], []
''' 1:n 数据存储,需根据相似度排序'''
PPrecise, PRecall = [], []
NPrecise, NRecall = [], []
Thresh = np.linspace(-0.2, 1, 100)
for th in Thresh:
'''(Precise, Recall) 计算方式, 若 1:1 与 1:SN 相似度选择方式相同,则可以合并'''
'''===================================== 1:1 最大值'''
_TP = sum(np.array(one2oneAA) >= th)
_FP = sum(np.array(one2oneAB) >= th)
_FN = sum(np.array(one2oneAA) < th)
_TN = sum(np.array(one2oneAB) < th)
_PPrecise.append(_TP/(_TP+_FP+1e-6))
_PRecall.append(_TP/(_TP+_FN+1e-6))
_NPrecise.append(_TN/(_TN+_FN+1e-6))
_NRecall.append(_TN/(_TN+_FP+1e-6))
'''===================================== 1:SN 均值'''
TP_ = sum(np.array(one2SNAA) >= th)
FP_ = sum(np.array(one2SNAB) >= th)
FN_ = sum(np.array(one2SNAA) < th)
TN_ = sum(np.array(one2SNAB) < th)
PPrecise_.append(TP_/(TP_+FP_+1e-6))
PRecall_.append(TP_/(len(one2SNAA)+1e-6))
NPrecise_.append(TN_/(TN_+FN_+1e-6))
NRecall_.append(TN_/(len(one2SNAB)+1e-6))
'''适用于 (Precise, Recall) 计算方式多个相似度计算并排序barcode相等且排名第一为 TP '''
'''===================================== 1:SN '''
TPX = sum(np.array(tp_simi) >= th)
FPX = sum(np.array(fp_simi) >= th)
FNX = sum(np.array(fn_simi) < th)
TNX = sum(np.array(tn_simi) < th)
PPreciseX.append(TPX/(TPX+FPX+1e-6))
PRecallX.append(TPX/(TPX+FNX+1e-6))
NPreciseX.append(TNX/(TNX+FNX+1e-6))
NRecallX.append(TNX/(TNX+FPX+1e-6))
'''===================================== 1:n'''
TP = sum(np.array(tpsimi) >= th)
FP = sum(np.array(fpsimi) >= th)
FN = sum(np.array(fnsimi) < th)
TN = sum(np.array(tnsimi) < th)
PPrecise.append(TP/(TP+FP+1e-6))
PRecall.append(TP/(TP+FN+1e-6))
NPrecise.append(TN/(TN+FN+1e-6))
NRecall.append(TN/(TN+FP+1e-6))
algtime = []
for tm, _ in alg_times:
algtime.append(tm)
fig, ax = plt.subplots()
ax.hist(np.array(algtime), bins=100, edgecolor='black')
ax.set_title('Algorthm Spend Time')
ax.set_xlabel(f"Event Num: {len(alg_times)}")
plt.show()
'''1. ============================= 1:1 最大值方案 曲线'''
fig, ax = plt.subplots()
ax.plot(Thresh, _PPrecise, 'r', label='Precise_Pos: TP/TPFP')
ax.plot(Thresh, _PRecall, 'b', label='Recall_Pos: TP/TPFN')
ax.plot(Thresh, _NPrecise, 'g', label='Precise_Neg: TN/TNFP')
ax.plot(Thresh, _NRecall, 'c', label='Recall_Neg: TN/TNFN')
ax.set_xlim([0, 1])
ax.set_ylim([0, 1])
ax.set_xticks(np.arange(0, 1, 0.1))
ax.set_yticks(np.arange(0, 1, 0.1))
ax.grid(True, linestyle='--')
ax.set_title('1:1 Precise & Recall')
ax.set_xlabel(f"Event Num: {len(one2oneAA)+len(one2oneAB)}")
ax.legend()
plt.show()
## ============================= 1:1 最大值方案 直方图'''
fig, axes = plt.subplots(2, 1)
axes[0].hist(np.array(one2oneAA), bins=60, edgecolor='black')
axes[0].set_xlim([-0.2, 1])
axes[0].set_title('AA')
axes[1].hist(np.array(one2oneAB), bins=60, edgecolor='black')
axes[1].set_xlim([-0.2, 1])
axes[1].set_title('BB')
plt.show()
'''2. ============================= 1:1 均值方案 曲线'''
# fig, ax = plt.subplots()
# ax.plot(Thresh, PPrecise_, 'r', label='Precise_Pos: TP/TPFP')
# ax.plot(Thresh, PRecall_, 'b', label='Recall_Pos: TP/TPFN')
# ax.plot(Thresh, NPrecise_, 'g', label='Precise_Neg: TN/TNFP')
# ax.plot(Thresh, NRecall_, 'c', label='Recall_Neg: TN/TNFN')
# ax.set_xlim([0, 1])
# ax.set_ylim([0, 1])
# ax.grid(True)
# ax.set_title('1:1 Precise & Recall')
# ax.set_xlabel(f"Event Num: {len(one2SNAA)}")
# ax.legend()
# plt.show()
# ## ============================= 1:1 均值方案 直方图'''
# fig, axes = plt.subplots(2, 1)
# axes[0].hist(np.array(one2SNAA), bins=60, edgecolor='black')
# axes[0].set_xlim([-0.2, 1])
# axes[0].set_title('AA')
# axes[0].set_xlabel(f"Event Num: {len(one2SNAA)}")
# axes[1].hist(np.array(one2SNAB), bins=60, edgecolor='black')
# axes[1].set_xlim([-0.2, 1])
# axes[1].set_title('BB')
# axes[1].set_xlabel(f"Event Num: {len(one2SNAB)}")
# plt.show()
''''3. ============================= 1:SN 曲线'''
fig, ax = plt.subplots()
ax.plot(Thresh, PPreciseX, 'r', label='Precise_Pos: TP/TPFP')
ax.plot(Thresh, PRecallX, 'b', label='Recall_Pos: TP/TPFN')
ax.plot(Thresh, NPreciseX, 'g', label='Precise_Neg: TN/TNFP')
ax.plot(Thresh, NRecallX, 'c', label='Recall_Neg: TN/TNFN')
ax.set_xlim([0, 1])
ax.set_ylim([0, 1])
ax.set_xticks(np.arange(0, 1, 0.1))
ax.set_yticks(np.arange(0, 1, 0.1))
ax.grid(True, linestyle='--')
ax.set_title('1:SN Precise & Recall')
ax.set_xlabel(f"Event Num: {len(one2SNAA)}")
ax.legend()
plt.show()
## ============================= 1:N 展厅 直方图'''
fig, axes = plt.subplots(2, 2)
axes[0, 0].hist(tp_simi, bins=60, edgecolor='black')
axes[0, 0].set_xlim([-0.2, 1])
axes[0, 0].set_title(f'TP({len(tp_simi)})')
axes[0, 1].hist(fp_simi, bins=60, edgecolor='black')
axes[0, 1].set_xlim([-0.2, 1])
axes[0, 1].set_title(f'FP({len(fp_simi)})')
axes[1, 0].hist(tn_simi, bins=60, edgecolor='black')
axes[1, 0].set_xlim([-0.2, 1])
axes[1, 0].set_title(f'TN({len(tn_simi)})')
axes[1, 1].hist(fn_simi, bins=60, edgecolor='black')
axes[1, 1].set_xlim([-0.2, 1])
axes[1, 1].set_title(f'FN({len(fn_simi)})')
plt.show()
'''4. ============================= 1:n 曲线,'''
fig, ax = plt.subplots()
ax.plot(Thresh, PPrecise, 'r', label='Precise_Pos: TP/TPFP')
ax.plot(Thresh, PRecall, 'b', label='Recall_Pos: TP/TPFN')
ax.plot(Thresh, NPrecise, 'g', label='Precise_Neg: TN/TNFP')
ax.plot(Thresh, NRecall, 'c', label='Recall_Neg: TN/TNFN')
ax.set_xlim([0, 1])
ax.set_ylim([0, 1])
ax.set_xticks(np.arange(0, 1, 0.1))
ax.set_yticks(np.arange(0, 1, 0.1))
ax.grid(True, linestyle='--')
ax.set_title('1:n Precise & Recall')
ax.set_xlabel(f"Event Num: {len(tpsimi)+len(fnsimi)}")
ax.legend()
plt.show()
## ============================= 1:n 直方图'''
fig, axes = plt.subplots(2, 2)
axes[0, 0].hist(tpsimi, bins=60, edgecolor='black')
axes[0, 0].set_xlim([-0.2, 1])
axes[0, 0].set_title(f'TP({len(tpsimi)})')
axes[0, 1].hist(fpsimi, bins=60, edgecolor='black')
axes[0, 1].set_xlim([-0.2, 1])
axes[0, 1].set_title(f'FP({len(fpsimi)})')
axes[1, 0].hist(tnsimi, bins=60, edgecolor='black')
axes[1, 0].set_xlim([-0.2, 1])
axes[1, 0].set_title(f'TN({len(tnsimi)})')
axes[1, 1].hist(fnsimi, bins=60, edgecolor='black')
axes[1, 1].set_xlim([-0.2, 1])
axes[1, 1].set_title(f'FN({len(fnsimi)})')
plt.show()
# fpsnErrFile = str(paths.joinpath("one2SN_Error.txt"))
# with open(fpsnErrFile, "w") as file:
# for item in fp_events:
# file.write(item + "\n")
# fpErrFile = str(paths.joinpath("one2n_Error.txt"))
# with open(fpErrFile, "w") as file:
# for item in fpevents:
# file.write(item + "\n")
# bcdSet = set(bcdList)
one2nErrFile = os.path.join(evtPaths, "one_2_Small_n_Error.txt")
with open(one2nErrFile, "w") as file:
for item in fnevents:
file.write(item + "\n")
one2NErrFile = os.path.join(evtPaths, "one_2_Big_N_Error.txt")
with open(one2NErrFile, "w") as file:
for item in fn_events:
file.write(item + "\n")
print('Done!')
if __name__ == "__main__":
evtpaths = r"\\192.168.1.28\share\测试视频数据以及日志\全实时测试\V12\2025-3-3"
contrast_pr(evtpaths)