# -*- coding: utf-8 -*- """ Created on Fri Jul 5 13:59:21 2024 函数 读取文件 extract_data() 0/1_track.data read_tracking_output() 0/1_tracking_output.data read_similar() process.data @author: ym """ import numpy as np import re import os from collections import OrderedDict import warnings import matplotlib.pyplot as plt def str_to_float_arr(s): # 移除字符串末尾的逗号（如果存在） if s.endswith(','): s = s[:-1] # 使用split()方法分割字符串，然后将每个元素转化为float float_array = [float(x) for x in s.split(",")] return float_array def find_samebox_in_array(arr, target): for i, st in enumerate(arr): if st[:4] == target[:4]: return i return -1 def array2list(boxes, feats): '''boxes: [x1, y1, x2, y2, track_id, score, cls, frame_index, box_index]''' trackID = np.unique(boxes[:, 4].astype(int)) track_ids = boxes[:, 4].astype(int) lboxes = [] for t_id in trackID: idx = np.where(track_ids == t_id)[0] box = boxes[idx, :] feat = feats[idx, :] assert len(set(box[:, 7])) == len(box), "Please check!!!" lboxes.append(box) return lboxes def extract_data(datapath): ''' 0/1_track.data 数据读取 对于特征，需要构造两个对象： (1) tracking输出 boxes，依据 (fid, bid) 找到对应的 feats, tracker_feat_dict 实现较为方便 (2) frameDictList 中元素为 frameDict，输出同一接口 ''' bboxes, ffeats = [], [] trackerboxes = np.empty((0, 9), dtype=np.float64) trackerfeats = np.empty((0, 256), dtype=np.float64) boxes, feats, tboxes, tfeats = [], [], [], [] timestamps, frameIds = [], [] with open(datapath, 'r', encoding='utf-8') as lines: for line in lines: line = line.strip() # 去除行尾的换行符和可能的空白字符 if not line: # 跳过空行 continue if line.find("CameraId")>=0: if len(boxes): bboxes.append(np.array(boxes)) if len(feats): ffeats.append(np.array(feats)) if len(tboxes): trackerboxes = np.concatenate((trackerboxes, np.array(tboxes))) if len(tfeats): trackerfeats = np.concatenate((trackerfeats, np.array(tfeats))) timestamp, frameId = [int(ln.split(":")[1]) for ln in line.split(",")[1:]] timestamps.append(timestamp) frameIds.append(frameId) boxes, feats, tboxes, tfeats = [], [], [], [] if line.find("box:") >= 0 and line.find("output_box:")<0 and line.find("out_boxes")<0: box = line[line.find("box:") + 4:].strip() # if len(box)==6: boxes.append(str_to_float_arr(box)) if line.find("feat:") >= 0: feat = line[line.find("feat:") + 5:].strip() # if len(feat)==256: feats.append(str_to_float_arr(feat)) if line.find("output_box:") >= 0: assert(len(boxes)>=0 and len(boxes)==len(feats)), f"{datapath}, {datapath}, len(boxes)!=len(feats)" box = str_to_float_arr(line[line.find("output_box:") + 11:].strip()) index = find_samebox_in_array(boxes, box) if index >= 0: tboxes.append(box) # 去掉'output_box:'并去除可能的空白字符 # feat_f = str_to_float_arr(input_feats[index]) feat_f = feats[index] norm_f = np.linalg.norm(feat_f) feat_f = feat_f / norm_f tfeats.append(feat_f) if len(boxes): bboxes.append(np.array(boxes)) if len(feats): ffeats.append(np.array(feats)) if len(tboxes): trackerboxes = np.concatenate((trackerboxes, np.array(tboxes))) if len(tfeats): trackerfeats = np.concatenate((trackerfeats, np.array(tfeats))) assert(len(bboxes)==len(ffeats)), "Error at Yolo output!" assert(len(trackerboxes)==len(trackerfeats)), "Error at tracker output!" ## 生成帧为单位的特征列表 tracker_feats = {} frmIDs = np.sort(np.unique(trackerboxes[:, 7].astype(int))) for fid in frmIDs: idx = np.where(trackerboxes[:, 7] == fid)[0] boxes = trackerboxes[idx, :] feats = trackerfeats[idx, :] for i in range(len(boxes)): f, b = int(boxes[i, 7]), int(boxes[i, 8]) tracker_feats.update({f"{f}_{b}": feats[i, :]}) boxes, trackingboxes= [], [] tracking_flag = False with open(datapath, 'r', encoding='utf-8') as lines: for line in lines: line = line.strip() # 去除行尾的换行符和可能的空白字符 if not line: # 跳过空行 tracking_flag = False continue if tracking_flag: if line.find("tracking_") >= 0: tracking_flag = False else: box = str_to_float_arr(line) boxes.append(box) if line.find("tracking_") >= 0: tracking_flag = True if len(boxes): trackingboxes.append(np.array(boxes)) boxes = [] if len(boxes): trackingboxes.append(np.array(boxes)) # tracking_feat_dict = {} # try: # for i, boxes in enumerate(trackingboxes): # for box in boxes: # tid, fid, bid = int(box[4]), int(box[7]), int(box[8]) # if f"track_{tid}" not in tracking_feat_dict: # tracking_feat_dict[f"track_{tid}"]= {"feats": {}} # tracking_feat_dict[f"track_{tid}"]["feats"].update({f"{fid}_{bid}": tracker_feat_dict[f"frame_{fid}"]["feats"][bid]}) # except Exception as e: # print(f'Path: {datapath}, tracking_feat_dict can not be structured correcttly, Error: {e}') # tracker_feat_dict = {} # tracker_feat_dict["timestamps"] = timestamps # tracker_feat_dict["frameIds"] = frameIds # for i in range(len(trackerboxes)): # fid, bid = int(trackerboxes[i, 7]), int(trackerboxes[i, 8]) # if f"frame_{fid}" not in tracker_feat_dict: # tracker_feat_dict[f"frame_{fid}"]= {"feats": {}} # tracker_feat_dict[f"frame_{fid}"]["feats"].update({bid: trackerfeats[i, :]}) trackingfeats = [] try: for i, boxes in enumerate(trackingboxes): feats = [] for box in boxes: fid, bid = int(box[7]), int(box[8]) feat = tracker_feats[f"{fid}_{bid}"] feats.append(feat) trackingfeats.append(np.array(feats)) except Exception as e: print(f'Path: {datapath}, trackingfeats can not be structured correcttly, Error: {e}') # return bboxes, ffeats, trackerboxes, tracker_feat_dict, trackingboxes, trackingfeats return bboxes, ffeats, trackerboxes, tracker_feats, trackingboxes, trackingfeats def extract_data_realtime(datapath): boxes, feats = [], [] tracker_feats = {} with open(datapath, 'r', encoding='utf-8') as lines: for line in lines: line = line.strip() # 去除行尾的换行符和可能的空白字符 if not line: # 跳过空行 continue if line.endswith(','): line = line[:-1] ftlist = [float(x) for x in line.split()] if len(ftlist) != 265: continue boxes.append(ftlist[:9]) feats.append(ftlist[9:]) trackerboxes = np.array(boxes) trackerfeats = np.array(feats) if len(trackerboxes)==0 or len(trackerboxes) != len(trackerfeats): return np.array([]), {} frmIDs = np.sort(np.unique(trackerboxes[:, 7].astype(int))) for fid in frmIDs: idx = np.where(trackerboxes[:, 7] == fid)[0] box = trackerboxes[idx, :] feat = trackerfeats[idx, :] for i in range(len(box)): f, b = int(box[i, 7]), int(box[i, 8]) tracker_feats.update({f"{f}_{b}": feat[i, :]}) return trackerboxes, tracker_feats def read_tracking_output_realtime(datapath): trackingboxes, trackingfeats = [], [] tracking_outboxes, tracking_outfeats = [], [] with open(datapath, 'r', encoding='utf-8') as lines: boxes, feats = [], [] Flag = False for line in lines: line = line.strip() # 去除行尾的换行符和可能的空白字符 if not line: # 跳过空行 continue if line.endswith(','): line = line[:-1] ftlist = [float(x) for x in line.split()] if len(ftlist) != 265: continue Flag = all(elem == 0 for elem in ftlist) if Flag: trackingboxes.append(np.array(boxes)) trackingfeats.append(np.array(feats)) boxes, feats = [], [] continue boxes.append(ftlist[:9]) feats.append(ftlist[9:]) if len(boxes): trackingboxes.append(np.array(boxes)) trackingfeats.append(np.array(feats)) if len(trackingboxes): tracking_outboxes = trackingboxes[:1] tracking_outfeats = trackingfeats[:1] return trackingboxes, trackingfeats, tracking_outboxes, tracking_outfeats def read_tracking_output(filepath): ''' 0/1_tracking_output.data 数据读取 ''' boxes = [] feats = [] if not os.path.isfile(filepath): return boxes, feats with open(filepath, 'r', encoding='utf-8') as file: for line in file: line = line.strip() # 去除行尾的换行符和可能的空白字符 if not line: continue if line.find("gift")>0: continue if line.endswith(','): line = line[:-1] data = np.array([float(x) for x in line.split(",")]) if data.size == 9: boxes.append(data) if data.size == 256: feats.append(data) if len(feats) != len(boxes) or len(boxes)==0: return [], [] return [np.array(boxes)], [np.array(feats)] def read_process(filePath): timeDict = {} with open(filePath, 'r', encoding='utf-8') as f: lines = f.readlines() clines = [line.strip().replace("'", '').replace('"', '') for line in lines] for i, line in enumerate(clines): line = line.strip() if line.endswith(','): line = line[:-1] if not line: continue lnList = line.split(":") if line.find("eventStart")>=0: timeDict["eventStart"] = int(lnList[-1]) if line.find("eventEnd")>=0: timeDict["eventEnd"] = int(lnList[-1]) if line.find("weightValue")>=0: timeDict["weightValue"] = int(lnList[-1]) return timeDict def read_similar(filePath): '''1:n时 Dict['type']字段提取和非全实时不一致，无 "=" 字符 ''' SimiDict = {} SimiDict['one2one'] = [] SimiDict['one2SN'] = [] SimiDict['one2n'] = [] with open(filePath, 'r', encoding='utf-8') as f: lines = f.readlines() clean_lines = [line.strip().replace("'", '').replace('"', '') for line in lines] one2one_list, one2SN_list, one2n_list = [], [], [] Flag_1to1, Flag_1toSN, Flag_1ton = False, False, False for i, line in enumerate(clean_lines): line = line.strip() if line.endswith(','): line = line[:-1] Dict = {} if not line: if len(one2one_list): SimiDict['one2one'] = one2one_list if len(one2SN_list): SimiDict['one2SN'] = one2SN_list if len(one2n_list): SimiDict['one2n'] = one2n_list one2one_list, one2SN_list, one2n_list = [], [], [] Flag_1to1, Flag_1toSN, Flag_1ton = False, False, False continue if line.find('oneToOne')>=0: Flag_1to1, Flag_1toSN, Flag_1ton = True, False,False continue if line.find('oneToSN')>=0: Flag_1to1, Flag_1toSN, Flag_1ton = False, True, False continue if line.find('oneTon')>=0: Flag_1to1, Flag_1toSN, Flag_1ton = False, False, True continue if Flag_1to1: barcode = line.split(',')[0].strip() value = line.split(',')[1].split(':')[1].strip() Dict['barcode'] = barcode Dict['similar'] = float(value) one2one_list.append(Dict) continue if Flag_1toSN: barcode = line.split(',')[0].strip() value = line.split(',')[1].split(':')[1].strip() Dict['barcode'] = barcode Dict['similar'] = float(value) one2SN_list.append(Dict) continue if Flag_1ton: label = line.split(':')[0].strip() value = line.split(':')[1].strip() bcd = label.split('_')[-1] if len(bcd)<8: continue Dict['event'] = label Dict['barcode'] = bcd Dict['similar'] = float(value.split(',')[0]) Dict['type'] = value.split(',')[1] one2n_list.append(Dict) if len(one2one_list): SimiDict['one2one'] = one2one_list if len(one2n_list): SimiDict['one2n'] = one2n_list if len(one2SN_list): SimiDict['one2SN'] = one2SN_list return SimiDict def read_weight_sensor(filepath): WeightDict = OrderedDict() with open(filepath, 'r', encoding='utf-8') as f: lines = f.readlines() clean_lines = [line.strip().replace("'", '').replace('"', '') for line in lines] for i, line in enumerate(clean_lines): line = line.strip() if line.find(':') < 0: continue if line.find("Weight") >= 0: label = "Weight" continue keyword = line.split(':')[0] value = line.split(':')[1] if label == "Weight": vdata = [float(s) for s in value.split(',') if len(s) and s.isdigit()] WeightDict[keyword] = vdata[-1] weights = [(float(t), w) for t, w in WeightDict.items()] weights = np.array(weights).astype(np.int64) return weights def read_weight_timeConsuming(filePth): WeightDict, SensorDict, ProcessTimeDict = OrderedDict(), OrderedDict(), OrderedDict() with open(filePth, 'r', encoding='utf-8') as f: lines = f.readlines() # label = '' for i, line in enumerate(lines): line = line.strip() if line.find(':') < 0: continue if line.find("Weight") >= 0: label = "Weight" continue if line.find("Sensor") >= 0: label = "Sensor" continue if line.find("processTime") >= 0: label = "ProcessTime" continue keyword = line.split(':')[0] value = line.split(':')[1] if label == "Weight": WeightDict[keyword] = float(value.strip(',')) if label == "Sensor": SensorDict[keyword] = [float(s) for s in value.split(',') if len(s)] if label == "ProcessTime": ProcessTimeDict[keyword] = float(value.strip(',')) # print("Done!") return WeightDict, SensorDict, ProcessTimeDict def read_deletedBarcode_file(filePath): with open(filePath, 'r', encoding='utf-8') as f: lines = f.readlines() split_flag, all_list = False, [] dict, barcode_list, similarity_list = {}, [], [] clean_lines = [line.strip().replace("'", '').replace('"', '') for line in lines] for i, line in enumerate(clean_lines): if line.endswith(','): line = line[:-1] stripped_line = line.strip() if not stripped_line: if len(barcode_list): dict['barcode'] = barcode_list if len(similarity_list): dict['similarity'] = similarity_list if len(dict): all_list.append(dict) split_flag = False dict, barcode_list, similarity_list = {}, [], [] continue if line.find(':')<0: continue label = line.split(':')[0] value = line.split(':')[1] if label == 'SeqDir': dict['SeqDir'] = value dict['filetype'] = "deletedBarcode" if label == 'Deleted': dict['Deleted'] = value if label == 'List': split_flag = True continue if split_flag: barcode_list.append(label) similarity_list.append(value) if len(barcode_list): dict['barcode'] = barcode_list if len(similarity_list): dict['similarity'] = similarity_list if len(dict): all_list.append(dict) return all_list def read_returnGoods_file(filePath): ''' 20241030开始，原 deletedBarcode.txt 中数据格式修改为 returnGoods.txt，读数方式随之变化 ''' with open(filePath, 'r', encoding='utf-8') as f: lines = f.readlines() clean_lines = [line.strip().replace("'", '').replace('"', '') for line in lines] all_list = [] split_flag, dict = False, {} barcode_list, similarity_list = [], [] event_list, type_list = [], [] for i, line in enumerate(clean_lines): stripped_line = line.strip() if line.endswith(','): line = line[:-1] if not stripped_line: if len(barcode_list): dict['barcode'] = barcode_list if len(similarity_list): dict['similarity'] = similarity_list if len(event_list): dict['event'] = event_list if len(type_list): dict['type'] = type_list if len(dict) and dict['SeqDir'].find('*')<0: all_list.append(dict) split_flag, dict = False, {} barcode_list, similarity_list = [], [] event_list, type_list = [], [] continue if line.find(':')<0: continue if line.find('1:n')==0: continue label = line.split(':')[0].strip() value = line.split(':')[1].strip() if label == 'SeqDir': dict['SeqDir'] = value dict['Deleted'] = value.split('_')[-1] dict['filetype'] = "returnGoods" if label == 'List': split_flag = True continue if split_flag: bcd = label.split('_')[-1] if len(bcd)<8: continue # event_list.append(label + '_' + bcd) event_list.append(label) barcode_list.append(bcd) similarity_list.append(value.split(',')[0]) type_list.append(value.split('=')[-1]) if len(barcode_list): dict['barcode'] = barcode_list if len(similarity_list): dict['similarity'] = similarity_list if len(event_list): dict['event'] = event_list if len(type_list): dict['type'] = type_list if len(dict) and dict['SeqDir'].find('*')<0: all_list.append(dict) return all_list def plot_sensor_curve(WeightDict, SensorDict, ProcessTimeDict): wtime, wdata = [], [] stime, sdata = [], [] for key, value in WeightDict.items(): wtime.append(int(key)) wdata.append(value) for key, value in SensorDict.items(): if len(value) != 9: continue stime.append(int(key)) sdata.append(np.array(value)) static_range = [] dynamic_range = [] windth = 8 nw = len(wdata) assert(nw) >= 8, "The num of weight data is less than 8!" # i1, i2 = 0, 7 # while i2 < nw: # data = wdata[i1:(i2+1)] # max(data) - min(data) # if i2<7: # i1 = 0 # else: # i1 = i2-windth min_t = min(wtime + stime) wtime = [t-min_t for t in wtime] stime = [t-min_t for t in stime] max_t = max(wtime + stime) fig = plt.figure(figsize=(16, 12)) gs = fig.add_gridspec(2, 1, left=0.1, right=0.9, bottom=0.1, top=0.9, wspace=0.05, hspace=0.15) # ax1, ax2 = axs ax1 = fig.add_subplot(gs[0,0]) ax2 = fig.add_subplot(gs[1,0]) ax1.plot(wtime, wdata, 'b--', linewidth=2 ) for i in range(9): ydata = [s[i] for s in sdata] ax2.plot(stime, ydata, linewidth=2 ) ax1.grid(True), ax1.set_xlim(0, max_t), ax1.set_title('Weight') ax1.set_label("(Time: ms)") # ax1.legend() ax2.grid(True), ax2.set_xlim(0, max_t), ax2.set_title('IMU') # ax2.legend() plt.show() def test_process(file_path): WeightDict, SensorDict, ProcessTimeDict = read_weight_timeConsuming(file_path) plot_sensor_curve(WeightDict, SensorDict, ProcessTimeDict) def main(): files_path = r'\\192.168.1.28\share\测试_202406\0814\0814\20240814-102227-62264578-a720-4eb9-b95e-cb8be009aa98_null' k = 0 for filename in os.listdir(files_path): filename = 'process.data' file_path = os.path.join(files_path, filename) if os.path.isfile(file_path) and filename.find("track.data")>0: extract_data(file_path) if os.path.isfile(file_path) and filename.find("process.data")>=0: test_process(file_path) k += 1 if k == 1: break def main1(): fpath = r'\\192.168.1.28\share\测试视频数据以及日志\各模块测试记录\比对测试\1209永辉超市测试\20241209-155924-117e1941-70f8-4287-8de1-4866868548a6_6926475209967\process.data' simidct = read_similar(fpath) print(simidct) if __name__ == "__main__": main1()